├── .gitignore ├── LICENSE.txt ├── README.md ├── datasets ├── bAbI │ ├── LICENSE.txt │ ├── README.txt │ └── en │ │ ├── qa10_indefinite-knowledge_test.txt │ │ ├── qa10_indefinite-knowledge_train.txt │ │ ├── qa11_basic-coreference_test.txt │ │ ├── qa11_basic-coreference_train.txt │ │ ├── qa12_conjunction_test.txt │ │ ├── qa12_conjunction_train.txt │ │ ├── qa13_compound-coreference_test.txt │ │ ├── qa13_compound-coreference_train.txt │ │ ├── qa14_time-reasoning_test.txt │ │ ├── qa14_time-reasoning_train.txt │ │ ├── qa15_basic-deduction_test.txt │ │ ├── qa15_basic-deduction_train.txt │ │ ├── qa16_basic-induction_test.txt │ │ ├── qa16_basic-induction_train.txt │ │ ├── qa17_positional-reasoning_test.txt │ │ ├── qa17_positional-reasoning_train.txt │ │ ├── qa18_size-reasoning_test.txt │ │ ├── qa18_size-reasoning_train.txt │ │ ├── qa19_path-finding_test.txt │ │ ├── qa19_path-finding_train.txt │ │ ├── qa1_single-supporting-fact_test.txt │ │ ├── qa1_single-supporting-fact_train.txt │ │ ├── qa20_agents-motivations_test.txt │ │ ├── qa20_agents-motivations_train.txt │ │ ├── qa2_two-supporting-facts_test.txt │ │ ├── qa2_two-supporting-facts_train.txt │ │ ├── qa3_three-supporting-facts_test.txt │ │ ├── qa3_three-supporting-facts_train.txt │ │ ├── qa4_two-arg-relations_test.txt │ │ ├── qa4_two-arg-relations_train.txt │ │ ├── qa5_three-arg-relations_test.txt │ │ ├── qa5_three-arg-relations_train.txt │ │ ├── qa6_yes-no-questions_test.txt │ │ ├── qa6_yes-no-questions_train.txt │ │ ├── qa7_counting_test.txt │ │ ├── qa7_counting_train.txt │ │ ├── qa8_lists-sets_test.txt │ │ ├── qa8_lists-sets_train.txt │ │ ├── qa9_simple-negation_test.txt │ │ └── qa9_simple-negation_train.txt └── text │ ├── CallOfCthulhu.txt │ ├── LilDicky.txt │ ├── PaulGraham.txt │ └── PaulGraham2.txt ├── saved_models ├── CallOfCthulhu.ser ├── LilDicky.ser ├── PaulGraham.ser └── PaulGraham2.ser └── src ├── ExampleEmbeddedReberGrammar.java ├── ExampleLilDicky.java ├── ExamplePaulGraham.java ├── ExampleQuestionAnswering.java ├── autodiff └── Graph.java ├── datasets ├── EmbeddedReberGrammar.java ├── SequentialParity.java ├── TextGeneration.java ├── TextGenerationUnbroken.java └── bAbI.java ├── datastructs ├── DataSequence.java ├── DataSet.java └── DataStep.java ├── loss ├── Loss.java ├── LossArgMax.java ├── LossMultiDimensionalBinary.java ├── LossSoftmax.java └── LossSumOfSquares.java ├── matrix └── Matrix.java ├── model ├── FeedForwardLayer.java ├── GruLayer.java ├── LinearLayer.java ├── LinearUnit.java ├── LstmLayer.java ├── Model.java ├── NeuralNetwork.java ├── Nonlinearity.java ├── RectifiedLinearUnit.java ├── RnnLayer.java ├── SigmoidUnit.java ├── SineUnit.java └── TanhUnit.java ├── trainer └── Trainer.java └── util ├── FileIO.java ├── NeuralNetworkHelper.java └── Util.java /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | .settings 4 | *.DS_Store 5 | bin/ 6 | src/experimental/ 7 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Thomas Lahore 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # RecurrentJava 3 | 4 | RecurrentJava is a reimplementation of Andrej Karpathy's [RecurrentJS](https://github.com/karpathy/recurrentjs), in Java. 5 | 6 | It currently features: 7 | 8 | - Deep Recurrent Neural Networks 9 | - Long Short-Term Memory Networks 10 | - Gated Recurrent Unit Neural Networks 11 | - Backpropagation Through Time handled via Automatic Differentiation. 12 | 13 | ExamplePaulGraham.java shows how to do character-by-character sentence prediction and generation. 14 | 15 | Sample output: 16 | 17 | ``` 18 | ======================================== 19 | REPORT: 20 | 21 | calculating perplexity over entire data set... 22 | 23 | Median Perplexity = 1.4959 24 | 25 | Temperature 1.0 prediction: 26 | "there's a more kemmaces of meanness that hade? tagh o; mool" 27 | "it fart dect about twish i could see gve..." 28 | 29 | Temperature 0.75 prediction: 30 | "that's not absolutely note a lot of the startup? path they'll should owt" 31 | "i realize how crazy all thi..." 32 | 33 | Temperature 0.5 prediction: 34 | "the most stripiess to more here that happens never get them" 35 | "if you do that role kropate that's the w..." 36 | 37 | Temperature 0.25 prediction: 38 | "the person who needs something making the same spignf befart" 39 | "the startup founders who never about wh..." 40 | 41 | Temperature 0.1 prediction: 42 | "the startup founders who never about which in your expanding, it's a sign when idea way we don't the..." 43 | 44 | Argmax prediction: 45 | "the problem is not that most towns kill startups" 46 | "the problem is not that most towns kill startups" 47 | "th..." 48 | ======================================== 49 | ``` 50 | 51 | ## License 52 | MIT -------------------------------------------------------------------------------- /datasets/bAbI/LICENSE.txt: -------------------------------------------------------------------------------- 1 | CC License 2 | 3 | bAbI tasks data 4 | 5 | Copyright (c) 2015-present, Facebook, Inc. All rights reserved. 6 | 7 | Creative Commons Legal Code 8 | 9 | Attribution 3.0 Unported 10 | 11 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 12 | LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN 13 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 14 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 15 | REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR 16 | DAMAGES RESULTING FROM ITS USE. 17 | 18 | License 19 | 20 | THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE 21 | COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY 22 | COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS 23 | AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. 24 | 25 | BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE 26 | TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY 27 | BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS 28 | CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND 29 | CONDITIONS. 30 | 31 | 1. Definitions 32 | 33 | a. "Adaptation" means a work based upon the Work, or upon the Work and 34 | other pre-existing works, such as a translation, adaptation, 35 | derivative work, arrangement of music or other alterations of a 36 | literary or artistic work, or phonogram or performance and includes 37 | cinematographic adaptations or any other form in which the Work may be 38 | recast, transformed, or adapted including in any form recognizably 39 | derived from the original, except that a work that constitutes a 40 | Collection will not be considered an Adaptation for the purpose of 41 | this License. For the avoidance of doubt, where the Work is a musical 42 | work, performance or phonogram, the synchronization of the Work in 43 | timed-relation with a moving image ("synching") will be considered an 44 | Adaptation for the purpose of this License. 45 | b. "Collection" means a collection of literary or artistic works, such as 46 | encyclopedias and anthologies, or performances, phonograms or 47 | broadcasts, or other works or subject matter other than works listed 48 | in Section 1(f) below, which, by reason of the selection and 49 | arrangement of their contents, constitute intellectual creations, in 50 | which the Work is included in its entirety in unmodified form along 51 | with one or more other contributions, each constituting separate and 52 | independent works in themselves, which together are assembled into a 53 | collective whole. A work that constitutes a Collection will not be 54 | considered an Adaptation (as defined above) for the purposes of this 55 | License. 56 | c. "Distribute" means to make available to the public the original and 57 | copies of the Work or Adaptation, as appropriate, through sale or 58 | other transfer of ownership. 59 | d. "Licensor" means the individual, individuals, entity or entities that 60 | offer(s) the Work under the terms of this License. 61 | e. "Original Author" means, in the case of a literary or artistic work, 62 | the individual, individuals, entity or entities who created the Work 63 | or if no individual or entity can be identified, the publisher; and in 64 | addition (i) in the case of a performance the actors, singers, 65 | musicians, dancers, and other persons who act, sing, deliver, declaim, 66 | play in, interpret or otherwise perform literary or artistic works or 67 | expressions of folklore; (ii) in the case of a phonogram the producer 68 | being the person or legal entity who first fixes the sounds of a 69 | performance or other sounds; and, (iii) in the case of broadcasts, the 70 | organization that transmits the broadcast. 71 | f. "Work" means the literary and/or artistic work offered under the terms 72 | of this License including without limitation any production in the 73 | literary, scientific and artistic domain, whatever may be the mode or 74 | form of its expression including digital form, such as a book, 75 | pamphlet and other writing; a lecture, address, sermon or other work 76 | of the same nature; a dramatic or dramatico-musical work; a 77 | choreographic work or entertainment in dumb show; a musical 78 | composition with or without words; a cinematographic work to which are 79 | assimilated works expressed by a process analogous to cinematography; 80 | a work of drawing, painting, architecture, sculpture, engraving or 81 | lithography; a photographic work to which are assimilated works 82 | expressed by a process analogous to photography; a work of applied 83 | art; an illustration, map, plan, sketch or three-dimensional work 84 | relative to geography, topography, architecture or science; a 85 | performance; a broadcast; a phonogram; a compilation of data to the 86 | extent it is protected as a copyrightable work; or a work performed by 87 | a variety or circus performer to the extent it is not otherwise 88 | considered a literary or artistic work. 89 | g. "You" means an individual or entity exercising rights under this 90 | License who has not previously violated the terms of this License with 91 | respect to the Work, or who has received express permission from the 92 | Licensor to exercise rights under this License despite a previous 93 | violation. 94 | h. "Publicly Perform" means to perform public recitations of the Work and 95 | to communicate to the public those public recitations, by any means or 96 | process, including by wire or wireless means or public digital 97 | performances; to make available to the public Works in such a way that 98 | members of the public may access these Works from a place and at a 99 | place individually chosen by them; to perform the Work to the public 100 | by any means or process and the communication to the public of the 101 | performances of the Work, including by public digital performance; to 102 | broadcast and rebroadcast the Work by any means including signs, 103 | sounds or images. 104 | i. "Reproduce" means to make copies of the Work by any means including 105 | without limitation by sound or visual recordings and the right of 106 | fixation and reproducing fixations of the Work, including storage of a 107 | protected performance or phonogram in digital form or other electronic 108 | medium. 109 | 110 | 2. Fair Dealing Rights. Nothing in this License is intended to reduce, 111 | limit, or restrict any uses free from copyright or rights arising from 112 | limitations or exceptions that are provided for in connection with the 113 | copyright protection under copyright law or other applicable laws. 114 | 115 | 3. License Grant. Subject to the terms and conditions of this License, 116 | Licensor hereby grants You a worldwide, royalty-free, non-exclusive, 117 | perpetual (for the duration of the applicable copyright) license to 118 | exercise the rights in the Work as stated below: 119 | 120 | a. to Reproduce the Work, to incorporate the Work into one or more 121 | Collections, and to Reproduce the Work as incorporated in the 122 | Collections; 123 | b. to create and Reproduce Adaptations provided that any such Adaptation, 124 | including any translation in any medium, takes reasonable steps to 125 | clearly label, demarcate or otherwise identify that changes were made 126 | to the original Work. For example, a translation could be marked "The 127 | original work was translated from English to Spanish," or a 128 | modification could indicate "The original work has been modified."; 129 | c. to Distribute and Publicly Perform the Work including as incorporated 130 | in Collections; and, 131 | d. to Distribute and Publicly Perform Adaptations. 132 | e. For the avoidance of doubt: 133 | 134 | i. Non-waivable Compulsory License Schemes. In those jurisdictions in 135 | which the right to collect royalties through any statutory or 136 | compulsory licensing scheme cannot be waived, the Licensor 137 | reserves the exclusive right to collect such royalties for any 138 | exercise by You of the rights granted under this License; 139 | ii. Waivable Compulsory License Schemes. In those jurisdictions in 140 | which the right to collect royalties through any statutory or 141 | compulsory licensing scheme can be waived, the Licensor waives the 142 | exclusive right to collect such royalties for any exercise by You 143 | of the rights granted under this License; and, 144 | iii. Voluntary License Schemes. The Licensor waives the right to 145 | collect royalties, whether individually or, in the event that the 146 | Licensor is a member of a collecting society that administers 147 | voluntary licensing schemes, via that society, from any exercise 148 | by You of the rights granted under this License. 149 | 150 | The above rights may be exercised in all media and formats whether now 151 | known or hereafter devised. The above rights include the right to make 152 | such modifications as are technically necessary to exercise the rights in 153 | other media and formats. Subject to Section 8(f), all rights not expressly 154 | granted by Licensor are hereby reserved. 155 | 156 | 4. Restrictions. The license granted in Section 3 above is expressly made 157 | subject to and limited by the following restrictions: 158 | 159 | a. You may Distribute or Publicly Perform the Work only under the terms 160 | of this License. You must include a copy of, or the Uniform Resource 161 | Identifier (URI) for, this License with every copy of the Work You 162 | Distribute or Publicly Perform. You may not offer or impose any terms 163 | on the Work that restrict the terms of this License or the ability of 164 | the recipient of the Work to exercise the rights granted to that 165 | recipient under the terms of the License. You may not sublicense the 166 | Work. You must keep intact all notices that refer to this License and 167 | to the disclaimer of warranties with every copy of the Work You 168 | Distribute or Publicly Perform. When You Distribute or Publicly 169 | Perform the Work, You may not impose any effective technological 170 | measures on the Work that restrict the ability of a recipient of the 171 | Work from You to exercise the rights granted to that recipient under 172 | the terms of the License. This Section 4(a) applies to the Work as 173 | incorporated in a Collection, but this does not require the Collection 174 | apart from the Work itself to be made subject to the terms of this 175 | License. If You create a Collection, upon notice from any Licensor You 176 | must, to the extent practicable, remove from the Collection any credit 177 | as required by Section 4(b), as requested. If You create an 178 | Adaptation, upon notice from any Licensor You must, to the extent 179 | practicable, remove from the Adaptation any credit as required by 180 | Section 4(b), as requested. 181 | b. If You Distribute, or Publicly Perform the Work or any Adaptations or 182 | Collections, You must, unless a request has been made pursuant to 183 | Section 4(a), keep intact all copyright notices for the Work and 184 | provide, reasonable to the medium or means You are utilizing: (i) the 185 | name of the Original Author (or pseudonym, if applicable) if supplied, 186 | and/or if the Original Author and/or Licensor designate another party 187 | or parties (e.g., a sponsor institute, publishing entity, journal) for 188 | attribution ("Attribution Parties") in Licensor's copyright notice, 189 | terms of service or by other reasonable means, the name of such party 190 | or parties; (ii) the title of the Work if supplied; (iii) to the 191 | extent reasonably practicable, the URI, if any, that Licensor 192 | specifies to be associated with the Work, unless such URI does not 193 | refer to the copyright notice or licensing information for the Work; 194 | and (iv) , consistent with Section 3(b), in the case of an Adaptation, 195 | a credit identifying the use of the Work in the Adaptation (e.g., 196 | "French translation of the Work by Original Author," or "Screenplay 197 | based on original Work by Original Author"). The credit required by 198 | this Section 4 (b) may be implemented in any reasonable manner; 199 | provided, however, that in the case of a Adaptation or Collection, at 200 | a minimum such credit will appear, if a credit for all contributing 201 | authors of the Adaptation or Collection appears, then as part of these 202 | credits and in a manner at least as prominent as the credits for the 203 | other contributing authors. For the avoidance of doubt, You may only 204 | use the credit required by this Section for the purpose of attribution 205 | in the manner set out above and, by exercising Your rights under this 206 | License, You may not implicitly or explicitly assert or imply any 207 | connection with, sponsorship or endorsement by the Original Author, 208 | Licensor and/or Attribution Parties, as appropriate, of You or Your 209 | use of the Work, without the separate, express prior written 210 | permission of the Original Author, Licensor and/or Attribution 211 | Parties. 212 | c. Except as otherwise agreed in writing by the Licensor or as may be 213 | otherwise permitted by applicable law, if You Reproduce, Distribute or 214 | Publicly Perform the Work either by itself or as part of any 215 | Adaptations or Collections, You must not distort, mutilate, modify or 216 | take other derogatory action in relation to the Work which would be 217 | prejudicial to the Original Author's honor or reputation. Licensor 218 | agrees that in those jurisdictions (e.g. Japan), in which any exercise 219 | of the right granted in Section 3(b) of this License (the right to 220 | make Adaptations) would be deemed to be a distortion, mutilation, 221 | modification or other derogatory action prejudicial to the Original 222 | Author's honor and reputation, the Licensor will waive or not assert, 223 | as appropriate, this Section, to the fullest extent permitted by the 224 | applicable national law, to enable You to reasonably exercise Your 225 | right under Section 3(b) of this License (right to make Adaptations) 226 | but not otherwise. 227 | 228 | 5. Representations, Warranties and Disclaimer 229 | 230 | UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR 231 | OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY 232 | KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, 233 | INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, 234 | FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF 235 | LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, 236 | WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION 237 | OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. 238 | 239 | 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE 240 | LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR 241 | ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES 242 | ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS 243 | BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 244 | 245 | 7. Termination 246 | 247 | a. This License and the rights granted hereunder will terminate 248 | automatically upon any breach by You of the terms of this License. 249 | Individuals or entities who have received Adaptations or Collections 250 | from You under this License, however, will not have their licenses 251 | terminated provided such individuals or entities remain in full 252 | compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will 253 | survive any termination of this License. 254 | b. Subject to the above terms and conditions, the license granted here is 255 | perpetual (for the duration of the applicable copyright in the Work). 256 | Notwithstanding the above, Licensor reserves the right to release the 257 | Work under different license terms or to stop distributing the Work at 258 | any time; provided, however that any such election will not serve to 259 | withdraw this License (or any other license that has been, or is 260 | required to be, granted under the terms of this License), and this 261 | License will continue in full force and effect unless terminated as 262 | stated above. 263 | 264 | 8. Miscellaneous 265 | 266 | a. Each time You Distribute or Publicly Perform the Work or a Collection, 267 | the Licensor offers to the recipient a license to the Work on the same 268 | terms and conditions as the license granted to You under this License. 269 | b. Each time You Distribute or Publicly Perform an Adaptation, Licensor 270 | offers to the recipient a license to the original Work on the same 271 | terms and conditions as the license granted to You under this License. 272 | c. If any provision of this License is invalid or unenforceable under 273 | applicable law, it shall not affect the validity or enforceability of 274 | the remainder of the terms of this License, and without further action 275 | by the parties to this agreement, such provision shall be reformed to 276 | the minimum extent necessary to make such provision valid and 277 | enforceable. 278 | d. No term or provision of this License shall be deemed waived and no 279 | breach consented to unless such waiver or consent shall be in writing 280 | and signed by the party to be charged with such waiver or consent. 281 | e. This License constitutes the entire agreement between the parties with 282 | respect to the Work licensed here. There are no understandings, 283 | agreements or representations with respect to the Work not specified 284 | here. Licensor shall not be bound by any additional provisions that 285 | may appear in any communication from You. This License may not be 286 | modified without the mutual written agreement of the Licensor and You. 287 | f. The rights granted under, and the subject matter referenced, in this 288 | License were drafted utilizing the terminology of the Berne Convention 289 | for the Protection of Literary and Artistic Works (as amended on 290 | September 28, 1979), the Rome Convention of 1961, the WIPO Copyright 291 | Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 292 | and the Universal Copyright Convention (as revised on July 24, 1971). 293 | These rights and subject matter take effect in the relevant 294 | jurisdiction in which the License terms are sought to be enforced 295 | according to the corresponding provisions of the implementation of 296 | those treaty provisions in the applicable national law. If the 297 | standard suite of rights granted under applicable copyright law 298 | includes additional rights not granted under this License, such 299 | additional rights are deemed to be included in the License; this 300 | License is not intended to restrict the license of any rights under 301 | applicable law. 302 | 303 | 304 | Creative Commons Notice 305 | 306 | Creative Commons is not a party to this License, and makes no warranty 307 | whatsoever in connection with the Work. Creative Commons will not be 308 | liable to You or any party on any legal theory for any damages 309 | whatsoever, including without limitation any general, special, 310 | incidental or consequential damages arising in connection to this 311 | license. Notwithstanding the foregoing two (2) sentences, if Creative 312 | Commons has expressly identified itself as the Licensor hereunder, it 313 | shall have all rights and obligations of Licensor. 314 | 315 | Except for the limited purpose of indicating to the public that the 316 | Work is licensed under the CCPL, Creative Commons does not authorize 317 | the use by either party of the trademark "Creative Commons" or any 318 | related trademark or logo of Creative Commons without the prior 319 | written consent of Creative Commons. Any permitted use will be in 320 | compliance with Creative Commons' then-current trademark usage 321 | guidelines, as may be published on its website or otherwise made 322 | available upon request from time to time. For the avoidance of doubt, 323 | this trademark restriction does not form part of this License. 324 | 325 | Creative Commons may be contacted at https://creativecommons.org/. 326 | -------------------------------------------------------------------------------- /datasets/bAbI/README.txt: -------------------------------------------------------------------------------- 1 | Towards AI Complete Question Answering: A Set of Prerequisite Toy Tasks 2 | ----------------------------------------------------------------------- 3 | In this directory is the first set of 20 tasks for testing text understanding and reasoning in the bAbI project. 4 | The aim is that each task tests a unique aspect of text and reasoning, and hence test different capabilities of learning models. More tasks are planned in the future to capture more aspects. 5 | 6 | For each task, there are 1000 questions for training, and 1000 for testing. 7 | However, we emphasize that the goal is still to use as little data as possible to do well on the task (i.e. if you can use less than 1000 that's even better) -- and without resorting to engineering task-specific tricks that will not generalize to other tasks, as they may not be of much use subsequently. Note that the aim during evaluation is to use the _same_ learner across all tasks to evaluate its skills and capabilities. 8 | Further while the MemNN results in the paper use full supervision (including of the supporting facts) results with weak supervision would also be ultimately preferable as this kind of data is easier to collect. Hence results of that form are very welcome. 9 | 10 | For the reasons above there are currently several directories: 11 | 12 | 1) en/ -- the tasks in English, readable by humans. 13 | 2) hn/ -- the tasks in Hindi, readable by humans. 14 | 3) shuffled/ -- the same tasks with shuffled letters so they are not readable by humans, and for existing parsers and taggers cannot be used in a straight-forward fashion to leverage extra resources-- in this case the learner is more forced to rely on the given training data. This mimics a learner being first presented a language and having to learn from scratch. 15 | 4) en-10k/ shuffled-10k/ and hn-10k/ -- the same tasks in the three formats, but with 10,000 training examples, rather than 1000 training examples. 16 | 17 | The file format for each task is as follows: 18 | ID text 19 | ID text 20 | ID text 21 | ID question[tab]answer[tab]supporting fact IDS. 22 | ... 23 | 24 | The IDs for a given "story" start at 1 and increase. 25 | When the IDs in a file reset back to 1 you can consider the following sentences as a new "story". 26 | Supporting fact IDs only ever reference the sentences within a "story". 27 | 28 | For Example: 29 | 1 Mary moved to the bathroom. 30 | 2 John went to the hallway. 31 | 3 Where is Mary? bathroom 1 32 | 4 Daniel went back to the hallway. 33 | 5 Sandra moved to the garden. 34 | 6 Where is Daniel? hallway 4 35 | 7 John moved to the office. 36 | 8 Sandra journeyed to the bathroom. 37 | 9 Where is Daniel? hallway 4 38 | 10 Mary moved to the hallway. 39 | 11 Daniel travelled to the office. 40 | 12 Where is Daniel? office 11 41 | 13 John went back to the garden. 42 | 14 John moved to the bedroom. 43 | 15 Where is Sandra? bathroom 8 44 | 1 Sandra travelled to the office. 45 | 2 Sandra went to the bathroom. 46 | 3 Where is Sandra? bathroom 2 47 | 48 | Changes between versions. 49 | ========================= 50 | V1.2 (this version) - Added Hindi versions of all the tasks. Fixed some problems with task 16, and added a separate set of directories for 10k training data, as we received requests for this. 51 | V1.1 (this version) - Fixed some problems with task 3, and reduced the training set size available to 1000 as this matches the results in the paper cited above, in order to avoid confusion. -------------------------------------------------------------------------------- /datasets/text/CallOfCthulhu.txt: -------------------------------------------------------------------------------- 1 | The Call of Cthulhu 2 | By H. P. Lovecraft 3 | 4 | ------=-O-=------ 5 | (Found Among the Papers of the Late 6 | Francis Wayland Thurston, of Boston) 7 | 8 | “Of such great powers or beings there may be conceivably a survival . . . a survival of a hugely remote period when . . . consciousness was manifested, perhaps, in shapes and forms long since withdrawn before the tide of advancing humanity . . . forms of which poetry and legend alone have caught a flying memory and called them gods, monsters, mythical beings of all sorts and kinds. . . .” 9 | —Algernon Blackwood. 10 | 11 | 12 | I. 13 | The Horror in Clay. 14 | 15 | The most merciful thing in the world, I think, is the inability of the human mind to correlate all its contents. We live on a placid island of ignorance in the midst of black seas of infinity, and it was not meant that we should voyage far. The sciences, each straining in its own direction, have hitherto harmed us little; but some day the piecing together of dissociated knowledge will open up such terrifying vistas of reality, and of our frightful position therein, that we shall either go mad from the revelation or flee from the deadly light into the peace and safety of a new dark age. 16 | Theosophists have guessed at the awesome grandeur of the cosmic cycle wherein our world and human race form transient incidents. They have hinted at strange survivals in terms which would freeze the blood if not masked by a bland optimism. But it is not from them that there came the single glimpse of forbidden aeons which chills me when I think of it and maddens me when I dream of it. That glimpse, like all dread glimpses of truth, flashed out from an accidental piecing together of separated things—in this case an old newspaper item and the notes of a dead professor. I hope that no one else will accomplish this piecing out; certainly, if I live, I shall never knowingly supply a link in so hideous a chain. I think that the professor, too, intended to keep silent regarding the part he knew, and that he would have destroyed his notes had not sudden death seized him. 17 | My knowledge of the thing began in the winter of 1926–27 with the death of my grand-uncle George Gammell Angell, Professor Emeritus of Semitic Languages in Brown University, Providence, Rhode Island. Professor Angell was widely known as an authority on ancient inscriptions, and had frequently been resorted to by the heads of prominent museums; so that his passing at the age of ninety-two may be recalled by many. Locally, interest was intensified by the obscurity of the cause of death. The professor had been stricken whilst returning from the Newport boat; falling suddenly, as witnesses said, after having been jostled by a nautical-looking negro who had come from one of the queer dark courts on the precipitous hillside which formed a short cut from the waterfront to the deceased’s home in Williams Street. Physicians were unable to find any visible disorder, but concluded after perplexed debate that some obscure lesion of the heart, induced by the brisk ascent of so steep a hill by so elderly a man, was responsible for the end. At the time I saw no reason to dissent from this dictum, but latterly I am inclined to wonder—and more than wonder. 18 | As my grand-uncle’s heir and executor, for he died a childless widower, I was expected to go over his papers with some thoroughness; and for that purpose moved his entire set of files and boxes to my quarters in Boston. Much of the material which I correlated will be later published by the American Archaeological Society, but there was one box which I found exceedingly puzzling, and which I felt much averse from shewing to other eyes. It had been locked, and I did not find the key till it occurred to me to examine the personal ring which the professor carried always in his pocket. Then indeed I succeeded in opening it, but when I did so seemed only to be confronted by a greater and more closely locked barrier. For what could be the meaning of the queer clay bas-relief and the disjointed jottings, ramblings, and cuttings which I found? Had my uncle, in his latter years, become credulous of the most superficial impostures? I resolved to search out the eccentric sculptor responsible for this apparent disturbance of an old man’s peace of mind. 19 | The bas-relief was a rough rectangle less than an inch thick and about five by six inches in area; obviously of modern origin. Its designs, however, were far from modern in atmosphere and suggestion; for although the vagaries of cubism and futurism are many and wild, they do not often reproduce that cryptic regularity which lurks in prehistoric writing. And writing of some kind the bulk of these designs seemed certainly to be; though my memory, despite much familiarity with the papers and collections of my uncle, failed in any way to identify this particular species, or even to hint at its remotest affiliations. 20 | Above these apparent hieroglyphics was a figure of evidently pictorial intent, though its impressionistic execution forbade a very clear idea of its nature. It seemed to be a sort of monster, or symbol representing a monster, of a form which only a diseased fancy could conceive. If I say that my somewhat extravagant imagination yielded simultaneous pictures of an octopus, a dragon, and a human caricature, I shall not be unfaithful to the spirit of the thing. A pulpy, tentacled head surmounted a grotesque and scaly body with rudimentary wings; but it was the general outline of the whole which made it most shockingly frightful. Behind the figure was a vague suggestion of a Cyclopean architectural background. 21 | The writing accompanying this oddity was, aside from a stack of press cuttings, in Professor Angell’s most recent hand; and made no pretence to literary style. What seemed to be the main document was headed “CTHULHU CULT” in characters painstakingly printed to avoid the erroneous reading of a word so unheard-of. The manuscript was divided into two sections, the first of which was headed “1925—Dream and Dream Work of H. A. Wilcox, 7 Thomas St., Providence, R.I.”, and the second, “Narrative of Inspector John R. Legrasse, 121 Bienville St., New Orleans, La., at 1908 A. A. S. Mtg.—Notes on Same, & Prof. Webb’s Acct.” The other manuscript papers were all brief notes, some of them accounts of the queer dreams of different persons, some of them citations from theosophical books and magazines (notably W. Scott-Elliot’s Atlantis and the Lost Lemuria), and the rest comments on long-surviving secret societies and hidden cults, with references to passages in such mythological and anthropological source-books as Frazer’s Golden Bough and Miss Murray’s Witch-Cult in Western Europe. The cuttings largely alluded to outré mental illnesses and outbreaks of group folly or mania in the spring of 1925. 22 | The first half of the principal manuscript told a very peculiar tale. It appears that on March 1st, 1925, a thin, dark young man of neurotic and excited aspect had called upon Professor Angell bearing the singular clay bas-relief, which was then exceedingly damp and fresh. His card bore the name of Henry Anthony Wilcox, and my uncle had recognised him as the youngest son of an excellent family slightly known to him, who had latterly been studying sculpture at the Rhode Island School of Design and living alone at the Fleur-de-Lys Building near that institution. Wilcox was a precocious youth of known genius but great eccentricity, and had from childhood excited attention through the strange stories and odd dreams he was in the habit of relating. He called himself “psychically hypersensitive”, but the staid folk of the ancient commercial city dismissed him as merely “queer”. Never mingling much with his kind, he had dropped gradually from social visibility, and was now known only to a small group of aesthetes from other towns. Even the Providence Art Club, anxious to preserve its conservatism, had found him quite hopeless. 23 | On the occasion of the visit, ran the professor’s manuscript, the sculptor abruptly asked for the benefit of his host’s archaeological knowledge in identifying the hieroglyphics on the bas-relief. He spoke in a dreamy, stilted manner which suggested pose and alienated sympathy; and my uncle shewed some sharpness in replying, for the conspicuous freshness of the tablet implied kinship with anything but archaeology. Young Wilcox’s rejoinder, which impressed my uncle enough to make him recall and record it verbatim, was of a fantastically poetic cast which must have typified his whole conversation, and which I have since found highly characteristic of him. He said, “It is new, indeed, for I made it last night in a dream of strange cities; and dreams are older than brooding Tyre, or the contemplative Sphinx, or garden-girdled Babylon.” 24 | It was then that he began that rambling tale which suddenly played upon a sleeping memory and won the fevered interest of my uncle. There had been a slight earthquake tremor the night before, the most considerable felt in New England for some years; and Wilcox’s imagination had been keenly affected. Upon retiring, he had had an unprecedented dream of great Cyclopean cities of titan blocks and sky-flung monoliths, all dripping with green ooze and sinister with latent horror. Hieroglyphics had covered the walls and pillars, and from some undetermined point below had come a voice that was not a voice; a chaotic sensation which only fancy could transmute into sound, but which he attempted to render by the almost unpronounceable jumble of letters, “Cthulhu fhtagn”. 25 | This verbal jumble was the key to the recollection which excited and disturbed Professor Angell. He questioned the sculptor with scientific minuteness; and studied with almost frantic intensity the bas-relief on which the youth had found himself working, chilled and clad only in his night-clothes, when waking had stolen bewilderingly over him. My uncle blamed his old age, Wilcox afterward said, for his slowness in recognising both hieroglyphics and pictorial design. Many of his questions seemed highly out-of-place to his visitor, especially those which tried to connect the latter with strange cults or societies; and Wilcox could not understand the repeated promises of silence which he was offered in exchange for an admission of membership in some widespread mystical or paganly religious body. When Professor Angell became convinced that the sculptor was indeed ignorant of any cult or system of cryptic lore, he besieged his visitor with demands for future reports of dreams. This bore regular fruit, for after the first interview the manuscript records daily calls of the young man, during which he related startling fragments of nocturnal imagery whose burden was always some terrible Cyclopean vista of dark and dripping stone, with a subterrene voice or intelligence shouting monotonously in enigmatical sense-impacts uninscribable save as gibberish. The two sounds most frequently repeated are those rendered by the letters “Cthulhu” and “R’lyeh”. 26 | On March 23d, the manuscript continued, Wilcox failed to appear; and inquiries at his quarters revealed that he had been stricken with an obscure sort of fever and taken to the home of his family in Waterman Street. He had cried out in the night, arousing several other artists in the building, and had manifested since then only alternations of unconsciousness and delirium. My uncle at once telephoned the family, and from that time forward kept close watch of the case; calling often at the Thayer Street office of Dr. Tobey, whom he learned to be in charge. The youth’s febrile mind, apparently, was dwelling on strange things; and the doctor shuddered now and then as he spoke of them. They included not only a repetition of what he had formerly dreamed, but touched wildly on a gigantic thing “miles high” which walked or lumbered about. He at no time fully described this object, but occasional frantic words, as repeated by Dr. Tobey, convinced the professor that it must be identical with the nameless monstrosity he had sought to depict in his dream-sculpture. Reference to this object, the doctor added, was invariably a prelude to the young man’s subsidence into lethargy. His temperature, oddly enough, was not greatly above normal; but his whole condition was otherwise such as to suggest true fever rather than mental disorder. 27 | On April 2nd at about 3 p.m. every trace of Wilcox’s malady suddenly ceased. He sat upright in bed, astonished to find himself at home and completely ignorant of what had happened in dream or reality since the night of March 22nd. Pronounced well by his physician, he returned to his quarters in three days; but to Professor Angell he was of no further assistance. All traces of strange dreaming had vanished with his recovery, and my uncle kept no record of his night-thoughts after a week of pointless and irrelevant accounts of thoroughly usual visions. 28 | Here the first part of the manuscript ended, but references to certain of the scattered notes gave me much material for thought—so much, in fact, that only the ingrained scepticism then forming my philosophy can account for my continued distrust of the artist. The notes in question were those descriptive of the dreams of various persons covering the same period as that in which young Wilcox had had his strange visitations. My uncle, it seems, had quickly instituted a prodigiously far-flung body of inquiries amongst nearly all the friends whom he could question without impertinence, asking for nightly reports of their dreams, and the dates of any notable visions for some time past. The reception of his request seems to have been varied; but he must, at the very least, have received more responses than any ordinary man could have handled without a secretary. This original correspondence was not preserved, but his notes formed a thorough and really significant digest. Average people in society and business—New England’s traditional “salt of the earth”—gave an almost completely negative result, though scattered cases of uneasy but formless nocturnal impressions appear here and there, always between March 23d and April 2nd—the period of young Wilcox’s delirium. Scientific men were little more affected, though four cases of vague description suggest fugitive glimpses of strange landscapes, and in one case there is mentioned a dread of something abnormal. 29 | It was from the artists and poets that the pertinent answers came, and I know that panic would have broken loose had they been able to compare notes. As it was, lacking their original letters, I half suspected the compiler of having asked leading questions, or of having edited the correspondence in corroboration of what he had latently resolved to see. That is why I continued to feel that Wilcox, somehow cognisant of the old data which my uncle had possessed, had been imposing on the veteran scientist. These responses from aesthetes told a disturbing tale. From February 28th to April 2nd a large proportion of them had dreamed very bizarre things, the intensity of the dreams being immeasurably the stronger during the period of the sculptor’s delirium. Over a fourth of those who reported anything, reported scenes and half-sounds not unlike those which Wilcox had described; and some of the dreamers confessed acute fear of the gigantic nameless thing visible toward the last. One case, which the note describes with emphasis, was very sad. The subject, a widely known architect with leanings toward theosophy and occultism, went violently insane on the date of young Wilcox’s seizure, and expired several months later after incessant screamings to be saved from some escaped denizen of hell. Had my uncle referred to these cases by name instead of merely by number, I should have attempted some corroboration and personal investigation; but as it was, I succeeded in tracing down only a few. All of these, however, bore out the notes in full. I have often wondered if all the objects of the professor’s questioning felt as puzzled as did this fraction. It is well that no explanation shall ever reach them. 30 | The press cuttings, as I have intimated, touched on cases of panic, mania, and eccentricity during the given period. Professor Angell must have employed a cutting bureau, for the number of extracts was tremendous and the sources scattered throughout the globe. Here was a nocturnal suicide in London, where a lone sleeper had leaped from a window after a shocking cry. Here likewise a rambling letter to the editor of a paper in South America, where a fanatic deduces a dire future from visions he has seen. A despatch from California describes a theosophist colony as donning white robes en masse for some “glorious fulfilment” which never arrives, whilst items from India speak guardedly of serious native unrest toward the end of March. Voodoo orgies multiply in Hayti, and African outposts report ominous mutterings. American officers in the Philippines find certain tribes bothersome about this time, and New York policemen are mobbed by hysterical Levantines on the night of March 22–23. The west of Ireland, too, is full of wild rumour and legendry, and a fantastic painter named Ardois-Bonnot hangs a blasphemous “Dream Landscape” in the Paris spring salon of 1926. And so numerous are the recorded troubles in insane asylums, that only a miracle can have stopped the medical fraternity from noting strange parallelisms and drawing mystified conclusions. A weird bunch of cuttings, all told; and I can at this date scarcely envisage the callous rationalism with which I set them aside. But I was then convinced that young Wilcox had known of the older matters mentioned by the professor. 31 | 32 | II. 33 | The Tale of Inspector Legrasse. 34 | 35 | The older matters which had made the sculptor’s dream and bas-relief so significant to my uncle formed the subject of the second half of his long manuscript. Once before, it appears, Professor Angell had seen the hellish outlines of the nameless monstrosity, puzzled over the unknown hieroglyphics, and heard the ominous syllables which can be rendered only as “Cthulhu”; and all this in so stirring and horrible a connexion that it is small wonder he pursued young Wilcox with queries and demands for data. 36 | The earlier experience had come in 1908, seventeen years before, when the American Archaeological Society held its annual meeting in St. Louis. Professor Angell, as befitted one of his authority and attainments, had had a prominent part in all the deliberations; and was one of the first to be approached by the several outsiders who took advantage of the convocation to offer questions for correct answering and problems for expert solution. 37 | The chief of these outsiders, and in a short time the focus of interest for the entire meeting, was a commonplace-looking middle-aged man who had travelled all the way from New Orleans for certain special information unobtainable from any local source. His name was John Raymond Legrasse, and he was by profession an Inspector of Police. With him he bore the subject of his visit, a grotesque, repulsive, and apparently very ancient stone statuette whose origin he was at a loss to determine. It must not be fancied that Inspector Legrasse had the least interest in archaeology. On the contrary, his wish for enlightenment was prompted by purely professional considerations. The statuette, idol, fetish, or whatever it was, had been captured some months before in the wooded swamps south of New Orleans during a raid on a supposed voodoo meeting; and so singular and hideous were the rites connected with it, that the police could not but realise that they had stumbled on a dark cult totally unknown to them, and infinitely more diabolic than even the blackest of the African voodoo circles. Of its origin, apart from the erratic and unbelievable tales extorted from the captured members, absolutely nothing was to be discovered; hence the anxiety of the police for any antiquarian lore which might help them to place the frightful symbol, and through it track down the cult to its fountain-head. 38 | Inspector Legrasse was scarcely prepared for the sensation which his offering created. One sight of the thing had been enough to throw the assembled men of science into a state of tense excitement, and they lost no time in crowding around him to gaze at the diminutive figure whose utter strangeness and air of genuinely abysmal antiquity hinted so potently at unopened and archaic vistas. No recognised school of sculpture had animated this terrible object, yet centuries and even thousands of years seemed recorded in its dim and greenish surface of unplaceable stone. 39 | The figure, which was finally passed slowly from man to man for close and careful study, was between seven and eight inches in height, and of exquisitely artistic workmanship. It represented a monster of vaguely anthropoid outline, but with an octopus-like head whose face was a mass of feelers, a scaly, rubbery-looking body, prodigious claws on hind and fore feet, and long, narrow wings behind. This thing, which seemed instinct with a fearsome and unnatural malignancy, was of a somewhat bloated corpulence, and squatted evilly on a rectangular block or pedestal covered with undecipherable characters. The tips of the wings touched the back edge of the block, the seat occupied the centre, whilst the long, curved claws of the doubled-up, crouching hind legs gripped the front edge and extended a quarter of the way down toward the bottom of the pedestal. The cephalopod head was bent forward, so that the ends of the facial feelers brushed the backs of huge fore paws which clasped the croucher’s elevated knees. The aspect of the whole was abnormally life-like, and the more subtly fearful because its source was so totally unknown. Its vast, awesome, and incalculable age was unmistakable; yet not one link did it shew with any known type of art belonging to civilisation’s youth—or indeed to any other time. Totally separate and apart, its very material was a mystery; for the soapy, greenish-black stone with its golden or iridescent flecks and striations resembled nothing familiar to geology or mineralogy. The characters along the base were equally baffling; and no member present, despite a representation of half the world’s expert learning in this field, could form the least notion of even their remotest linguistic kinship. They, like the subject and material, belonged to something horribly remote and distinct from mankind as we know it; something frightfully suggestive of old and unhallowed cycles of life in which our world and our conceptions have no part. 40 | And yet, as the members severally shook their heads and confessed defeat at the Inspector’s problem, there was one man in that gathering who suspected a touch of bizarre familiarity in the monstrous shape and writing, and who presently told with some diffidence of the odd trifle he knew. This person was the late William Channing Webb, Professor of Anthropology in Princeton University, and an explorer of no slight note. Professor Webb had been engaged, forty-eight years before, in a tour of Greenland and Iceland in search of some Runic inscriptions which he failed to unearth; and whilst high up on the West Greenland coast had encountered a singular tribe or cult of degenerate Esquimaux whose religion, a curious form of devil-worship, chilled him with its deliberate bloodthirstiness and repulsiveness. It was a faith of which other Esquimaux knew little, and which they mentioned only with shudders, saying that it had come down from horribly ancient aeons before ever the world was made. Besides nameless rites and human sacrifices there were certain queer hereditary rituals addressed to a supreme elder devil or tornasuk; and of this Professor Webb had taken a careful phonetic copy from an aged angekok or wizard-priest, expressing the sounds in Roman letters as best he knew how. But just now of prime significance was the fetish which this cult had cherished, and around which they danced when the aurora leaped high over the ice cliffs. It was, the professor stated, a very crude bas-relief of stone, comprising a hideous picture and some cryptic writing. And so far as he could tell, it was a rough parallel in all essential features of the bestial thing now lying before the meeting. 41 | This data, received with suspense and astonishment by the assembled members, proved doubly exciting to Inspector Legrasse; and he began at once to ply his informant with questions. Having noted and copied an oral ritual among the swamp cult-worshippers his men had arrested, he besought the professor to remember as best he might the syllables taken down amongst the diabolist Esquimaux. There then followed an exhaustive comparison of details, and a moment of really awed silence when both detective and scientist agreed on the virtual identity of the phrase common to two hellish rituals so many worlds of distance apart. What, in substance, both the Esquimau wizards and the Louisiana swamp-priests had chanted to their kindred idols was something very like this—the word-divisions being guessed at from traditional breaks in the phrase as chanted aloud: 42 | “Ph’nglui mglw’nafh Cthulhu R’lyeh wgah’nagl fhtagn.” 43 | Legrasse had one point in advance of Professor Webb, for several among his mongrel prisoners had repeated to him what older celebrants had told them the words meant. This text, as given, ran something like this: 44 | “In his house at R’lyeh dead Cthulhu waits dreaming.” 45 | And now, in response to a general and urgent demand, Inspector Legrasse related as fully as possible his experience with the swamp worshippers; telling a story to which I could see my uncle attached profound significance. It savoured of the wildest dreams of myth-maker and theosophist, and disclosed an astonishing degree of cosmic imagination among such half-castes and pariahs as might be least expected to possess it. 46 | On November 1st, 1907, there had come to the New Orleans police a frantic summons from the swamp and lagoon country to the south. The squatters there, mostly primitive but good-natured descendants of Lafitte’s men, were in the grip of stark terror from an unknown thing which had stolen upon them in the night. It was voodoo, apparently, but voodoo of a more terrible sort than they had ever known; and some of their women and children had disappeared since the malevolent tom-tom had begun its incessant beating far within the black haunted woods where no dweller ventured. There were insane shouts and harrowing screams, soul-chilling chants and dancing devil-flames; and, the frightened messenger added, the people could stand it no more. 47 | So a body of twenty police, filling two carriages and an automobile, had set out in the late afternoon with the shivering squatter as a guide. At the end of the passable road they alighted, and for miles splashed on in silence through the terrible cypress woods where day never came. Ugly roots and malignant hanging nooses of Spanish moss beset them, and now and then a pile of dank stones or fragment of a rotting wall intensified by its hint of morbid habitation a depression which every malformed tree and every fungous islet combined to create. At length the squatter settlement, a miserable huddle of huts, hove in sight; and hysterical dwellers ran out to cluster around the group of bobbing lanterns. The muffled beat of tom-toms was now faintly audible far, far ahead; and a curdling shriek came at infrequent intervals when the wind shifted. A reddish glare, too, seemed to filter through the pale undergrowth beyond endless avenues of forest night. Reluctant even to be left alone again, each one of the cowed squatters refused point-blank to advance another inch toward the scene of unholy worship, so Inspector Legrasse and his nineteen colleagues plunged on unguided into black arcades of horror that none of them had ever trod before. 48 | The region now entered by the police was one of traditionally evil repute, substantially unknown and untraversed by white men. There were legends of a hidden lake unglimpsed by mortal sight, in which dwelt a huge, formless white polypous thing with luminous eyes; and squatters whispered that bat-winged devils flew up out of caverns in inner earth to worship it at midnight. They said it had been there before D’Iberville, before La Salle, before the Indians, and before even the wholesome beasts and birds of the woods. It was nightmare itself, and to see it was to die. But it made men dream, and so they knew enough to keep away. The present voodoo orgy was, indeed, on the merest fringe of this abhorred area, but that location was bad enough; hence perhaps the very place of the worship had terrified the squatters more than the shocking sounds and incidents. 49 | Only poetry or madness could do justice to the noises heard by Legrasse’s men as they ploughed on through the black morass toward the red glare and the muffled tom-toms. There are vocal qualities peculiar to men, and vocal qualities peculiar to beasts; and it is terrible to hear the one when the source should yield the other. Animal fury and orgiastic licence here whipped themselves to daemoniac heights by howls and squawking ecstasies that tore and reverberated through those nighted woods like pestilential tempests from the gulfs of hell. Now and then the less organised ululation would cease, and from what seemed a well-drilled chorus of hoarse voices would rise in sing-song chant that hideous phrase or ritual: 50 | “Ph’nglui mglw’nafh Cthulhu R’lyeh wgah’nagl fhtagn.” 51 | Then the men, having reached a spot where the trees were thinner, came suddenly in sight of the spectacle itself. Four of them reeled, one fainted, and two were shaken into a frantic cry which the mad cacophony of the orgy fortunately deadened. Legrasse dashed swamp water on the face of the fainting man, and all stood trembling and nearly hypnotised with horror. 52 | In a natural glade of the swamp stood a grassy island of perhaps an acre’s extent, clear of trees and tolerably dry. On this now leaped and twisted a more indescribable horde of human abnormality than any but a Sime or an Angarola could paint. Void of clothing, this hybrid spawn were braying, bellowing, and writhing about a monstrous ring-shaped bonfire; in the centre of which, revealed by occasional rifts in the curtain of flame, stood a great granite monolith some eight feet in height; on top of which, incongruous with its diminutiveness, rested the noxious carven statuette. From a wide circle of ten scaffolds set up at regular intervals with the flame-girt monolith as a centre hung, head downward, the oddly marred bodies of the helpless squatters who had disappeared. It was inside this circle that the ring of worshippers jumped and roared, the general direction of the mass motion being from left to right in endless Bacchanal between the ring of bodies and the ring of fire. 53 | It may have been only imagination and it may have been only echoes which induced one of the men, an excitable Spaniard, to fancy he heard antiphonal responses to the ritual from some far and unillumined spot deeper within the wood of ancient legendry and horror. This man, Joseph D. Galvez, I later met and questioned; and he proved distractingly imaginative. He indeed went so far as to hint of the faint beating of great wings, and of a glimpse of shining eyes and a mountainous white bulk beyond the remotest trees—but I suppose he had been hearing too much native superstition. 54 | Actually, the horrified pause of the men was of comparatively brief duration. Duty came first; and although there must have been nearly a hundred mongrel celebrants in the throng, the police relied on their firearms and plunged determinedly into the nauseous rout. For five minutes the resultant din and chaos were beyond description. Wild blows were struck, shots were fired, and escapes were made; but in the end Legrasse was able to count some forty-seven sullen prisoners, whom he forced to dress in haste and fall into line between two rows of policemen. Five of the worshippers lay dead, and two severely wounded ones were carried away on improvised stretchers by their fellow-prisoners. The image on the monolith, of course, was carefully removed and carried back by Legrasse. 55 | Examined at headquarters after a trip of intense strain and weariness, the prisoners all proved to be men of a very low, mixed-blooded, and mentally aberrant type. Most were seamen, and a sprinkling of negroes and mulattoes, largely West Indians or Brava Portuguese from the Cape Verde Islands, gave a colouring of voodooism to the heterogeneous cult. But before many questions were asked, it became manifest that something far deeper and older than negro fetichism was involved. Degraded and ignorant as they were, the creatures held with surprising consistency to the central idea of their loathsome faith. 56 | They worshipped, so they said, the Great Old Ones who lived ages before there were any men, and who came to the young world out of the sky. Those Old Ones were gone now, inside the earth and under the sea; but their dead bodies had told their secrets in dreams to the first men, who formed a cult which had never died. This was that cult, and the prisoners said it had always existed and always would exist, hidden in distant wastes and dark places all over the world until the time when the great priest Cthulhu, from his dark house in the mighty city of R’lyeh under the waters, should rise and bring the earth again beneath his sway. Some day he would call, when the stars were ready, and the secret cult would always be waiting to liberate him. 57 | Meanwhile no more must be told. There was a secret which even torture could not extract. Mankind was not absolutely alone among the conscious things of earth, for shapes came out of the dark to visit the faithful few. But these were not the Great Old Ones. No man had ever seen the Old Ones. The carven idol was great Cthulhu, but none might say whether or not the others were precisely like him. No one could read the old writing now, but things were told by word of mouth. The chanted ritual was not the secret—that was never spoken aloud, only whispered. The chant meant only this: “In his house at R’lyeh dead Cthulhu waits dreaming.” 58 | Only two of the prisoners were found sane enough to be hanged, and the rest were committed to various institutions. All denied a part in the ritual murders, and averred that the killing had been done by Black Winged Ones which had come to them from their immemorial meeting-place in the haunted wood. But of those mysterious allies no coherent account could ever be gained. What the police did extract, came mainly from an immensely aged mestizo named Castro, who claimed to have sailed to strange ports and talked with undying leaders of the cult in the mountains of China. 59 | Old Castro remembered bits of hideous legend that paled the speculations of theosophists and made man and the world seem recent and transient indeed. There had been aeons when other Things ruled on the earth, and They had had great cities. Remains of Them, he said the deathless Chinamen had told him, were still to be found as Cyclopean stones on islands in the Pacific. They all died vast epochs of time before men came, but there were arts which could revive Them when the stars had come round again to the right positions in the cycle of eternity. They had, indeed, come themselves from the stars, and brought Their images with Them. 60 | These Great Old Ones, Castro continued, were not composed altogether of flesh and blood. They had shape—for did not this star-fashioned image prove it?—but that shape was not made of matter. When the stars were right, They could plunge from world to world through the sky; but when the stars were wrong, They could not live. But although They no longer lived, They would never really die. They all lay in stone houses in Their great city of R’lyeh, preserved by the spells of mighty Cthulhu for a glorious resurrection when the stars and the earth might once more be ready for Them. But at that time some force from outside must serve to liberate Their bodies. The spells that preserved Them intact likewise prevented Them from making an initial move, and They could only lie awake in the dark and think whilst uncounted millions of years rolled by. They knew all that was occurring in the universe, but Their mode of speech was transmitted thought. Even now They talked in Their tombs. When, after infinities of chaos, the first men came, the Great Old Ones spoke to the sensitive among them by moulding their dreams; for only thus could Their language reach the fleshly minds of mammals. 61 | Then, whispered Castro, those first men formed the cult around small idols which the Great Ones shewed them; idols brought in dim aeras from dark stars. That cult would never die till the stars came right again, and the secret priests would take great Cthulhu from His tomb to revive His subjects and resume His rule of earth. The time would be easy to know, for then mankind would have become as the Great Old Ones; free and wild and beyond good and evil, with laws and morals thrown aside and all men shouting and killing and revelling in joy. Then the liberated Old Ones would teach them new ways to shout and kill and revel and enjoy themselves, and all the earth would flame with a holocaust of ecstasy and freedom. Meanwhile the cult, by appropriate rites, must keep alive the memory of those ancient ways and shadow forth the prophecy of their return. 62 | In the elder time chosen men had talked with the entombed Old Ones in dreams, but then something had happened. The great stone city R’lyeh, with its monoliths and sepulchres, had sunk beneath the waves; and the deep waters, full of the one primal mystery through which not even thought can pass, had cut off the spectral intercourse. But memory never died, and high-priests said that the city would rise again when the stars were right. Then came out of the earth the black spirits of earth, mouldy and shadowy, and full of dim rumours picked up in caverns beneath forgotten sea-bottoms. But of them old Castro dared not speak much. He cut himself off hurriedly, and no amount of persuasion or subtlety could elicit more in this direction. The size of the Old Ones, too, he curiously declined to mention. Of the cult, he said that he thought the centre lay amid the pathless deserts of Arabia, where Irem, the City of Pillars, dreams hidden and untouched. It was not allied to the European witch-cult, and was virtually unknown beyond its members. No book had ever really hinted of it, though the deathless Chinamen said that there were double meanings in the Necronomicon of the mad Arab Abdul Alhazred which the initiated might read as they chose, especially the much-discussed couplet: 63 | 64 | “That is not dead which can eternal lie, 65 | And with strange aeons even death may die.” 66 | 67 | Legrasse, deeply impressed and not a little bewildered, had inquired in vain concerning the historic affiliations of the cult. Castro, apparently, had told the truth when he said that it was wholly secret. The authorities at Tulane University could shed no light upon either cult or image, and now the detective had come to the highest authorities in the country and met with no more than the Greenland tale of Professor Webb. 68 | The feverish interest aroused at the meeting by Legrasse’s tale, corroborated as it was by the statuette, is echoed in the subsequent correspondence of those who attended; although scant mention occurs in the formal publications of the society. Caution is the first care of those accustomed to face occasional charlatanry and imposture. Legrasse for some time lent the image to Professor Webb, but at the latter’s death it was returned to him and remains in his possession, where I viewed it not long ago. It is truly a terrible thing, and unmistakably akin to the dream-sculpture of young Wilcox. 69 | That my uncle was excited by the tale of the sculptor I did not wonder, for what thoughts must arise upon hearing, after a knowledge of what Legrasse had learned of the cult, of a sensitive young man who had dreamed not only the figure and exact hieroglyphics of the swamp-found image and the Greenland devil tablet, but had come in his dreams upon at least three of the precise words of the formula uttered alike by Esquimau diabolists and mongrel Louisianans? Professor Angell’s instant start on an investigation of the utmost thoroughness was eminently natural; though privately I suspected young Wilcox of having heard of the cult in some indirect way, and of having invented a series of dreams to heighten and continue the mystery at my uncle’s expense. The dream-narratives and cuttings collected by the professor were, of course, strong corroboration; but the rationalism of my mind and the extravagance of the whole subject led me to adopt what I thought the most sensible conclusions. So, after thoroughly studying the manuscript again and correlating the theosophical and anthropological notes with the cult narrative of Legrasse, I made a trip to Providence to see the sculptor and give him the rebuke I thought proper for so boldly imposing upon a learned and aged man. 70 | Wilcox still lived alone in the Fleur-de-Lys Building in Thomas Street, a hideous Victorian imitation of seventeenth-century Breton architecture which flaunts its stuccoed front amidst the lovely colonial houses on the ancient hill, and under the very shadow of the finest Georgian steeple in America. I found him at work in his rooms, and at once conceded from the specimens scattered about that his genius is indeed profound and authentic. He will, I believe, some time be heard from as one of the great decadents; for he has crystallised in clay and will one day mirror in marble those nightmares and phantasies which Arthur Machen evokes in prose, and Clark Ashton Smith makes visible in verse and in painting. 71 | Dark, frail, and somewhat unkempt in aspect, he turned languidly at my knock and asked me my business without rising. When I told him who I was, he displayed some interest; for my uncle had excited his curiosity in probing his strange dreams, yet had never explained the reason for the study. I did not enlarge his knowledge in this regard, but sought with some subtlety to draw him out. In a short time I became convinced of his absolute sincerity, for he spoke of the dreams in a manner none could mistake. They and their subconscious residuum had influenced his art profoundly, and he shewed me a morbid statue whose contours almost made me shake with the potency of its black suggestion. He could not recall having seen the original of this thing except in his own dream bas-relief, but the outlines had formed themselves insensibly under his hands. It was, no doubt, the giant shape he had raved of in delirium. That he really knew nothing of the hidden cult, save from what my uncle’s relentless catechism had let fall, he soon made clear; and again I strove to think of some way in which he could possibly have received the weird impressions. 72 | He talked of his dreams in a strangely poetic fashion; making me see with terrible vividness the damp Cyclopean city of slimy green stone—whose geometry, he oddly said, was all wrong—and hear with frightened expectancy the ceaseless, half-mental calling from underground: “Cthulhu fhtagn”, “Cthulhu fhtagn”. These words had formed part of that dread ritual which told of dead Cthulhu’s dream-vigil in his stone vault at R’lyeh, and I felt deeply moved despite my rational beliefs. Wilcox, I was sure, had heard of the cult in some casual way, and had soon forgotten it amidst the mass of his equally weird reading and imagining. Later, by virtue of its sheer impressiveness, it had found subconscious expression in dreams, in the bas-relief, and in the terrible statue I now beheld; so that his imposture upon my uncle had been a very innocent one. The youth was of a type, at once slightly affected and slightly ill-mannered, which I could never like; but I was willing enough now to admit both his genius and his honesty. I took leave of him amicably, and wish him all the success his talent promises. 73 | The matter of the cult still remained to fascinate me, and at times I had visions of personal fame from researches into its origin and connexions. I visited New Orleans, talked with Legrasse and others of that old-time raiding-party, saw the frightful image, and even questioned such of the mongrel prisoners as still survived. Old Castro, unfortunately, had been dead for some years. What I now heard so graphically at first-hand, though it was really no more than a detailed confirmation of what my uncle had written, excited me afresh; for I felt sure that I was on the track of a very real, very secret, and very ancient religion whose discovery would make me an anthropologist of note. My attitude was still one of absolute materialism, as I wish it still were, and I discounted with almost inexplicable perversity the coincidence of the dream notes and odd cuttings collected by Professor Angell. 74 | One thing I began to suspect, and which I now fear I know, is that my uncle’s death was far from natural. He fell on a narrow hill street leading up from an ancient waterfront swarming with foreign mongrels, after a careless push from a negro sailor. I did not forget the mixed blood and marine pursuits of the cult-members in Louisiana, and would not be surprised to learn of secret methods and poison needles as ruthless and as anciently known as the cryptic rites and beliefs. Legrasse and his men, it is true, have been let alone; but in Norway a certain seaman who saw things is dead. Might not the deeper inquiries of my uncle after encountering the sculptor’s data have come to sinister ears? I think Professor Angell died because he knew too much, or because he was likely to learn too much. Whether I shall go as he did remains to be seen, for I have learned much now. 75 | 76 | III. 77 | The Madness from the Sea. 78 | 79 | If heaven ever wishes to grant me a boon, it will be a total effacing of the results of a mere chance which fixed my eye on a certain stray piece of shelf-paper. It was nothing on which I would naturally have stumbled in the course of my daily round, for it was an old number of an Australian journal, the Sydney Bulletin for April 18, 1925. It had escaped even the cutting bureau which had at the time of its issuance been avidly collecting material for my uncle’s research. 80 | I had largely given over my inquiries into what Professor Angell called the “Cthulhu Cult”, and was visiting a learned friend in Paterson, New Jersey; the curator of a local museum and a mineralogist of note. Examining one day the reserve specimens roughly set on the storage shelves in a rear room of the museum, my eye was caught by an odd picture in one of the old papers spread beneath the stones. It was the Sydney Bulletin I have mentioned, for my friend has wide affiliations in all conceivable foreign parts; and the picture was a half-tone cut of a hideous stone image almost identical with that which Legrasse had found in the swamp. 81 | Eagerly clearing the sheet of its precious contents, I scanned the item in detail; and was disappointed to find it of only moderate length. What it suggested, however, was of portentous significance to my flagging quest; and I carefully tore it out for immediate action. It read as follows: 82 | 83 | MYSTERY DERELICT FOUND AT SEA 84 | Vigilant Arrives With Helpless Armed New Zealand Yacht in Tow. 85 | One Survivor and Dead Man Found Aboard. Tale of 86 | Desperate Battle and Deaths at Sea. 87 | Rescued Seaman Refuses 88 | Particulars of Strange Experience. 89 | Odd Idol Found in His Possession. Inquiry 90 | to Follow. 91 | 92 | The Morrison Co.’s freighter Vigilant, bound from Valparaiso, arrived this morning at its wharf in Darling Harbour, having in tow the battled and disabled but heavily armed steam yacht Alert of Dunedin, N. Z., which was sighted April 12th in S. Latitude 34° 21′, W. Longitude 152° 17′ with one living and one dead man aboard. 93 | The Vigilant left Valparaiso March 25th, and on April 2nd was driven considerably south of her course by exceptionally heavy storms and monster waves. On April 12th the derelict was sighted; and though apparently deserted, was found upon boarding to contain one survivor in a half-delirious condition and one man who had evidently been dead for more than a week. The living man was clutching a horrible stone idol of unknown origin, about a foot in height, regarding whose nature authorities at Sydney University, the Royal Society, and the Museum in College Street all profess complete bafflement, and which the survivor says he found in the cabin of the yacht, in a small carved shrine of common pattern. 94 | This man, after recovering his senses, told an exceedingly strange story of piracy and slaughter. He is Gustaf Johansen, a Norwegian of some intelligence, and had been second mate of the two-masted schooner Emma of Auckland, which sailed for Callao February 20th with a complement of eleven men. The Emma, he says, was delayed and thrown widely south of her course by the great storm of March 1st, and on March 22nd, in S. Latitude 49° 51′, W. Longitude 128° 34′, encountered the Alert, manned by a queer and evil-looking crew of Kanakas and half-castes. Being ordered peremptorily to turn back, Capt. Collins refused; whereupon the strange crew began to fire savagely and without warning upon the schooner with a peculiarly heavy battery of brass cannon forming part of the yacht’s equipment. The Emma’s men shewed fight, says the survivor, and though the schooner began to sink from shots beneath the waterline they managed to heave alongside their enemy and board her, grappling with the savage crew on the yacht’s deck, and being forced to kill them all, the number being slightly superior, because of their particularly abhorrent and desperate though rather clumsy mode of fighting. 95 | Three of the Emma’s men, including Capt. Collins and First Mate Green, were killed; and the remaining eight under Second Mate Johansen proceeded to navigate the captured yacht, going ahead in their original direction to see if any reason for their ordering back had existed. The next day, it appears, they raised and landed on a small island, although none is known to exist in that part of the ocean; and six of the men somehow died ashore, though Johansen is queerly reticent about this part of his story, and speaks only of their falling into a rock chasm. Later, it seems, he and one companion boarded the yacht and tried to manage her, but were beaten about by the storm of April 2nd. From that time till his rescue on the 12th the man remembers little, and he does not even recall when William Briden, his companion, died. Briden’s death reveals no apparent cause, and was probably due to excitement or exposure. Cable advices from Dunedin report that the Alert was well known there as an island trader, and bore an evil reputation along the waterfront. It was owned by a curious group of half-castes whose frequent meetings and night trips to the woods attracted no little curiosity; and it had set sail in great haste just after the storm and earth tremors of March 1st. Our Auckland correspondent gives the Emma and her crew an excellent reputation, and Johansen is described as a sober and worthy man. The admiralty will institute an inquiry on the whole matter beginning tomorrow, at which every effort will be made to induce Johansen to speak more freely than he has done hitherto. 96 | 97 | This was all, together with the picture of the hellish image; but what a train of ideas it started in my mind! Here were new treasuries of data on the Cthulhu Cult, and evidence that it had strange interests at sea as well as on land. What motive prompted the hybrid crew to order back the Emma as they sailed about with their hideous idol? What was the unknown island on which six of the Emma’s crew had died, and about which the mate Johansen was so secretive? What had the vice-admiralty’s investigation brought out, and what was known of the noxious cult in Dunedin? And most marvellous of all, what deep and more than natural linkage of dates was this which gave a malign and now undeniable significance to the various turns of events so carefully noted by my uncle? 98 | March 1st—our February 28th according to the International Date Line—the earthquake and storm had come. From Dunedin the Alert and her noisome crew had darted eagerly forth as if imperiously summoned, and on the other side of the earth poets and artists had begun to dream of a strange, dank Cyclopean city whilst a young sculptor had moulded in his sleep the form of the dreaded Cthulhu. March 23d the crew of the Emma landed on an unknown island and left six men dead; and on that date the dreams of sensitive men assumed a heightened vividness and darkened with dread of a giant monster’s malign pursuit, whilst an architect had gone mad and a sculptor had lapsed suddenly into delirium! And what of this storm of April 2nd—the date on which all dreams of the dank city ceased, and Wilcox emerged unharmed from the bondage of strange fever? What of all this—and of those hints of old Castro about the sunken, star-born Old Ones and their coming reign; their faithful cult and their mastery of dreams? Was I tottering on the brink of cosmic horrors beyond man’s power to bear? If so, they must be horrors of the mind alone, for in some way the second of April had put a stop to whatever monstrous menace had begun its siege of mankind’s soul. 99 | That evening, after a day of hurried cabling and arranging, I bade my host adieu and took a train for San Francisco. In less than a month I was in Dunedin; where, however, I found that little was known of the strange cult-members who had lingered in the old sea-taverns. Waterfront scum was far too common for special mention; though there was vague talk about one inland trip these mongrels had made, during which faint drumming and red flame were noted on the distant hills. In Auckland I learned that Johansen had returned with yellow hair turned white after a perfunctory and inconclusive questioning at Sydney, and had thereafter sold his cottage in West Street and sailed with his wife to his old home in Oslo. Of his stirring experience he would tell his friends no more than he had told the admiralty officials, and all they could do was to give me his Oslo address. 100 | After that I went to Sydney and talked profitlessly with seamen and members of the vice-admiralty court. I saw the Alert, now sold and in commercial use, at Circular Quay in Sydney Cove, but gained nothing from its non-committal bulk. The crouching image with its cuttlefish head, dragon body, scaly wings, and hieroglyphed pedestal, was preserved in the Museum at Hyde Park; and I studied it long and well, finding it a thing of balefully exquisite workmanship, and with the same utter mystery, terrible antiquity, and unearthly strangeness of material which I had noted in Legrasse’s smaller specimen. Geologists, the curator told me, had found it a monstrous puzzle; for they vowed that the world held no rock like it. Then I thought with a shudder of what old Castro had told Legrasse about the primal Great Ones: “They had come from the stars, and had brought Their images with Them.” 101 | Shaken with such a mental revolution as I had never before known, I now resolved to visit Mate Johansen in Oslo. Sailing for London, I reëmbarked at once for the Norwegian capital; and one autumn day landed at the trim wharves in the shadow of the Egeberg. Johansen’s address, I discovered, lay in the Old Town of King Harold Haardrada, which kept alive the name of Oslo during all the centuries that the greater city masqueraded as “Christiana”. I made the brief trip by taxicab, and knocked with palpitant heart at the door of a neat and ancient building with plastered front. A sad-faced woman in black answered my summons, and I was stung with disappointment when she told me in halting English that Gustaf Johansen was no more. 102 | He had not survived his return, said his wife, for the doings at sea in 1925 had broken him. He had told her no more than he had told the public, but had left a long manuscript—of “technical matters” as he said—written in English, evidently in order to safeguard her from the peril of casual perusal. During a walk through a narrow lane near the Gothenburg dock, a bundle of papers falling from an attic window had knocked him down. Two Lascar sailors at once helped him to his feet, but before the ambulance could reach him he was dead. Physicians found no adequate cause for the end, and laid it to heart trouble and a weakened constitution. 103 | I now felt gnawing at my vitals that dark terror which will never leave me till I, too, am at rest; “accidentally” or otherwise. Persuading the widow that my connexion with her husband’s “technical matters” was sufficient to entitle me to his manuscript, I bore the document away and began to read it on the London boat. It was a simple, rambling thing—a naive sailor’s effort at a post-facto diary—and strove to recall day by day that last awful voyage. I cannot attempt to transcribe it verbatim in all its cloudiness and redundance, but I will tell its gist enough to shew why the sound of the water against the vessel’s sides became so unendurable to me that I stopped my ears with cotton. 104 | Johansen, thank God, did not know quite all, even though he saw the city and the Thing, but I shall never sleep calmly again when I think of the horrors that lurk ceaselessly behind life in time and in space, and of those unhallowed blasphemies from elder stars which dream beneath the sea, known and favoured by a nightmare cult ready and eager to loose them on the world whenever another earthquake shall heave their monstrous stone city again to the sun and air. 105 | Johansen’s voyage had begun just as he told it to the vice-admiralty. The Emma, in ballast, had cleared Auckland on February 20th, and had felt the full force of that earthquake-born tempest which must have heaved up from the sea-bottom the horrors that filled men’s dreams. Once more under control, the ship was making good progress when held up by the Alert on March 22nd, and I could feel the mate’s regret as he wrote of her bombardment and sinking. Of the swarthy cult-fiends on the Alert he speaks with significant horror. There was some peculiarly abominable quality about them which made their destruction seem almost a duty, and Johansen shews ingenuous wonder at the charge of ruthlessness brought against his party during the proceedings of the court of inquiry. Then, driven ahead by curiosity in their captured yacht under Johansen’s command, the men sight a great stone pillar sticking out of the sea, and in S. Latitude 47° 9′, W. Longitude 126° 43′ come upon a coast-line of mingled mud, ooze, and weedy Cyclopean masonry which can be nothing less than the tangible substance of earth’s supreme terror—the nightmare corpse-city of R’lyeh, that was built in measureless aeons behind history by the vast, loathsome shapes that seeped down from the dark stars. There lay great Cthulhu and his hordes, hidden in green slimy vaults and sending out at last, after cycles incalculable, the thoughts that spread fear to the dreams of the sensitive and called imperiously to the faithful to come on a pilgrimage of liberation and restoration. All this Johansen did not suspect, but God knows he soon saw enough! 106 | I suppose that only a single mountain-top, the hideous monolith-crowned citadel whereon great Cthulhu was buried, actually emerged from the waters. When I think of the extent of all that may be brooding down there I almost wish to kill myself forthwith. Johansen and his men were awed by the cosmic majesty of this dripping Babylon of elder daemons, and must have guessed without guidance that it was nothing of this or of any sane planet. Awe at the unbelievable size of the greenish stone blocks, at the dizzying height of the great carven monolith, and at the stupefying identity of the colossal statues and bas-reliefs with the queer image found in the shrine on the Alert, is poignantly visible in every line of the mate’s frightened description. 107 | Without knowing what futurism is like, Johansen achieved something very close to it when he spoke of the city; for instead of describing any definite structure or building, he dwells only on broad impressions of vast angles and stone surfaces—surfaces too great to belong to any thing right or proper for this earth, and impious with horrible images and hieroglyphs. I mention his talk about angles because it suggests something Wilcox had told me of his awful dreams. He had said that the geometry of the dream-place he saw was abnormal, non-Euclidean, and loathsomely redolent of spheres and dimensions apart from ours. Now an unlettered seaman felt the same thing whilst gazing at the terrible reality. 108 | Johansen and his men landed at a sloping mud-bank on this monstrous Acropolis, and clambered slipperily up over titan oozy blocks which could have been no mortal staircase. The very sun of heaven seemed distorted when viewed through the polarising miasma welling out from this sea-soaked perversion, and twisted menace and suspense lurked leeringly in those crazily elusive angles of carven rock where a second glance shewed concavity after the first shewed convexity. 109 | Something very like fright had come over all the explorers before anything more definite than rock and ooze and weed was seen. Each would have fled had he not feared the scorn of the others, and it was only half-heartedly that they searched—vainly, as it proved—for some portable souvenir to bear away. 110 | It was Rodriguez the Portuguese who climbed up the foot of the monolith and shouted of what he had found. The rest followed him, and looked curiously at the immense carved door with the now familiar squid-dragon bas-relief. It was, Johansen said, like a great barn-door; and they all felt that it was a door because of the ornate lintel, threshold, and jambs around it, though they could not decide whether it lay flat like a trap-door or slantwise like an outside cellar-door. As Wilcox would have said, the geometry of the place was all wrong. One could not be sure that the sea and the ground were horizontal, hence the relative position of everything else seemed phantasmally variable. 111 | Briden pushed at the stone in several places without result. Then Donovan felt over it delicately around the edge, pressing each point separately as he went. He climbed interminably along the grotesque stone moulding—that is, one would call it climbing if the thing was not after all horizontal—and the men wondered how any door in the universe could be so vast. Then, very softly and slowly, the acre-great panel began to give inward at the top; and they saw that it was balanced. Donovan slid or somehow propelled himself down or along the jamb and rejoined his fellows, and everyone watched the queer recession of the monstrously carven portal. In this phantasy of prismatic distortion it moved anomalously in a diagonal way, so that all the rules of matter and perspective seemed upset. 112 | The aperture was black with a darkness almost material. That tenebrousness was indeed a positive quality; for it obscured such parts of the inner walls as ought to have been revealed, and actually burst forth like smoke from its aeon-long imprisonment, visibly darkening the sun as it slunk away into the shrunken and gibbous sky on flapping membraneous wings. The odour arising from the newly opened depths was intolerable, and at length the quick-eared Hawkins thought he heard a nasty, slopping sound down there. Everyone listened, and everyone was listening still when It lumbered slobberingly into sight and gropingly squeezed Its gelatinous green immensity through the black doorway into the tainted outside air of that poison city of madness. 113 | Poor Johansen’s handwriting almost gave out when he wrote of this. Of the six men who never reached the ship, he thinks two perished of pure fright in that accursed instant. The Thing cannot be described—there is no language for such abysms of shrieking and immemorial lunacy, such eldritch contradictions of all matter, force, and cosmic order. A mountain walked or stumbled. God! What wonder that across the earth a great architect went mad, and poor Wilcox raved with fever in that telepathic instant? The Thing of the idols, the green, sticky spawn of the stars, had awaked to claim his own. The stars were right again, and what an age-old cult had failed to do by design, a band of innocent sailors had done by accident. After vigintillions of years great Cthulhu was loose again, and ravening for delight. 114 | Three men were swept up by the flabby claws before anybody turned. God rest them, if there be any rest in the universe. They were Donovan, Guerrera, and Ångstrom. Parker slipped as the other three were plunging frenziedly over endless vistas of green-crusted rock to the boat, and Johansen swears he was swallowed up by an angle of masonry which shouldn’t have been there; an angle which was acute, but behaved as if it were obtuse. So only Briden and Johansen reached the boat, and pulled desperately for the Alert as the mountainous monstrosity flopped down the slimy stones and hesitated floundering at the edge of the water. 115 | Steam had not been suffered to go down entirely, despite the departure of all hands for the shore; and it was the work of only a few moments of feverish rushing up and down between wheel and engines to get the Alert under way. Slowly, amidst the distorted horrors of that indescribable scene, she began to churn the lethal waters; whilst on the masonry of that charnel shore that was not of earth the titan Thing from the stars slavered and gibbered like Polypheme cursing the fleeing ship of Odysseus. Then, bolder than the storied Cyclops, great Cthulhu slid greasily into the water and began to pursue with vast wave-raising strokes of cosmic potency. Briden looked back and went mad, laughing shrilly as he kept on laughing at intervals till death found him one night in the cabin whilst Johansen was wandering deliriously. 116 | But Johansen had not given out yet. Knowing that the Thing could surely overtake the Alert until steam was fully up, he resolved on a desperate chance; and, setting the engine for full speed, ran lightning-like on deck and reversed the wheel. There was a mighty eddying and foaming in the noisome brine, and as the steam mounted higher and higher the brave Norwegian drove his vessel head on against the pursuing jelly which rose above the unclean froth like the stern of a daemon galleon. The awful squid-head with writhing feelers came nearly up to the bowsprit of the sturdy yacht, but Johansen drove on relentlessly. There was a bursting as of an exploding bladder, a slushy nastiness as of a cloven sunfish, a stench as of a thousand opened graves, and a sound that the chronicler would not put on paper. For an instant the ship was befouled by an acrid and blinding green cloud, and then there was only a venomous seething astern; where—God in heaven!—the scattered plasticity of that nameless sky-spawn was nebulously recombining in its hateful original form, whilst its distance widened every second as the Alert gained impetus from its mounting steam. 117 | That was all. After that Johansen only brooded over the idol in the cabin and attended to a few matters of food for himself and the laughing maniac by his side. He did not try to navigate after the first bold flight, for the reaction had taken something out of his soul. Then came the storm of April 2nd, and a gathering of the clouds about his consciousness. There is a sense of spectral whirling through liquid gulfs of infinity, of dizzying rides through reeling universes on a comet’s tail, and of hysterical plunges from the pit to the moon and from the moon back again to the pit, all livened by a cachinnating chorus of the distorted, hilarious elder gods and the green, bat-winged mocking imps of Tartarus. 118 | Out of that dream came rescue—the Vigilant, the vice-admiralty court, the streets of Dunedin, and the long voyage back home to the old house by the Egeberg. He could not tell—they would think him mad. He would write of what he knew before death came, but his wife must not guess. Death would be a boon if only it could blot out the memories. 119 | That was the document I read, and now I have placed it in the tin box beside the bas-relief and the papers of Professor Angell. With it shall go this record of mine—this test of my own sanity, wherein is pieced together that which I hope may never be pieced together again. I have looked upon all that the universe has to hold of horror, and even the skies of spring and the flowers of summer must ever afterward be poison to me. But I do not think my life will be long. As my uncle went, as poor Johansen went, so I shall go. I know too much, and the cult still lives. 120 | Cthulhu still lives, too, I suppose, again in that chasm of stone which has shielded him since the sun was young. His accursed city is sunken once more, for the Vigilant sailed over the spot after the April storm; but his ministers on earth still bellow and prance and slay around idol-capped monoliths in lonely places. He must have been trapped by the sinking whilst within his black abyss, or else the world would by now be screaming with fright and frenzy. Who knows the end? What has risen may sink, and what has sunk may rise. Loathsomeness waits and dreams in the deep, and decay spreads over the tottering cities of men. A time will come—but I must not and cannot think! Let me pray that, if I do not survive this manuscript, my executors may put caution before audacity and see that it meets no other eye. -------------------------------------------------------------------------------- /datasets/text/LilDicky.txt: -------------------------------------------------------------------------------- 1 | [Spoken:] 2 | “Hello, how can I help you?” 3 | "Hi, my name’s Dave Burd. I have an appointment today. it’s probably under 'Lil Dicky,' actually. I’m a… I’m a rapper, so…” 4 | “Ok, and who are you here to see?” 5 | "Uh, Snoop Dogg? Says here “Snoop D-o-g-g.” 6 | "Oh, yeah. He’s right down the hall. First door to your right. Give him a knock and he should be right with you." 7 | "Thanks. It’s uh, this door right here, right?" 8 | "Mm-hmm.” 9 | “Thank you.” 10 | 11 | [Lil Dicky & Snoop Dogg:] 12 | Grab a seat 13 | Thanks I’m Dave very nice to meet 14 | Dave, what it do? You can call me Dogg 15 | Let me just check your sheet 16 | No problem, sir, let me just say 17 | I’m a big fan of the shit that y’all make 18 | I can’t lie, I been thinking about this all day 19 | All good, there’s a reason that we called, Dave 20 | Let me start with your background 21 | Where you come from? 22 | Sure, I was undergrad down there in Richmond 23 | Before that 24 | Oh, you talking bout my hometown? 25 | Yeah 26 | I was born out of Philly grew up in a little silly old town called Cheltenham 27 | It was in the suburbs, upper-middle wealth around 28 | So real shit you ain’t never had to struggle for much 29 | I wouldn’t say it like that, we just had a different kind of trap 30 | Elaborate 31 | Well I ain’t never had a tool, but I had to be the man at school 32 | Like I was doing shit I had to do so when I finished undergrad, I’m cool and I can get whatever job I wanted 33 | But the job you wanted wasn’t all that bumping 34 | Yeah, and I saw it quick all the flaws that be coming when you grow up like that 35 | Know you been racing them rats, you ain’t been making them raps 36 | Boo hoo what a hardship 37 | How you paid to get the rap shit started? 38 | Man, my Bar Mitzvah money 39 | But don’t diss me buddy, I wasn’t one of them younguns up on the block who had nothing to lose 40 | I must’ve wanted this a lot, I had something to choose 41 | Look at that, I can say there’s something to prove up in your ass 42 | Yeah, I guess 43 | That’s enough of the past, what makes you want to do rap? 44 | Oh my god, it’s the best 45 | Bitches let me draw up on their breasts 46 | Literally I can reinvent myself 47 | I get a forum to project myself 48 | It’s never boring, every morning I wake up and try to best myself 49 | I never got to be suppressed or stealthy to express myself 50 | It’s kinda healthy 51 | Cool, now let me put you through a couple hypotheticals 52 | Sure 53 | In a club and a couple niggas threaten you 54 | Uh, I wouldn’t do shit 55 | And I don’t even get what that got to do with this? 56 | Shut the fuck up, rap's like life 57 | If you wanna do this, then you won’t get far acting like a little bitch 58 | Nah, that’s my niche! 59 | Don’t get offended by this, but that’s the market y’all missed 60 | That’s the target I’ll hit 61 | And that’s the heart of my pitch 62 | I wanna do this whole thing different 63 | What the fuck you mean you wanna do the whole thing different? 64 | Uh, you know, I-I think, like, you know 65 | Traditionally people have been doing the job the same kind of way for a long time 66 | But traditionally speaking, this shit works, right? 67 | Nah, like, I get that, but I-I just think that, you know 68 | You don’t know if it could be working even better 69 | And I think you should look at me as an opportunity to find that out 70 | I don’t mean any disrespect by that 71 | I’m just saying I have a different background 72 | Like a different perspective way of looking at things than your typical applicant 73 | 74 | [Lil Dicky & Snoop Dogg:] 75 | So what are your biggest strengths? 76 | Well my flow is crazy, I can switch that amazing 77 | For example, I can be like ain’t nobody fucking with Lil Dicky 78 | When he get up on that shit and spit it sick and ridiculous 79 | Ripping this shit like it’s never been did and the rhythm is never predictable 80 | Cause I can switch it, deliver this shit to whatever cadence you bitches prefer 81 | Alright, alright, I get it 82 | Top of that, lyrically I’m pretty damn clever (Go) 83 | Got a long bottom bitch, called a hoe Neville (Whoa) 84 | And I blow it from the jump, Derrick Rose knee (Oh) 85 | You can say I’m pretty smart how I know better 86 | And y’all been winning like my story [?] it’s no feat 87 | Bro, I can go on and on 88 | I’m taking over any man like father in laws 89 | OK enough of the punchline, that ain’t showing me you different from the other guys 90 | Well I don’t know if they can run it like that 91 | But ain’t nobody else doing funny type rap 92 | What’s that? 93 | Well I can tell a story about my morning 94 | Watching Boy meets World and jerking off to Topanga Lawrence 95 | Like the 14 year old Topanga 96 | Oh this shit like a joke to you? 97 | I don’t get it 98 | Nah it ain’t like that I just happen to be a nutty abundantly funny type of individual 99 | Like it’s a guy 100 | So when i get up on the mic I ain’t finna just lie 101 | Real recognize real, right? 102 | Yeah, I guess 103 | So what you trying to do? 104 | Five years from now, tell me bout the dude 105 | Well, I don’t care about the money 106 | Like ,it’s the respect that I’m wanting 107 | Honestly, I just want to be one of the greats 108 | But I gotta bring your boy up, bury the bait 109 | I don’t wanna leave the game the same 110 | In a nutshell, what’s your legacy? 111 | Well, I wanna be the dude that came in and made the stand up rap 112 | With the random rap and the man like that for the people that was anti-rap 113 | Yeah the fans of rap started recognizing anti-rap is ironically one of the real brands of rap left 114 | That shit sound pretty damn complex 115 | I guess, I wanna be the best 116 | I just wanna do it my way 117 | And turn the whole game sideways 118 | And show people you ain’t gotta be resigned to the highway 119 | You can make a path while these motherfuckers drive straight 120 | I ain’t mad at that 121 | Well thanks, man, that’s my plan of my attack 122 | Just let me in and I’mma rap the plaques 123 | And I ain’t about to win them back to back 124 | I’m bout to win them like it’s back to back to back to back to back 125 | Til I stop rapping 126 | You ain’t even sold a damn album yet 127 | True 128 | That’s a lot of talk from a wiley vet 129 | Let alone a young rook nigga 130 | You misread me 131 | Nah this ain’t a book nigga 132 | Well that’s one of my flaws, I’m too competitive dawg 133 | No, you just fucking repetitive dawg 134 | I get it, you trying to be better than all 135 | But you ain’t shit until cheddars involved 136 | I get that I gotta prove myself 137 | No shit you ain’t done shit 138 | But think about that though 139 | Only doing this a couple of years 140 | One tape and I’m like this? 141 | This shit is aight for your first shit 142 | But you don’t know if it’s your worst shit 143 | You don’t think finding out would be worth it? 144 | It might be, it might not 145 | But I’mma put your ass to work, kid 146 | Wait, like, like I have the job? Like really? 147 | Well one more thing, let me see if you could put a hook together 148 | What would you do for the hook for this song? 149 | You want me to make a hook up right now? 150 | Do I look like the type of nigga that like repeating himself? 151 | Alright, uh, okay 152 | Man do the hook! 153 | I’m bout to be professional 154 | Homie, I’m professional 155 | Uh, maybe like a low voice type thing? 156 | Nigga, that shit is garbage man 157 | Well, you know, I feel like there might have been some upside to it 158 | But I’m not good at thinking of things on the spot like that 159 | Don’t y’all normally like outsource for that kind of thing sometimes? 160 | Sometimes, but you know, that was some next level Dogg shit right there 161 | Yeah, you know, I’m just way better when I get to think things through 162 | Have my like that dual screen action that type of thing 163 | Whatever man, you trying to smoke a blunt? 164 | Right now? 165 | Yeah 166 | Sure 167 | 168 | [Lil Dicky, Snoop Dogg & (Juanita):] 169 | Dope. Juanita, bring some weed in here, please. Thank you 170 | (You got it, Snoop. I’ll be right there.) 171 | Thanks, Juanita. Super nice woman 172 | Nigga, why does everything you say sound so soft? 173 | Uh, I really don’t wanna be spoken to like that, so… 174 | Whatever. Thanks 175 | (Hey, I got y’all y’all weed.) 176 | Damn, that’s some great looking weed. It’s just so early 177 | (Can I get you guys anything else while I’m here? Coffee? Tea? Head? Bottled water?) 178 | D-Did you just say head? 179 | (Yeah. You ain’t never got no head before?) 180 | I mean… Just feels like… A lot, right now 181 | (Ok.) 182 | But what kind of tea do you guys have? 183 | (We got mint, raspberry, earl grey, english breakfast...) 184 | Actually, I’ll take head 185 | (I knew your little dirty ass wanted some head.) 186 | 187 | LD 188 | A.k.a the Time Traveler's Wife 189 | 190 | [Verse 1 - Lil Dicky:] 191 | I like to play it cool like I'm not that 192 | On the low, who'd assume that I got that 193 | I don't know, but the dude with the tall frappe 194 | Looking all aloof being all that 195 | Even Babe Ruth wouldn't call that 196 | I don't even sweat it though 197 | They been hesitant as if my credit low 198 | And shit'll hit them quicker than an edible 199 | I'm 'bout to run for Senate ho, you ain't even centerfold 200 | Been on top of cheese, I ain't talking 'bout oregano 201 | I'm talking 'bout your cheddar homie, revel in that 202 | I'll hit a college and I'm fucked, like I'm pledging a frat 203 | They 'bout to silhouette my nuts on American flags 204 | Estoy contento, muy estupendo 205 | Better hearse word to rent-a-car, Dicky Tony K 206 | Y'all the Le Batard speaking to you lames, that's a seminar 207 | I'm straight like a pleasant bar, ain't nobody ready for my repertoire 208 | I wasn't getting credit like a debit card 209 | But never mind, had to give them time to adapt 210 | I'm kind of like a rap rendition of a fry in a wrap 211 | Just try it as that 212 | You rappers Rebel Wilson's vagina, you stank! 213 | I take it back, I don't know that ho 214 | And bro they used to 215 | 216 | [Hook:] 217 | Look around the boy, wouldn't raised they head 218 | Now they looking at the boy like the main event 219 | He don't even got a rap sheet, looking like a mathlete 220 | How the fuck is he the one that come in with the crack 221 | We like "Who knew, who knew?" 222 | Used to look at me like "who you, who you?" 223 | Now they look at me like "Who knew, who knew?" 224 | Now it's Dicky with this, who knew, who knew? 225 | Now they look at me like 226 | I am [?] 227 | Young boy got dough for a quiet nerd 228 | I am rap game Walter White 229 | You might get killed thinking that he all polite 230 | 231 | [Verse 2 - Lil Dicky:] 232 | Buzz around the city, coming out of Philly 233 | I'm about to get a milli, being me that's word to milli 234 | I'm looking super silly, but cooking like at Chili's 235 | You look at me like "Really?" but I look at you like "Who that?" 236 | Oh you new here, I'm the bomb 237 | Ok I'm Lebron, ok I'm the one 238 | Ok all that shit confusing that's a quandry 239 | What you call a pussy with a movement, that's a Ghandi 240 | I'm tryna get better but science preventing 241 | Because I'm undeniably clever, the highest of levels 242 | I'm high in a sweater but rhyming like I'm lying in pepper 243 | Don't mind the endeavor, I bet I do better than veterans 244 | Cheddaring, let him on Letterman 245 | Get him on L and I'm on, boy 246 | I think I need a therapist the way I get in my dome 247 | Doing D like they was Syracuse, when they up in they zone 248 | Used to load it on chrome, now I really ball 249 | Living like a fucking letter man, never mailing the song 250 | Though the dime flow rubbing combos in Tom's shoes 251 | With blonde hoes getting Peyton like the Broncos 252 | I'm on ho, vanilla looking but the rest of y'all the John Does 253 | It's pretty odd bro, cause they used to 254 | 255 | [Hook:] 256 | Look around the boy, wouldn't raised they head 257 | Now they looking at the boy like the main event 258 | He don't even got a rap sheet, looking like a mathlete 259 | How the fuck is he the one that come in with the crack 260 | We like "Who knew, who knew?" 261 | Used to look at me like "who you, who you?" 262 | Now they look at me like "Who knew, who knew?" 263 | Now it's Dicky with this, who knew, who knew? 264 | Now they look at me like 265 | I am [?] 266 | Young boy got dough for a quiet nerd 267 | I am rap game Walter White 268 | You might get killed thinking that he all polite 269 | 270 | Get up off my dick, ho 271 | That's an unassuming dick, though 272 | Get up off my dick, ho 273 | That's an unassuming dick, though 274 | ("Who knew, who knew?" 275 | Used to look at me like "Who you, who you?") 276 | Get up off my dick, ho 277 | That's an unassuming dick, though 278 | (Now they look at me like "Who knew, who knew?" 279 | Now it's Dicky with this, who knew, who knew?) 280 | Get up off my dick, ho 281 | That's an unassuming dick, though 282 | 283 | [Verse 1:] 284 | Drunk, faded, browned out, looking all shady 285 | Two months since I fucked a lady, young man dick going crazy 286 | Standard Saturday, I'm about to get a cab and masturbate 287 | Then I see a hot girl, look back at Dave 288 | Lemme get your cat girl, no Hathaway 289 | That's what I thought not what I said to her 290 | Instead I walk up to her, like "I know you heard of the kid 291 | You heard what I did, you heard of my shit, the murderous spit." 292 | She's like, "Naw, you're being weird, what are you talking about?" 293 | I'm like, "Aw..." 294 | Pulling out my phone, like a boss, YouTube 'bout to get her off 295 | But as I type the y-o-u, some porno pop up 296 | I'm like, "Hold up, how did, that's not even me 297 | Like, I don't jerk off mobily..." 298 | Then her friends coming up 299 | And wanna know if everything is okay 300 | I'm like, "Naw, this a rape, you can't tell? 301 | Gimme a break, please. Get the heck away." 302 | I turned to her, "Let me buy you a drink," she like, "Fine." 303 | Told the man, "Two Patron," she like, "Lime." 304 | Said he tried twice, the card got declined 305 | I'm like, "All ri-, is this a, could it be a machine issue? 306 | He was like, "Naw," 307 | I was like, "Obviously there's something wrong 308 | I got dough, like I'm not broke... 309 | I got donuts with the same card like about an hour ago." 310 | He like, "Dude, I don't know." 311 | Now I'm looking at the girl she just wanna go 312 | And I ain't talking 'bout with me, bro, I mean alone 313 | I got one more chance to prove myself, so I'm like 314 | 315 | [Pre-Hook:] 316 | "Look, I'm athletic, girl, I've gotten several Rec League MVP's 317 | At my crib I've got some pizza, plus a little bit of weed 318 | In my room I've got a TV, plus I recently did sheets 319 | Girl, I even have a fridge that has the water on the door 320 | Like with the crushed ice" 321 | 322 | [Hook:] 323 | You know I don't give a damn, what you playing right now 324 | This is me coming at you as a man right now 325 | Lemme freak, lemme freak, god dammit, lemme freak 326 | Just lemme freak, please god, just lemme freak 327 | 328 | [Verse 2:] 329 | The girl from the first verse, somehow let me fuck 330 | Fast forward, seven months, we in love 331 | Some real serious relationship type shit 332 | Despite this, I ain't fucked for days 333 | Tried last night, but was pushed away 334 | But I've been acting well behaved today, I smell okay 335 | Adele Pandora's playing, now she laying 336 | On the floor and I'm praying this bitch is horny 337 | I go and give her a smooch 338 | She kissing Dicky back, so I play with her boobs 339 | Bad move, she don't like that 340 | Bad mood, her boss being mean to her 341 | Via email, she wanna write back 342 | I'm like, "Right now?" 343 | She like, "Yeah, what do I say, can you just help me?" 344 | "Oh, okay, what's the context?" 345 | Then she like, "Well, he talks to Jane before coming to me." 346 | "Who's Jane again?" I said, she like, "Wow, you should know that..." 347 | "Okay, I guess I forgot," she like, "Dude, that's my other boss." 348 | "Okay, true, so shouldn't your first boss 349 | Go to the other boss, before you? Right?" 350 | "But Jane isn't hands on!" she yelled 351 | "Okay, well I just don't understand the dynamic then 352 | I don't work with these people." She started crying 353 | "Wait stop, what the fuck is this?" 354 | "Naw, you don't ever take my side." 355 | About an hour later til' I'm on her good side 356 | We in bed, hands on her good thighs 357 | I try to kiss, she like, "Good night" but I'm like 358 | 359 | [Pre-Hook:] 360 | "Look, I just turned off The Departed for a movie 'bout a bee 361 | I've been cutting back on farting, tweeting, arguing and weed 362 | Yesterday I wore a cardigan at dinner with your sweet 363 | And supportive aunt during the fourth quarter, of the Eagles 364 | Now reward it.." 365 | 366 | [Hook] 367 | 368 | [Verse 3:] 369 | Fast forward some more, June sixteenth, twenty seventy four 370 | Old LD looking old as fuck, still with the same ho, holding up 371 | But my life sucks, legs hurt, friends dead, real terse 372 | My dick looking like it's tinfoil 373 | Her tits looking like they hard boiled eggs 374 | On top of that, she insane 375 | Like, I don't even think she knows who I am 376 | Our kids moved away, we've been doing the same shit for days 377 | Maybe months, who knows, probably going to die soon 378 | Pretty scared, I ain't even gonna lie to you 379 | Think about it all day, on my last legs 380 | But I'm going out with a bang 381 | I got pills they invented back in fifty seven 382 | I took five, that's a bit excessive 383 | I look high, take a look at my thighs 384 | And good god that's a little erection, it'll work though 385 | Now I've gotta find the ho, I'm pretty much blind at this point 386 | I don't know if I mentioned that yet, but I am 387 | Made my way into the bedroom and there she is, I'm like 388 | 389 | [Pre-Hook:] 390 | "Look, I don't know if you're aware that 391 | You've been throwing out my shoes 392 | What I do know is, I'm sick of doing nothing here with you 393 | All my blood is in my dick, I'm probably dying pretty soon 394 | Are you even comprehending what I'm saying 395 | Please acknowledge that you hear me..." 396 | 397 | [Hook] 398 | 399 | I'm like girl that was great 400 | Talk about a connection 401 | Am I alone in my praise 402 | Do you share my assessment? 403 | She like yeah that was nice 404 | I enjoyed myself 405 | Great, yeah 406 | She like can you reach that shelf 407 | For my phone, I'll take care the alarm 408 | I'm like "oh, for the morning, like to sleep?" 409 | She like yeah we just had sex 410 | What did you think? 411 | I was just gonna fuck you and leave? 412 | No, I - I hadn't thought that far ahead 413 | But of course, can I get you a T-Shirt? 414 | OH, can you not tickle my hair? 415 | It's just a personal thing, it's not you 416 | Okay sorry for doing it 417 | No, it's not a personal thing, it's not you 418 | Okay cool. Yeah 419 | Uhhh, so you're from Minnesota? 420 | Only thing I know is that it's cold 421 | Break it down, how's it compare 422 | To living in LA? 423 | Well, I can't compare them they're so different 424 | Y'know what I'm sayin? 425 | Yeah, for sure 426 | She like apples to oranges 427 | Yeah, well you can still compare them but I hear ya 428 | What's the deal with your family? Got any siblings? 429 | She like I have a brother, I'm like oh what he do 430 | She like he in the Army. Oh, true 431 | Uhh, do you fuck with the war? 432 | She like huh, what did you just say? 433 | Huh, like just now? 434 | Yeah 435 | Do you fuck to the war? 436 | No I don't fuck with the war 437 | I'm like no I don't fuck with the war 438 | Just don't know how to react to the forces 439 | I should have just thanked you of course 440 | Uh, why would you thank me? 441 | Uh... I guess I assumed it would extend to the families but okay 442 | 443 | While we on the topic 444 | I been actually thinking about some shit 445 | About the Army and Navy 446 | What if tomorrow is the day 447 | That the fucking aliens came 448 | And invaded our nation? 449 | Like, would we even be able to fuck with their shit? 450 | Like do we have the type of weaponry to fuck with their shit? 451 | So not at all, would they just walk up in this motherfucker 452 | Laughin at us, and blastin at us 453 | And makin everybody disentegrate and assimilate 454 | Without a hint of intimidation? 455 | And can we be doin some shit to make they heart race? 456 | Granted I don't know the alien heart, but 457 | You get what the fuck I'm sayin? 458 | Like what the fuck would it be like? 459 | Would they be like Earth go hard? 460 | Or would it be just another conquest? 461 | Or would they be like damn earth go hard 462 | They was harder than Simian 463 | You ever talk to your brother about this type of stuff? 464 | Then she like no 465 | I'm like well what you think? 466 | She like I don't concern myself with hypotheticals that couldn't be 467 | I'm like girl what you mean? 468 | She like I don't believe 469 | In the war? 470 | In aliens 471 | Oh... hold up 472 | Wait, hold up 473 | Hold up 474 | Girl what you sayin? 475 | That all of the life in the universe happens to be where you stand? 476 | I guess 477 | What an enormous coincidence that shit would be 478 | Do you see what I'm sayin? 479 | I can't hear what you're sayin 480 | Well the whole universe is expandin 481 | There are infinite galaxies 482 | Why would we be the anomaly? 483 | She like that's no coincidence 484 | I call that shit god 485 | I'm like oh you're religous, bitch 486 | You think it's all god 487 | No I know it's all god 488 | No I know that there might be a god 489 | First off nobody knows there's a god 490 | Girl I think we should drop this 491 | No, what's goin on in your brain right now? 492 | Brain.. uhh, brain on some other shit tho 493 | I been all up in the club for the year hey brain 494 | I been killin shit don't mind me 495 | I'm just top flight 496 | I could sure use a Sprite 497 | I'm thirsty as fuck 498 | Who gettin money? 499 | That's brain 500 | Who make decisions? 501 | That's brain 502 | Who make the hit's 503 | That's been brain 504 | Who run the whole operation 505 | That be brain so 506 | Brain, can you just get back to the issue here 507 | Uh, can you just remind the brain? 508 | You're such an idiot, we're talking about God 509 | And if he exists 510 | Don't call brain names 511 | The brain couldn't recall 512 | But if I'm not mistaken the bitch on the left 513 | Guaranteed there's a god 514 | I believe in a god, yes 515 | So god never wrong 516 | God never wrong, right? 517 | Yeah, that's like the whole point 518 | Brain just get there, please 519 | Hold up. So god made the earth 520 | And god was like hold up, this shit is boring 521 | It need more shit 522 | God was like imma put dinosaurs on that bitch 523 | Dinosaurs on that bitch 524 | Then he like why'd I put dinosaurs on that shit 525 | What is your brain even saying? 526 | Can he get to the point 527 | Hold up brain you just did it 528 | God ain't wrong what the fuck was he thinking 529 | About what? 530 | Bitch the dinosaurs 531 | He made the Earth for them 532 | But then he like nah 533 | Dinosaurs was just blah 534 | Imma cook up some blondes 535 | He was way off 536 | I don't look like a dinosaur 537 | Ho them things 35 foot 538 | I'm like 5 foot 11 539 | But on Tinder I'm 6 foot 540 | Seen that Broncheosaurus 541 | That thing fuckin neck go to heaven 542 | And that's just an expression bitch 543 | There ain't no heaven 544 | I'm just messin but if he ain't wrong 545 | I guess this the exception 546 | Can I talk now? 547 | Go ahead 548 | Look 549 | Everything in life has purpose 550 | You, chickens, a midget at a circus 551 | I don't interpret 552 | Shit like that's not for me to determine 553 | So dinosaurs purpose was like to just die? 554 | That's not for me to interpret 555 | I'm just a person 556 | But think of the root of the argument girl 557 | View how we started this girl, you don't believe in the aliens 558 | You can find god the earth girl i find that shit silly and 559 | I just don't presume to know the plan 560 | Bitch me neither but that's not what I'm sayin 561 | But what are you sayin? 562 | Why can't God fuck with aliens? 563 | Why can't Earth just be like a fucking sideproject for this guy? 564 | Look we just don't see eye to eye 565 | Yeah but logically you don't believe in your stuff 566 | Yeah but you're like bringing up the dinosaurs 567 | We could have shared the earth with them nobody knows 568 | Damn there is hard soil evidence girl 569 | Like whats next you don't fuck with Pangea? 570 | She like lets change the subject 571 | That bitch don't know about Pangea 572 | Brain leave it alone 573 | I'm starving, are you hungry by chance? 574 | Oh my god I'm so hungry 575 | Yes, do you wanna get a pizza? 576 | Fuck yeah. fuck yeah, I'm on it 577 | Also I'm sorry about that whole interrogation thing 578 | Come on dude I'm a grown up 579 | I'm capable of having 580 | Intelligent conversations 581 | Okay i just fuck it 582 | Okay back to this pizza I'll eat pepperoni 583 | Ugh 584 | Sausage? 585 | How can you eat that shit 586 | No no no, this not dominoes 587 | This legit 588 | This high quality meat and shit 589 | No, no, I'm vegitarion 590 | Oh, well we can get half pepperoni 591 | And half of whatever you like 592 | Uh, ew, I can't have that shit 593 | In the same box 594 | Seriously? you're that against meat? 595 | No I'm that against us being gluttonous 596 | Thinking that animals live on this earth to get eat 597 | Okay, is it how we treat them? 598 | Uh, it sure doesn't help 599 | She like I'm not opposed to us huntin 600 | But now we not trackin them down 601 | We just breed them to eat 602 | That's disgusting 603 | You should honestly read on this subject 604 | 605 | Alright hold up 606 | Just because we not runnin around with a bow and arrow 607 | Doesn't mean we're not hunting these chickens 608 | We just decided the place to do our hunting intelligently enough 609 | To manipulate these animals and get them to do exactly what we want 610 | That's my point we just incubate animals 611 | Instead of just letting them live how they come 612 | But, what? that's just your God at work bitch 613 | Why we go hard on earth 614 | What you think happened right now 615 | If you left my place 616 | And there was like three wolves waiting for you 617 | Wolves? 618 | Bitch they would tear your ass up 619 | Viciously 620 | Like they would eat your titties 621 | And your pussy 622 | That's what they do 623 | And your face and shit 624 | That's what they do 625 | Humans are smarter bitch 626 | That's what we do 627 | She like apples to oranges 628 | Bitch that phrase don't make no sense 629 | Why can't fruit be compared? 630 | She like I'm gonna leave 631 | There's just all of these conflicting principals 632 | Right, enjoy your pepperoni pizza 633 | Right, i will, that's my god given right 634 | She like where is my bag 635 | Oh that leather one next to the trash? 636 | That's the one right? made out of cow, I think? 637 | Uhh why are there no ubers in this area 638 | So you don't eat the meat, you just wear the shit 639 | I'm not listening dude 640 | That's barbaric as shit 641 | Do you come from the vikings? 642 | Do you take peoples land? 643 | Do you fuck with the war? 644 | How are there no ubers out here? 645 | Just download lift, they have a bigger presence around here 646 | I don't wanna add shit to my phone 647 | Okay the choices are clear 648 | Ubers are not around here 649 | Taxis will come bu they real slow 650 | And the brain gotta poop 651 | T minus 5 till the brain gotta shit 652 | I just downloaded lift 653 | But it's being so glitchy 654 | Okay look do you just want to sleep here 655 | Uhh no 656 | It's past 2 am 657 | I'm not sharing a bed with you 658 | You can sleep on the bed I can sleep on the floor 659 | Fine you're sure 660 | Yeah its cool I won't throw you out to the wolves 661 | Oh my god you are so annoying 662 | I couldn't help myself 663 | But can you at least throw me a pillow 664 | 665 | I'm just pillow talkin with a bitch 666 | I'm just pillow talkin with a bitch 667 | I just finished passing on a bitch 668 | Now I'm pillow talking with a bitch 669 | 670 | Brain gotta poop still 671 | Please don't neglect the brain 672 | Please don't neglect the brain 673 | 674 | [bell rings] 675 | Wait is that my lift? 676 | 677 | Nah I got a pizza 678 | Full pepperoni too 679 | So you're not gonna wanna be a part of it 680 | 681 | But first you should poop 682 | Please don't neglect the brain -------------------------------------------------------------------------------- /saved_models/CallOfCthulhu.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evolvingstuff/RecurrentJava/b04a50c27e685585580edd33d5090f1dd5d349f0/saved_models/CallOfCthulhu.ser -------------------------------------------------------------------------------- /saved_models/LilDicky.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evolvingstuff/RecurrentJava/b04a50c27e685585580edd33d5090f1dd5d349f0/saved_models/LilDicky.ser -------------------------------------------------------------------------------- /saved_models/PaulGraham.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evolvingstuff/RecurrentJava/b04a50c27e685585580edd33d5090f1dd5d349f0/saved_models/PaulGraham.ser -------------------------------------------------------------------------------- /saved_models/PaulGraham2.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evolvingstuff/RecurrentJava/b04a50c27e685585580edd33d5090f1dd5d349f0/saved_models/PaulGraham2.ser -------------------------------------------------------------------------------- /src/ExampleEmbeddedReberGrammar.java: -------------------------------------------------------------------------------- 1 | import java.util.Random; 2 | 3 | import model.Model; 4 | import trainer.Trainer; 5 | import util.NeuralNetworkHelper; 6 | import datasets.EmbeddedReberGrammar; 7 | import datastructs.DataSet; 8 | 9 | public class ExampleEmbeddedReberGrammar { 10 | public static void main(String[] args) throws Exception { 11 | 12 | Random rng = new Random(); 13 | 14 | DataSet data = new EmbeddedReberGrammar(rng); 15 | 16 | int hiddenDimension = 12; 17 | int hiddenLayers = 1; 18 | double learningRate = 0.001; 19 | double initParamsStdDev = 0.08; 20 | 21 | Model nn = NeuralNetworkHelper.makeLstm( 22 | data.inputDimension, 23 | hiddenDimension, hiddenLayers, 24 | data.outputDimension, data.getModelOutputUnitToUse(), 25 | initParamsStdDev, rng); 26 | 27 | int reportEveryNthEpoch = 10; 28 | int trainingEpochs = 1000; 29 | 30 | Trainer.train(trainingEpochs, learningRate, nn, data, reportEveryNthEpoch, rng); 31 | 32 | System.out.println("done."); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/ExampleLilDicky.java: -------------------------------------------------------------------------------- 1 | import java.util.Random; 2 | 3 | import model.Model; 4 | import trainer.Trainer; 5 | import util.NeuralNetworkHelper; 6 | import datasets.TextGenerationUnbroken; 7 | import datastructs.DataSet; 8 | 9 | public class ExampleLilDicky { 10 | public static void main(String[] args) throws Exception { 11 | 12 | Random rng = new Random(); 13 | int totalSequences = 2000; 14 | int sequenceMinLength = 10; 15 | int sequenceMaxLength = 100; 16 | String textSource = "LilDicky"; 17 | DataSet data = new TextGenerationUnbroken("datasets/text/"+textSource+".txt", totalSequences, sequenceMinLength, sequenceMaxLength, rng); 18 | String savePath = "saved_models/"+textSource+".ser"; 19 | boolean initFromSaved = true; //set this to false to start with a fresh model 20 | boolean overwriteSaved = true; 21 | 22 | TextGenerationUnbroken.reportSequenceLength = 500; 23 | 24 | int bottleneckSize = 10; //one-hot input is squeezed through this 25 | int hiddenDimension = 200; 26 | int hiddenLayers = 1; 27 | double learningRate = 0.001; 28 | double initParamsStdDev = 0.08; 29 | 30 | Model lstm = NeuralNetworkHelper.makeLstmWithInputBottleneck( 31 | data.inputDimension, bottleneckSize, 32 | hiddenDimension, hiddenLayers, 33 | data.outputDimension, data.getModelOutputUnitToUse(), 34 | initParamsStdDev, rng); 35 | 36 | int reportEveryNthEpoch = 10; 37 | int trainingEpochs = 1000; 38 | 39 | Trainer.train(trainingEpochs, learningRate, lstm, data, reportEveryNthEpoch, initFromSaved, overwriteSaved, savePath, rng); 40 | 41 | System.out.println("done."); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/ExamplePaulGraham.java: -------------------------------------------------------------------------------- 1 | import java.util.Random; 2 | 3 | import model.Model; 4 | import trainer.Trainer; 5 | import util.NeuralNetworkHelper; 6 | import datasets.TextGeneration; 7 | import datastructs.DataSet; 8 | 9 | public class ExamplePaulGraham { 10 | public static void main(String[] args) throws Exception { 11 | 12 | /* 13 | * Character-by-character sentence prediction and generation, closely following the example here: 14 | * http://cs.stanford.edu/people/karpathy/recurrentjs/ 15 | */ 16 | 17 | String textSource = "PaulGraham"; 18 | DataSet data = new TextGeneration("datasets/text/"+textSource+".txt"); 19 | String savePath = "saved_models/"+textSource+".ser"; 20 | boolean initFromSaved = true; //set this to false to start with a fresh model 21 | boolean overwriteSaved = true; 22 | 23 | TextGeneration.reportSequenceLength = 100; 24 | TextGeneration.singleWordAutocorrect = false; //set this to true to constrain generated sentences to contain only words observed in the training data. 25 | 26 | int bottleneckSize = 10; //one-hot input is squeezed through this 27 | int hiddenDimension = 200; 28 | int hiddenLayers = 1; 29 | double learningRate = 0.001; 30 | double initParamsStdDev = 0.08; 31 | 32 | Random rng = new Random(); 33 | Model lstm = NeuralNetworkHelper.makeLstmWithInputBottleneck( 34 | data.inputDimension, bottleneckSize, 35 | hiddenDimension, hiddenLayers, 36 | data.outputDimension, data.getModelOutputUnitToUse(), 37 | initParamsStdDev, rng); 38 | 39 | int reportEveryNthEpoch = 10; 40 | int trainingEpochs = 1000; 41 | 42 | Trainer.train(trainingEpochs, learningRate, lstm, data, reportEveryNthEpoch, initFromSaved, overwriteSaved, savePath, rng); 43 | 44 | System.out.println("done."); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/ExampleQuestionAnswering.java: -------------------------------------------------------------------------------- 1 | import java.util.Random; 2 | 3 | import model.Model; 4 | import trainer.Trainer; 5 | import util.NeuralNetworkHelper; 6 | import datasets.bAbI; 7 | import datastructs.DataSet; 8 | 9 | public class ExampleQuestionAnswering { 10 | public static void main(String[] args) throws Exception { 11 | 12 | /* 13 | EXAMPLE OF LSTM RESULTS: 14 | 47.0% avg. accuracy on #1: Single Supporting Fact 15 | 32.7% avg. accuracy on #2: Two Supporting Facts 16 | 24.0% avg. accuracy on #3: Three Supporting Facts 17 | 58.6% avg. accuracy on #4: Two Arg. Relations 18 | 60.5% avg. accuracy on #5: Three Arg. Relations 19 | 64.1% avg. accuracy on #6: Yes/No Questions 20 | 76.3% avg. accuracy on #7: Counting 21 | 69.9% avg. accuracy on #8: Lists/Sets 22 | 61.2% avg. accuracy on #9: Simple Negation 23 | 52.6% avg. accuracy on #10: Indefinite Knowledge 24 | 67.8% avg. accuracy on #11: Basic Coreference 25 | 64.4% avg. accuracy on #12: Conjunction 26 | 89.6% avg. accuracy on #13: Compound Coreference 27 | 24.2% avg. accuracy on #14: Time Reasoning 28 | 29.5% avg. accuracy on #15: Basic Deduction 29 | 46.2% avg. accuracy on #16: Basic Induction 30 | 52.1% avg. accuracy on #17: Positional Reasoning 31 | 91.2% avg. accuracy on #18: Size Reasoning 32 | 8.0% avg. accuracy on #19: Path Finding 33 | 94.0% avg. accuracy on #20: Agent's Motivations 34 | 35 | EXAMPLE OF GRU RESULTS: 36 | 45.1% avg. accuracy on #1: Single Supporting Fact 37 | 28.3% avg. accuracy on #2: Two Supporting Facts 38 | 22.9% avg. accuracy on #3: Three Supporting Facts 39 | 64.0% avg. accuracy on #4: Two Arg. Relations 40 | 51.0% avg. accuracy on #5: Three Arg. Relations 41 | 62.3% avg. accuracy on #6: Yes/No Questions 42 | 72.1% avg. accuracy on #7: Counting 43 | 72.9% avg. accuracy on #8: Lists/Sets 44 | 64.2% avg. accuracy on #9: Simple Negation 45 | 52.5% avg. accuracy on #10: Indefinite Knowledge 46 | 64.1% avg. accuracy on #11: Basic Coreference 47 | 63.2% avg. accuracy on #12: Conjunction 48 | 92.7% avg. accuracy on #13: Compound Coreference 49 | 23.8% avg. accuracy on #14: Time Reasoning 50 | 29.3% avg. accuracy on #15: Basic Deduction 51 | 43.9% avg. accuracy on #16: Basic Induction 52 | 51.0% avg. accuracy on #17: Positional Reasoning 53 | 90.6% avg. accuracy on #18: Size Reasoning 54 | 9.2% avg. accuracy on #19: Path Finding 55 | 93.7% avg. accuracy on #20: Agent's Motivations 56 | */ 57 | 58 | Random rng = new Random(); 59 | int hiddenDimension = 10; 60 | 61 | int hiddenLayers = 1; 62 | double learningRate = 0.005; 63 | double initParamsStdDev = 0.08; 64 | int epochsPerTask = 50; 65 | int experiments = 1; 66 | 67 | boolean onlyShowSupportingFacts = false; 68 | 69 | double[] losses = new double[bAbI.TASK_NAMES.length]; 70 | 71 | for (int experiment = 0; experiment < experiments; experiment++) { 72 | for (int task = 0; task < bAbI.TASK_NAMES.length; task++) { 73 | 74 | int setId = task + 1; 75 | System.out.println("\n=============================================================="); 76 | System.out.println("bAbI experiment "+(experiment+1)+" of "+experiments); 77 | System.out.println("Task #" + setId + ": "+bAbI.TASK_NAMES[task]+"\n"); 78 | 79 | int totalExamples = 1000; 80 | 81 | DataSet data = new bAbI(setId, totalExamples, onlyShowSupportingFacts, rng); 82 | 83 | Model nn = NeuralNetworkHelper.makeLstm( 84 | data.inputDimension, 85 | hiddenDimension, hiddenLayers, 86 | data.outputDimension, data.getModelOutputUnitToUse(), 87 | initParamsStdDev, rng); 88 | 89 | /* 90 | Model nn = NeuralNetworkHelper.makeGru( 91 | data.inputDimension, 92 | hiddenDimension, hiddenLayers, 93 | data.outputDimension, data.getModelOutputUnitToUse(), 94 | initParamsStdDev, rng); 95 | //*/ 96 | 97 | int reportEveryNthEpoch = 10; 98 | double loss = Trainer.train(epochsPerTask, learningRate, nn, data, reportEveryNthEpoch, rng); 99 | losses[task] += loss; 100 | System.out.println("\nFINAL: " + String.format("%.1f", (100*(1-loss))) + "% accuracy"); 101 | } 102 | } 103 | 104 | System.out.println("\n\n=============================================================="); 105 | System.out.println("SUMMED RESULTS:"); 106 | for (int task = 0; task < bAbI.TASK_NAMES.length; task++) { 107 | System.out.println("\t" + String.format("%.1f", (100*(1-(losses[task]/(double)experiments)))) + "% avg. accuracy on #"+(task+1)+": " + bAbI.TASK_NAMES[task]); 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/autodiff/Graph.java: -------------------------------------------------------------------------------- 1 | package autodiff; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | 5 | import matrix.Matrix; 6 | import model.Nonlinearity; 7 | 8 | 9 | public class Graph { 10 | boolean applyBackprop; 11 | 12 | List backprop = new ArrayList<>(); 13 | 14 | public Graph() { 15 | this.applyBackprop = true; 16 | } 17 | 18 | public Graph(boolean applyBackprop) { 19 | this.applyBackprop = applyBackprop; 20 | } 21 | 22 | public void backward() { 23 | for (int i = backprop.size()-1; i >= 0; i--) { 24 | backprop.get(i).run(); 25 | } 26 | } 27 | 28 | public Matrix concatVectors(final Matrix m1, final Matrix m2) throws Exception { 29 | if (m1.cols > 1 || m2.cols > 1) { 30 | throw new Exception("Expected column vectors"); 31 | } 32 | final Matrix out = new Matrix(m1.rows + m2.rows); 33 | int loc = 0; 34 | for (int i = 0; i < m1.w.length; i++) { 35 | out.w[loc] = m1.w[i]; 36 | out.dw[loc] = m1.dw[i]; 37 | out.stepCache[loc] = m1.stepCache[i]; 38 | loc++; 39 | } 40 | for (int i = 0; i < m2.w.length; i++) { 41 | out.w[loc] = m2.w[i]; 42 | out.dw[loc] = m2.dw[i]; 43 | out.stepCache[loc] = m2.stepCache[i]; 44 | loc++; 45 | } 46 | if (this.applyBackprop) { 47 | Runnable bp = new Runnable() { 48 | public void run() { 49 | int loc = 0; 50 | for (int i = 0; i < m1.w.length; i++) { 51 | m1.w[i] = out.w[loc]; 52 | m1.dw[i] = out.dw[loc]; 53 | m1.stepCache[i] = out.stepCache[loc]; 54 | loc++; 55 | } 56 | for (int i = 0; i < m2.w.length; i++) { 57 | m2.w[i] = out.w[loc]; 58 | m2.dw[i] = out.dw[loc]; 59 | m2.stepCache[i] = out.stepCache[loc]; 60 | loc++; 61 | } 62 | } 63 | }; 64 | backprop.add(bp); 65 | } 66 | return out; 67 | } 68 | 69 | public Matrix nonlin(final Nonlinearity neuron, final Matrix m) throws Exception { 70 | final Matrix out = new Matrix(m.rows, m.cols); 71 | final int n = m.w.length; 72 | for (int i = 0; i < n; i++) { 73 | out.w[i] = neuron.forward(m.w[i]); 74 | } 75 | if (this.applyBackprop) { 76 | Runnable bp = new Runnable() { 77 | public void run() { 78 | for (int i = 0; i < n; i++) { 79 | m.dw[i] += neuron.backward(m.w[i]) * out.dw[i]; 80 | } 81 | } 82 | }; 83 | backprop.add(bp); 84 | } 85 | return out; 86 | } 87 | 88 | public Matrix mul(final Matrix m1, final Matrix m2) throws Exception { 89 | if (m1.cols != m2.rows) { 90 | throw new Exception("matrix dimension mismatch"); 91 | } 92 | 93 | final int m1rows = m1.rows; 94 | final int m1cols = m1.cols; 95 | final int m2cols = m2.cols; 96 | final Matrix out = new Matrix(m1rows, m2cols); 97 | final int outcols = m2cols; 98 | for (int i = 0; i < m1rows; i++) { 99 | int m1col = m1cols*i; 100 | for (int j = 0; j < m2cols; j++) { 101 | double dot = 0; 102 | for (int k = 0; k < m1cols; k++) { 103 | dot += m1.w[m1col + k] * m2.w[m2cols*k + j]; 104 | } 105 | out.w[outcols*i + j] = dot; 106 | } 107 | } 108 | if (this.applyBackprop) { 109 | Runnable bp = new Runnable() { 110 | public void run() { 111 | for (int i = 0; i < m1.rows; i++) { 112 | int outcol = outcols*i; 113 | for (int j = 0; j < m2.cols; j++) { 114 | double b = out.dw[outcol + j]; 115 | for (int k = 0; k < m1.cols; k++) { 116 | m1.dw[m1cols*i + k] += m2.w[m2cols*k + j] * b; 117 | m2.dw[m2cols*k + j] += m1.w[m1cols*i + k] * b; 118 | } 119 | } 120 | } 121 | } 122 | }; 123 | backprop.add(bp); 124 | } 125 | return out; 126 | } 127 | 128 | public Matrix add(final Matrix m1, final Matrix m2) throws Exception { 129 | if (m1.rows != m2.rows || m1.cols != m2.cols) { 130 | throw new Exception("matrix dimension mismatch"); 131 | } 132 | final Matrix out = new Matrix(m1.rows, m1.cols); 133 | for (int i = 0; i < m1.w.length; i++) { 134 | out.w[i] = m1.w[i] + m2.w[i]; 135 | } 136 | if (this.applyBackprop) { 137 | Runnable bp = new Runnable() { 138 | public void run() { 139 | for (int i = 0; i < m1.w.length; i++) { 140 | m1.dw[i] += out.dw[i]; 141 | m2.dw[i] += out.dw[i]; 142 | } 143 | } 144 | }; 145 | backprop.add(bp); 146 | } 147 | return out; 148 | } 149 | 150 | public Matrix oneMinus(final Matrix m) throws Exception { 151 | Matrix ones = Matrix.ones(m.rows, m.cols); 152 | Matrix out = sub(ones, m); 153 | return out; 154 | } 155 | 156 | public Matrix sub(final Matrix m1, final Matrix m2) throws Exception { 157 | Matrix out = add(m1, neg(m2)); 158 | return out; 159 | } 160 | 161 | public Matrix smul(final Matrix m, final double s) throws Exception { 162 | Matrix m2 = Matrix.uniform(m.rows, m.cols, s); 163 | Matrix out = elmul(m, m2); 164 | return out; 165 | } 166 | 167 | public Matrix smul(final double s, final Matrix m) throws Exception { 168 | Matrix out = smul(m, s); 169 | return out; 170 | } 171 | 172 | public Matrix neg(final Matrix m) throws Exception { 173 | Matrix negones = Matrix.negones(m.rows, m.cols); 174 | Matrix out = elmul(negones, m); 175 | return out; 176 | } 177 | 178 | public Matrix elmul(final Matrix m1, final Matrix m2) throws Exception { 179 | if (m1.rows != m2.rows || m1.cols != m2.cols) { 180 | throw new Exception("matrix dimension mismatch"); 181 | } 182 | final Matrix out = new Matrix(m1.rows, m1.cols); 183 | for (int i = 0; i < m1.w.length; i++) { 184 | out.w[i] = m1.w[i] * m2.w[i]; 185 | } 186 | if (this.applyBackprop) { 187 | Runnable bp = new Runnable() { 188 | public void run() { 189 | for (int i = 0; i < m1.w.length; i++) { 190 | m1.dw[i] += m2.w[i] * out.dw[i]; 191 | m2.dw[i] += m1.w[i] * out.dw[i]; 192 | } 193 | } 194 | }; 195 | backprop.add(bp); 196 | } 197 | return out; 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /src/datasets/EmbeddedReberGrammar.java: -------------------------------------------------------------------------------- 1 | package datasets; 2 | 3 | import java.util.*; 4 | 5 | import datastructs.DataSequence; 6 | import datastructs.DataSet; 7 | import datastructs.DataStep; 8 | import loss.LossMultiDimensionalBinary; 9 | import loss.LossSumOfSquares; 10 | import model.Model; 11 | import model.Nonlinearity; 12 | import model.SigmoidUnit; 13 | 14 | public class EmbeddedReberGrammar extends DataSet { 15 | 16 | static public class State { 17 | public State(Transition[] transitions) { 18 | this.transitions = transitions; 19 | } 20 | public Transition[] transitions; 21 | } 22 | 23 | static public class Transition { 24 | public Transition(int next_state_id, int token) { 25 | this.next_state_id = next_state_id; 26 | this.token = token; 27 | } 28 | public int next_state_id; 29 | public int token; 30 | } 31 | 32 | public EmbeddedReberGrammar(Random r) throws Exception { 33 | int total_sequences = 1000; 34 | inputDimension = 7; 35 | outputDimension = 7; 36 | lossTraining = new LossSumOfSquares(); 37 | lossReporting = new LossMultiDimensionalBinary(); 38 | training = generateSequences(r, total_sequences); 39 | validation = generateSequences(r, total_sequences); 40 | testing = generateSequences(r, total_sequences); 41 | } 42 | 43 | public static List generateSequences(Random r, int sequences) { 44 | 45 | List result = new ArrayList<>(); 46 | 47 | final int B = 0; 48 | final int T = 1; 49 | final int P = 2; 50 | final int S = 3; 51 | final int X = 4; 52 | final int V = 5; 53 | final int E = 6; 54 | 55 | State[] states = new State[19]; 56 | states[0] = new State(new Transition[] {new Transition(1,B)}); 57 | states[1] = new State(new Transition[] {new Transition(2,T), new Transition(11,P)}); 58 | states[2] = new State(new Transition[] {new Transition(3,B)}); 59 | states[3] = new State(new Transition[] {new Transition(4,T), new Transition(9,P)}); 60 | states[4] = new State(new Transition[] {new Transition(4,S), new Transition(5,X)}); 61 | states[5] = new State(new Transition[] {new Transition(6,S), new Transition(9,X)}); 62 | states[6] = new State(new Transition[] {new Transition(7,E)}); 63 | states[7] = new State(new Transition[] {new Transition(8,T)}); 64 | states[8] = new State(new Transition[] {new Transition(0,E)}); 65 | states[9] = new State(new Transition[] {new Transition(9,T), new Transition(10,V)}); 66 | states[10] = new State(new Transition[] {new Transition(5,P), new Transition(6,V)}); 67 | states[11] = new State(new Transition[] {new Transition(12,B)}); 68 | states[12] = new State(new Transition[] {new Transition(13,T), new Transition(17,P)}); 69 | states[13] = new State(new Transition[] {new Transition(13,S), new Transition(14,X)}); 70 | states[14] = new State(new Transition[] {new Transition(15,S), new Transition(17,X)}); 71 | states[15] = new State(new Transition[] {new Transition(16,E)}); 72 | states[16] = new State(new Transition[] {new Transition(8,P)}); 73 | states[17] = new State(new Transition[] {new Transition(17,T), new Transition(18,V)}); 74 | states[18] = new State(new Transition[] {new Transition(14,P), new Transition(15,V)}); 75 | 76 | for (int sequence = 0; sequence < sequences; sequence++) { 77 | List steps = new ArrayList<>();; 78 | int state_id = 0; 79 | while (true) { 80 | int transition = -1; 81 | if (states[state_id].transitions.length == 1) { 82 | transition = 0; 83 | } 84 | else if (states[state_id].transitions.length == 2) { 85 | transition = r.nextInt(2); 86 | } 87 | double[] observation = null; 88 | 89 | observation = new double[7]; 90 | observation[states[state_id].transitions[transition].token] = 1.0; 91 | 92 | state_id = states[state_id].transitions[transition].next_state_id; 93 | if (state_id == 0) { //exit at end of sequence 94 | break; 95 | } 96 | double[] target_output = new double[7]; 97 | for (int i = 0; i < states[state_id].transitions.length; i++) { 98 | target_output[states[state_id].transitions[i].token] = 1.0; 99 | } 100 | steps.add(new DataStep(observation, target_output)); 101 | } 102 | result.add(new DataSequence(steps)); 103 | } 104 | return result; 105 | } 106 | 107 | @Override 108 | public void DisplayReport(Model model, Random rng) throws Exception { 109 | // TODO Auto-generated method stub 110 | 111 | } 112 | 113 | @Override 114 | public Nonlinearity getModelOutputUnitToUse() { 115 | return new SigmoidUnit(); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/datasets/SequentialParity.java: -------------------------------------------------------------------------------- 1 | package datasets; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import loss.LossMultiDimensionalBinary; 7 | import loss.LossSumOfSquares; 8 | import matrix.Matrix; 9 | import model.Model; 10 | import model.Nonlinearity; 11 | import model.SigmoidUnit; 12 | import datastructs.DataSequence; 13 | import datastructs.DataSet; 14 | import datastructs.DataStep; 15 | 16 | 17 | public class SequentialParity extends DataSet { 18 | 19 | public SequentialParity(Random r, int total_sequences, int max_sequence_length_train, int max_sequence_length_test) { 20 | inputDimension = 1; 21 | outputDimension = 1; 22 | lossTraining = new LossSumOfSquares(); 23 | lossReporting = new LossMultiDimensionalBinary(); 24 | training = generateSequences(r, total_sequences, max_sequence_length_train); 25 | 26 | //training.addAll(generateSequences(r, total_sequences, max_sequence_length_test)); 27 | 28 | validation = generateSequences(r, total_sequences, max_sequence_length_train); 29 | testing = generateSequences(r, total_sequences, max_sequence_length_test); 30 | } 31 | 32 | private static List generateSequences(Random r, int total_sequences, int max_sequence_length) { 33 | List result = new ArrayList<>();; 34 | for (int s = 0; s < total_sequences; s++) { 35 | DataSequence sequence = new DataSequence(); 36 | int tot = 0; 37 | int tempSequenceLength = r.nextInt(max_sequence_length) + 1; 38 | for (int t = 0; t < tempSequenceLength; t++) { 39 | DataStep step = new DataStep(); 40 | double[] input = {0.0}; 41 | 42 | if (r.nextDouble() < 0.5) { 43 | input[0] = 1.0; 44 | tot++; 45 | } 46 | step.input = new Matrix(input); 47 | 48 | double[] targetOutput = null; 49 | if (t == tempSequenceLength - 1) { 50 | targetOutput = new double[1]; 51 | targetOutput[0] = tot%2; 52 | step.targetOutput = new Matrix(targetOutput); 53 | } 54 | sequence.steps.add(step); 55 | } 56 | result.add(sequence); 57 | } 58 | return result; 59 | } 60 | 61 | @Override 62 | public void DisplayReport(Model model, Random rng) throws Exception { 63 | // TODO Auto-generated method stub 64 | } 65 | 66 | @Override 67 | public Nonlinearity getModelOutputUnitToUse() { 68 | return new SigmoidUnit(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/datasets/TextGeneration.java: -------------------------------------------------------------------------------- 1 | package datasets; 2 | import java.io.File; 3 | import java.nio.charset.Charset; 4 | import java.nio.file.Files; 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.HashSet; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Random; 11 | import java.util.Set; 12 | 13 | import autodiff.Graph; 14 | import datastructs.DataSequence; 15 | import datastructs.DataSet; 16 | import datastructs.DataStep; 17 | import util.Util; 18 | import loss.LossSoftmax; 19 | import matrix.Matrix; 20 | import model.LinearUnit; 21 | import model.Model; 22 | import model.Nonlinearity; 23 | 24 | 25 | public class TextGeneration extends DataSet { 26 | 27 | public static int reportSequenceLength = 100; 28 | public static boolean singleWordAutocorrect = false; 29 | public static boolean reportPerplexity = true; 30 | private static Map charToIndex = new HashMap<>(); 31 | private static Map indexToChar = new HashMap<>(); 32 | private static int dimension; 33 | private static double[] vecStartEnd; 34 | private static final int START_END_TOKEN_INDEX = 0; 35 | private static Set words = new HashSet<>(); 36 | 37 | public static List generateText(Model model, int steps, boolean argmax, double temperature, Random rng) throws Exception { 38 | List lines = new ArrayList<>(); 39 | Matrix start = new Matrix(dimension); 40 | start.w[START_END_TOKEN_INDEX] = 1.0; 41 | model.resetState(); 42 | Graph g = new Graph(false); 43 | Matrix input = start.clone(); 44 | String line = ""; 45 | for (int s = 0; s < steps; s++) { 46 | Matrix logprobs = model.forward(input, g); 47 | Matrix probs = LossSoftmax.getSoftmaxProbs(logprobs, temperature); 48 | 49 | if (singleWordAutocorrect) { 50 | Matrix possible = Matrix.ones(dimension, 1); 51 | try { 52 | possible = singleWordAutocorrect(line); 53 | } 54 | catch (Exception e) { 55 | //TODO: still may be some lingering bugs, so don't constrain by possible if a problem occurs. Fix later.. 56 | } 57 | double tot = 0; 58 | //remove impossible transitions 59 | for (int i = 0; i < probs.w.length; i++) { 60 | probs.w[i] *= possible.w[i]; 61 | tot += probs.w[i]; 62 | } 63 | 64 | //normalize to sum of 1.0 again 65 | for (int i = 0; i < probs.w.length; i++) { 66 | probs.w[i] /= tot; 67 | } 68 | 69 | for (int i = 0; i < probs.w.length; i++) { 70 | if (probs.w[i] > 0 && possible.w[i] == 0) { 71 | throw new Exception("Illegal transition"); 72 | } 73 | } 74 | } 75 | 76 | int indxChosen = -1; 77 | if (argmax) { 78 | double high = Double.NEGATIVE_INFINITY; 79 | for (int i = 0; i < probs.w.length; i++) { 80 | if (probs.w[i] > high) { 81 | high = probs.w[i]; 82 | indxChosen = i; 83 | } 84 | } 85 | } 86 | else { 87 | indxChosen = Util.pickIndexFromRandomVector(probs, rng); 88 | } 89 | if (indxChosen == START_END_TOKEN_INDEX) { 90 | lines.add(line); 91 | line = ""; 92 | input = start.clone(); 93 | g = new Graph(false); 94 | model.resetState(); 95 | input = start.clone(); 96 | } 97 | else { 98 | String ch = indexToChar.get(indxChosen); 99 | line += ch; 100 | for (int i = 0; i < input.w.length; i++) { 101 | input.w[i] = 0; 102 | } 103 | input.w[indxChosen] = 1.0; 104 | } 105 | } 106 | if (line.equals("") == false) { 107 | lines.add(line); 108 | } 109 | return lines; 110 | } 111 | 112 | private static Matrix singleWordAutocorrect(String sequence) throws Exception { 113 | 114 | /* 115 | * This restricts the output of the RNN to being composed of words found in the source text. 116 | * It makes no attempts to account for probabilities in any way. 117 | */ 118 | 119 | sequence = sequence.replace("\"\n\"", " "); 120 | if (sequence.equals("") || sequence.endsWith(" ")) { //anything is possible after a space 121 | return Matrix.ones(dimension, 1); 122 | } 123 | String[] parts = sequence.split(" "); 124 | String lastPartialWord = parts[parts.length-1].trim(); 125 | if (lastPartialWord.equals(" ") || lastPartialWord.contains(" ")) { 126 | throw new Exception("unexpected"); 127 | } 128 | List matches = new ArrayList<>(); 129 | for (String word : words) { 130 | if (word.startsWith(lastPartialWord)) { 131 | matches.add(word); 132 | } 133 | } 134 | if (matches.size() == 0) { 135 | throw new Exception("unexpected, no matches for '"+lastPartialWord+"'"); 136 | } 137 | Matrix result = new Matrix(dimension); 138 | boolean hit = false; 139 | for (String match : matches) { 140 | if (match.length() < lastPartialWord.length()) { 141 | throw new Exception("How is match shorter than partial word?"); 142 | } 143 | if (lastPartialWord.equals(match)) { 144 | result.w[charToIndex.get(" ")] = 1.0; 145 | result.w[START_END_TOKEN_INDEX] = 1.0; 146 | continue; 147 | } 148 | 149 | String nextChar = match.charAt(lastPartialWord.length()) + ""; 150 | result.w[charToIndex.get(nextChar)] = 1.0; 151 | hit = true; 152 | } 153 | if (hit == false) { 154 | result.w[charToIndex.get(" ")] = 1.0; 155 | result.w[START_END_TOKEN_INDEX] = 1.0; 156 | } 157 | return result; 158 | 159 | } 160 | 161 | public static String sequenceToSentence(DataSequence sequence) { 162 | String result = "\""; 163 | for (int s = 0; s < sequence.steps.size() - 1; s++) { 164 | DataStep step = sequence.steps.get(s); 165 | int index = -1; 166 | for (int i = 0; i < step.targetOutput.w.length; i++) { 167 | if (step.targetOutput.w[i] == 1) { 168 | index = i; 169 | break; 170 | } 171 | } 172 | String ch = indexToChar.get(index); 173 | result += ch; 174 | } 175 | result += "\"\n"; 176 | return result; 177 | } 178 | 179 | public TextGeneration(String path) throws Exception { 180 | 181 | System.out.println("Text generation task"); 182 | System.out.println("loading " + path + "..."); 183 | 184 | File file = new File(path); 185 | List lines = Files.readAllLines(file.toPath(), Charset.defaultCharset()); 186 | Set chars = new HashSet<>(); 187 | int id = 0; 188 | 189 | charToIndex.put("[START/END]", id); 190 | indexToChar.put(id, "[START/END]"); 191 | id++; 192 | 193 | System.out.println("Characters:"); 194 | 195 | System.out.print("\t"); 196 | 197 | for (String line : lines) { 198 | for (int i = 0; i < line.length(); i++) { 199 | 200 | String[] parts = line.split(" "); 201 | for (String part : parts) { 202 | words.add(part.trim()); 203 | } 204 | 205 | String ch = line.charAt(i) + ""; 206 | if (chars.contains(ch) == false) { 207 | System.out.print(ch); 208 | chars.add(ch); 209 | charToIndex.put(ch, id); 210 | indexToChar.put(id, ch); 211 | id++; 212 | } 213 | } 214 | } 215 | 216 | dimension = chars.size() + 1; 217 | vecStartEnd = new double[dimension]; 218 | vecStartEnd[START_END_TOKEN_INDEX] = 1.0; 219 | 220 | List sequences = new ArrayList<>(); 221 | int size = 0; 222 | for (String line : lines) { 223 | List vecs = new ArrayList<>(); 224 | vecs.add(vecStartEnd); 225 | for (int i = 0; i < line.length(); i++) { 226 | String ch = line.charAt(i) + ""; 227 | int index = charToIndex.get(ch); 228 | double[] vec = new double[dimension]; 229 | vec[index] = 1.0; 230 | vecs.add(vec); 231 | } 232 | vecs.add(vecStartEnd); 233 | 234 | DataSequence sequence = new DataSequence(); 235 | for (int i = 0; i < vecs.size() - 1; i++) { 236 | sequence.steps.add(new DataStep(vecs.get(i), vecs.get(i+1))); 237 | size++; 238 | } 239 | sequences.add(sequence); 240 | } 241 | System.out.println("Total unique chars = " + chars.size()); 242 | System.out.println(size + " steps in training set."); 243 | 244 | training = sequences; 245 | lossTraining = new LossSoftmax(); 246 | lossReporting = new LossSoftmax(); 247 | inputDimension = sequences.get(0).steps.get(0).input.w.length; 248 | int loc = 0; 249 | while (sequences.get(0).steps.get(loc).targetOutput == null) { 250 | loc++; 251 | } 252 | outputDimension = sequences.get(0).steps.get(loc).targetOutput.w.length; 253 | } 254 | 255 | @Override 256 | public void DisplayReport(Model model, Random rng) throws Exception { 257 | System.out.println("========================================"); 258 | System.out.println("REPORT:"); 259 | if (reportPerplexity) { 260 | System.out.println("\ncalculating perplexity over entire data set..."); 261 | double perplexity = LossSoftmax.calculateMedianPerplexity(model, training); 262 | System.out.println("\nMedian Perplexity = " + String.format("%.4f", perplexity)); 263 | } 264 | double[] temperatures = {1, 0.75, 0.5, 0.25, 0.1}; 265 | for (double temperature : temperatures) { 266 | if (TextGeneration.singleWordAutocorrect) { 267 | System.out.println("\nTemperature "+temperature+" prediction (with single word autocorrect):"); 268 | } 269 | else { 270 | System.out.println("\nTemperature "+temperature+" prediction:"); 271 | } 272 | List guess = TextGeneration.generateText(model, reportSequenceLength, false, temperature, rng); 273 | for (int i = 0; i < guess.size(); i++) { 274 | if (i == guess.size()-1) { 275 | System.out.println("\t\"" + guess.get(i) + "...\""); 276 | } 277 | else { 278 | System.out.println("\t\"" + guess.get(i) + "\""); 279 | } 280 | 281 | } 282 | } 283 | if (TextGeneration.singleWordAutocorrect) { 284 | System.out.println("\nArgmax prediction (with single word autocorrect):"); 285 | } 286 | else { 287 | System.out.println("\nArgmax prediction:"); 288 | } 289 | List guess = TextGeneration.generateText(model, reportSequenceLength, true, 1.0, rng); 290 | for (int i = 0; i < guess.size(); i++) { 291 | if (i == guess.size()-1) { 292 | System.out.println("\t\"" + guess.get(i) + "...\""); 293 | } 294 | else { 295 | System.out.println("\t\"" + guess.get(i) + "\""); 296 | } 297 | 298 | } 299 | System.out.println("========================================"); 300 | } 301 | 302 | @Override 303 | public Nonlinearity getModelOutputUnitToUse() { 304 | return new LinearUnit(); 305 | } 306 | } 307 | -------------------------------------------------------------------------------- /src/datasets/TextGenerationUnbroken.java: -------------------------------------------------------------------------------- 1 | package datasets; 2 | import java.io.File; 3 | import java.nio.charset.Charset; 4 | import java.nio.file.Files; 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.HashSet; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Random; 11 | import java.util.Set; 12 | 13 | import autodiff.Graph; 14 | import datastructs.DataSequence; 15 | import datastructs.DataSet; 16 | import datastructs.DataStep; 17 | import util.Util; 18 | import loss.LossSoftmax; 19 | import matrix.Matrix; 20 | import model.LinearUnit; 21 | import model.Model; 22 | import model.Nonlinearity; 23 | 24 | 25 | public class TextGenerationUnbroken extends DataSet { 26 | 27 | private static final long serialVersionUID = 1L; 28 | public static int reportSequenceLength = 100; 29 | public static boolean reportPerplexity = true; 30 | private static Map charToIndex = new HashMap<>(); 31 | private static Map indexToChar = new HashMap<>(); 32 | private static int dimension; 33 | 34 | public static String generateText(Model model, int steps, boolean argmax, double temperature, Random rng) throws Exception { 35 | Matrix start = new Matrix(dimension); 36 | model.resetState(); 37 | Graph g = new Graph(false); 38 | Matrix input = start.clone(); 39 | String result = ""; 40 | for (int s = 0; s < steps; s++) { 41 | Matrix logprobs = model.forward(input, g); 42 | Matrix probs = LossSoftmax.getSoftmaxProbs(logprobs, temperature); 43 | 44 | int indxChosen = -1; 45 | if (argmax) { 46 | double high = Double.NEGATIVE_INFINITY; 47 | for (int i = 0; i < probs.w.length; i++) { 48 | if (probs.w[i] > high) { 49 | high = probs.w[i]; 50 | indxChosen = i; 51 | } 52 | } 53 | } 54 | else { 55 | indxChosen = Util.pickIndexFromRandomVector(probs, rng); 56 | } 57 | String ch = indexToChar.get(indxChosen); 58 | result += ch; 59 | for (int i = 0; i < input.w.length; i++) { 60 | input.w[i] = 0; 61 | } 62 | input.w[indxChosen] = 1.0; 63 | } 64 | result = result.replace("\n", "\"\n\t\""); 65 | return result; 66 | } 67 | 68 | public TextGenerationUnbroken(String path, int totalSequences, int sequenceMinLength, int sequenceMaxLength, Random rng) throws Exception { 69 | 70 | System.out.println("Text generation task"); 71 | System.out.println("loading " + path + "..."); 72 | 73 | File file = new File(path); 74 | List lines_ = Files.readAllLines(file.toPath(), Charset.defaultCharset()); 75 | 76 | String text = ""; 77 | for (String line : lines_) { 78 | text += line + "\n"; 79 | } 80 | 81 | Set chars = new HashSet<>(); 82 | int id = 0; 83 | 84 | System.out.println("Characters:"); 85 | 86 | System.out.print("\t"); 87 | 88 | for (int i = 0; i < text.length(); i++) { 89 | String ch = text.charAt(i) + ""; 90 | if (chars.contains(ch) == false) { 91 | if (ch.equals("\n")) { 92 | System.out.print("\\n"); 93 | } 94 | else { 95 | System.out.print(ch); 96 | } 97 | chars.add(ch); 98 | charToIndex.put(ch, id); 99 | indexToChar.put(id, ch); 100 | id++; 101 | } 102 | } 103 | System.out.println(""); 104 | 105 | dimension = chars.size(); 106 | 107 | List sequences = new ArrayList<>(); 108 | 109 | for (int s = 0; s < totalSequences; s++) { 110 | List vecs = new ArrayList<>(); 111 | int len = rng.nextInt(sequenceMaxLength - sequenceMinLength + 1) + sequenceMinLength; 112 | int start = rng.nextInt(text.length() - len); 113 | for (int i = 0; i < len; i++) { 114 | String ch = text.charAt(i+start) + ""; 115 | int index = charToIndex.get(ch); 116 | double[] vec = new double[dimension]; 117 | vec[index] = 1.0; 118 | vecs.add(vec); 119 | } 120 | DataSequence sequence = new DataSequence(); 121 | for (int i = 0; i < vecs.size() - 1; i++) { 122 | sequence.steps.add(new DataStep(vecs.get(i), vecs.get(i+1))); 123 | } 124 | sequences.add(sequence); 125 | } 126 | 127 | System.out.println("Total unique chars = " + chars.size()); 128 | 129 | training = sequences; 130 | lossTraining = new LossSoftmax(); 131 | lossReporting = new LossSoftmax(); 132 | inputDimension = sequences.get(0).steps.get(0).input.w.length; 133 | int loc = 0; 134 | while (sequences.get(0).steps.get(loc).targetOutput == null) { 135 | loc++; 136 | } 137 | outputDimension = sequences.get(0).steps.get(loc).targetOutput.w.length; 138 | } 139 | 140 | @Override 141 | public void DisplayReport(Model model, Random rng) throws Exception { 142 | System.out.println("========================================"); 143 | System.out.println("REPORT:"); 144 | if (reportPerplexity) { 145 | System.out.println("\ncalculating perplexity over entire data set..."); 146 | double perplexity = LossSoftmax.calculateMedianPerplexity(model, training); 147 | System.out.println("\nMedian Perplexity = " + String.format("%.4f", perplexity)); 148 | } 149 | double[] temperatures = {1, 0.75, 0.5, 0.25, 0.1}; 150 | for (double temperature : temperatures) { 151 | System.out.println("\nTemperature "+temperature+" prediction:"); 152 | String guess = TextGenerationUnbroken.generateText(model, reportSequenceLength, false, temperature, rng); 153 | System.out.println("\t\"..." + guess + "...\""); 154 | } 155 | System.out.println("\nArgmax prediction:"); 156 | String guess = TextGenerationUnbroken.generateText(model, reportSequenceLength, true, 1.0, rng); 157 | System.out.println("\t\"..." + guess + "...\""); 158 | System.out.println("========================================"); 159 | } 160 | 161 | @Override 162 | public Nonlinearity getModelOutputUnitToUse() { 163 | return new LinearUnit(); 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /src/datasets/bAbI.java: -------------------------------------------------------------------------------- 1 | package datasets; 2 | 3 | 4 | import java.io.File; 5 | import java.nio.charset.Charset; 6 | import java.nio.file.Files; 7 | import java.util.ArrayList; 8 | import java.util.Collections; 9 | import java.util.HashSet; 10 | import java.util.List; 11 | import java.util.Random; 12 | import java.util.Set; 13 | 14 | import datastructs.DataSequence; 15 | import datastructs.DataSet; 16 | import datastructs.DataStep; 17 | import loss.LossArgMax; 18 | import loss.LossSoftmax; 19 | import model.LinearUnit; 20 | import model.Model; 21 | import model.Nonlinearity; 22 | 23 | /* 24 | * Based on https://research.facebook.com/researchers/1543934539189348 25 | */ 26 | 27 | public class bAbI extends DataSet{ 28 | 29 | public static void main(String[] args) throws Exception { 30 | System.out.println("testing..."); 31 | 32 | Random rng = new Random(); 33 | bAbI data = new bAbI(3, 100, true, rng); 34 | 35 | System.out.println("done."); 36 | } 37 | 38 | public bAbI(int setId, int totalExamples, boolean onlySupportingFacts, Random rng) throws Exception { 39 | final File folder = new File("datasets/bAbI/en/"); 40 | List fileNamesTrain = new ArrayList<>(); 41 | List fileNamesTest = new ArrayList<>(); 42 | for (final File fileEntry : folder.listFiles()) { 43 | String path = fileEntry.getPath(); 44 | if (path.contains("train")) { 45 | if (path.contains("qa"+setId+"_")) { 46 | fileNamesTrain.add(path); 47 | } 48 | } 49 | else if (path.contains("test")) { 50 | if (path.contains("qa"+setId+"_")) { 51 | fileNamesTest.add(path); 52 | } 53 | } 54 | else { 55 | throw new Exception("Unknown file type"); 56 | } 57 | } 58 | 59 | List storiesTrain = getStories(fileNamesTrain, onlySupportingFacts); 60 | List storiesTest = getStories(fileNamesTest, onlySupportingFacts); 61 | 62 | while (storiesTrain.size() > totalExamples) { 63 | storiesTrain.remove(rng.nextInt(storiesTrain.size())); 64 | } 65 | while (storiesTest.size() > totalExamples) { 66 | storiesTest.remove(rng.nextInt(storiesTest.size())); 67 | } 68 | 69 | configureVocab(storiesTrain, storiesTest); 70 | 71 | training = getSequences(storiesTrain); 72 | testing = getSequences(storiesTest); 73 | validation = null; 74 | 75 | lossTraining = new LossSoftmax(); 76 | lossReporting = new LossArgMax(); 77 | 78 | inputDimension = training.get(0).steps.get(0).input.w.length; 79 | 80 | int loc = 0; 81 | while (training.get(0).steps.get(loc).targetOutput == null) { 82 | loc++; 83 | } 84 | outputDimension = training.get(0).steps.get(loc).targetOutput.w.length; 85 | } 86 | 87 | public static final String[] TASK_NAMES = { 88 | "Single Supporting Fact", 89 | "Two Supporting Facts", 90 | "Three Supporting Facts", 91 | "Two Arg. Relations", 92 | "Three Arg. Relations", 93 | "Yes/No Questions", 94 | "Counting", 95 | "Lists/Sets", 96 | "Simple Negation", 97 | "Indefinite Knowledge", 98 | "Basic Coreference", 99 | "Conjunction", 100 | "Compound Coreference", 101 | "Time Reasoning", 102 | "Basic Deduction", 103 | "Basic Induction", 104 | "Positional Reasoning", 105 | "Size Reasoning", 106 | "Path Finding", 107 | "Agent's Motivations" 108 | }; 109 | 110 | class Statement { 111 | public Statement(String line) { 112 | String[] parts = line.split("\t"); 113 | if (parts.length > 1) { 114 | String[] words = parts[0].replace("?", " ?").split(" "); 115 | lineNum = Integer.parseInt(words[0]); 116 | for (int i = 1; i < words.length; i++) { 117 | question.add(words[i].toLowerCase()); 118 | } 119 | 120 | answer = parts[1].toLowerCase(); 121 | String[] facts = parts[2].split(" "); 122 | for (int i = 0; i < facts.length; i++) { 123 | supportingFacts.add(Integer.parseInt(facts[i])); 124 | } 125 | isFact = false; 126 | } 127 | else { 128 | String[] words = line.replace("."," .").split(" "); 129 | lineNum = Integer.parseInt(words[0]); 130 | for (int i = 1; i < words.length; i++) { 131 | fact.add(words[i].toLowerCase()); 132 | } 133 | isFact = true; 134 | } 135 | } 136 | 137 | @Override 138 | public String toString() { 139 | String result = lineNum + ""; 140 | if (isFact) { 141 | for (String word : fact) { 142 | result += " " + word; 143 | } 144 | } 145 | else { 146 | for (String word : question) { 147 | result += " " + word; 148 | } 149 | result += " -> " + answer; 150 | for (Integer i : supportingFacts) { 151 | result += " " + i; 152 | } 153 | } 154 | return result; 155 | } 156 | boolean isFact = true; 157 | List fact = new ArrayList<>(); 158 | List question = new ArrayList<>(); 159 | String answer; 160 | List supportingFacts = new ArrayList<>(); 161 | int lineNum; 162 | } 163 | 164 | class Story { 165 | public Story(List statements, boolean onlySupportingFacts) { 166 | 167 | if (onlySupportingFacts == true) { 168 | Set supportingFactsAndQuestions = new HashSet<>(); 169 | for (Statement statement : statements) { 170 | if (statement.isFact == false) { 171 | supportingFactsAndQuestions.add(statement.lineNum); 172 | supportingFactsAndQuestions.addAll(statement.supportingFacts); 173 | } 174 | } 175 | List trimmed = new ArrayList<>(); 176 | for (Statement statement : statements) { 177 | if (supportingFactsAndQuestions.contains(statement.lineNum)) { 178 | trimmed.add(statement); 179 | } 180 | } 181 | this.statements = trimmed; 182 | } 183 | else { 184 | this.statements = statements; 185 | } 186 | } 187 | List statements; 188 | 189 | @Override 190 | public String toString() { 191 | String result = ""; 192 | for (Statement statement : statements) { 193 | result += statement.toString() + "\n"; 194 | } 195 | return result; 196 | } 197 | } 198 | 199 | List getStories(List fileNames, boolean onlySupportingFacts) throws Exception { 200 | List statements = new ArrayList<>(); 201 | for (String fileName : fileNames) { 202 | File file = new File(fileName); 203 | List lines = Files.readAllLines(file.toPath(), Charset.defaultCharset()); 204 | for (String line : lines) { 205 | statements.add(new Statement(line)); 206 | } 207 | } 208 | List stories = new ArrayList<>(); 209 | int prevNum = 0; 210 | List storyList = new ArrayList<>(); 211 | boolean containsQuestion = false; 212 | int errors = 0; 213 | for (Statement statement : statements) { 214 | if (statement.lineNum < prevNum) { 215 | if (containsQuestion == false) { 216 | //System.out.println("Incomplete story"); 217 | //for (Statement st : storyList) { 218 | // System.out.println(st); 219 | //} 220 | errors++; 221 | } 222 | else { 223 | Story story = new Story(storyList, onlySupportingFacts); 224 | //System.out.println(story); 225 | stories.add(story); 226 | } 227 | containsQuestion = false; 228 | storyList = new ArrayList<>(); 229 | } 230 | if (statement.isFact == false) { 231 | containsQuestion = true; 232 | } 233 | storyList.add(statement); 234 | prevNum = statement.lineNum; 235 | } 236 | Story story = new Story(storyList, onlySupportingFacts); 237 | //System.out.println(story); 238 | stories.add(story); 239 | if (errors > 0) { 240 | System.out.println("WARNING: " + errors + " INCORRECT STORIES REMOVED."); 241 | } 242 | return stories; 243 | } 244 | 245 | 246 | List inputVocab = new ArrayList<>(); 247 | List outputVocab = new ArrayList<>(); 248 | 249 | private void configureVocab(List storiesTrain, List storiesTest) { 250 | Set inputVocabSet = new HashSet<>(); 251 | Set outputVocabSet = new HashSet<>(); 252 | List allStories = new ArrayList<>(); 253 | allStories.addAll(storiesTrain); 254 | allStories.addAll(storiesTest); 255 | for (Story story : allStories) { 256 | for (Statement statement : story.statements) { 257 | if (statement.isFact) { 258 | for (String word : statement.fact) { 259 | inputVocabSet.add(word); 260 | } 261 | } 262 | else { 263 | for (String word : statement.question) { 264 | inputVocabSet.add(word); 265 | } 266 | outputVocabSet.add(statement.answer); 267 | } 268 | } 269 | } 270 | 271 | for (String word : inputVocabSet) { 272 | inputVocab.add(word); 273 | } 274 | for (String word : outputVocabSet) { 275 | outputVocab.add(word); 276 | } 277 | 278 | Collections.sort(inputVocab); 279 | Collections.sort(outputVocab); 280 | 281 | System.out.println("Possible answers: "); 282 | for (int i = 0; i < outputVocab.size(); i++) { 283 | System.out.println("\t["+i+"]: " + outputVocab.get(i)); 284 | } 285 | } 286 | 287 | private List getSequences(List stories) { 288 | int inputDimension = inputVocab.size(); 289 | int outputDimension = outputVocab.size(); 290 | List sequences = new ArrayList<>();; 291 | for (Story story : stories) { 292 | 293 | List steps = new ArrayList<>(); 294 | 295 | for (Statement statement : story.statements) { 296 | if (statement.isFact) { 297 | for (int w = 0; w < statement.fact.size(); w++) { 298 | double[] input = new double[inputDimension]; 299 | for (int i = 0; i < inputDimension; i++) { 300 | if (statement.fact.get(w).equals(inputVocab.get(i))) { 301 | input[i] = 1.0; 302 | break; 303 | } 304 | } 305 | steps.add(new DataStep(input, null)); 306 | } 307 | } 308 | else { 309 | for (int w = 0; w < statement.question.size(); w++) { 310 | double[] input = new double[inputDimension]; 311 | double[] targetOutput = null; 312 | for (int i = 0; i < inputDimension; i++) { 313 | if (statement.question.get(w).equals(inputVocab.get(i))) { 314 | input[i] = 1.0; 315 | break; 316 | } 317 | } 318 | steps.add(new DataStep(input, targetOutput)); 319 | } 320 | double[] input = new double[inputDimension]; 321 | double[] targetOutput = new double[outputDimension]; 322 | for (int i = 0; i < outputDimension; i++) { 323 | if (statement.answer.equals(outputVocab.get(i))) { 324 | targetOutput[i] = 1.0; 325 | break; 326 | } 327 | } 328 | steps.add(new DataStep(input, targetOutput)); 329 | } 330 | } 331 | sequences.add(new DataSequence(steps)); 332 | } 333 | return sequences; 334 | } 335 | 336 | @Override 337 | public void DisplayReport(Model model, Random rng) throws Exception { 338 | 339 | } 340 | 341 | @Override 342 | public Nonlinearity getModelOutputUnitToUse() { 343 | return new LinearUnit(); 344 | } 345 | } 346 | -------------------------------------------------------------------------------- /src/datastructs/DataSequence.java: -------------------------------------------------------------------------------- 1 | package datastructs; 2 | import java.io.Serializable; 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | 7 | public class DataSequence implements Serializable { 8 | 9 | private static final long serialVersionUID = 1L; 10 | public List steps = new ArrayList<>(); 11 | 12 | public DataSequence() { 13 | 14 | } 15 | 16 | public DataSequence(List steps) { 17 | this.steps = steps; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | String result = ""; 23 | result += "========================================================\n"; 24 | for (DataStep step : steps) { 25 | result += step.toString() + "\n"; 26 | } 27 | result += "========================================================\n"; 28 | return result; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/datastructs/DataSet.java: -------------------------------------------------------------------------------- 1 | package datastructs; 2 | import java.io.Serializable; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import loss.Loss; 7 | import model.Model; 8 | import model.Nonlinearity; 9 | 10 | public abstract class DataSet implements Serializable { 11 | public int inputDimension; 12 | public int outputDimension; 13 | public Loss lossTraining; 14 | public Loss lossReporting; 15 | public List training; 16 | public List validation; 17 | public List testing; 18 | public abstract void DisplayReport(Model model, Random rng) throws Exception; 19 | public abstract Nonlinearity getModelOutputUnitToUse(); 20 | } 21 | -------------------------------------------------------------------------------- /src/datastructs/DataStep.java: -------------------------------------------------------------------------------- 1 | package datastructs; 2 | import java.io.Serializable; 3 | 4 | import matrix.Matrix; 5 | 6 | 7 | public class DataStep implements Serializable { 8 | 9 | private static final long serialVersionUID = 1L; 10 | public Matrix input = null; 11 | public Matrix targetOutput = null; 12 | 13 | public DataStep() { 14 | 15 | } 16 | 17 | public DataStep(double[] input, double[] targetOutput) { 18 | this.input = new Matrix(input); 19 | if (targetOutput != null) { 20 | this.targetOutput = new Matrix(targetOutput); 21 | } 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | String result = ""; 27 | for (int i = 0; i < input.w.length; i++) { 28 | result += String.format("%.5f", input.w[i]) + "\t"; 29 | } 30 | result += "\t->\t"; 31 | if (targetOutput != null) { 32 | for (int i = 0; i < targetOutput.w.length; i++) { 33 | result += String.format("%.5f", targetOutput.w[i]) + "\t"; 34 | } 35 | } 36 | else { 37 | result += "___\t"; 38 | } 39 | return result; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/loss/Loss.java: -------------------------------------------------------------------------------- 1 | package loss; 2 | 3 | import java.io.Serializable; 4 | 5 | import matrix.Matrix; 6 | 7 | public interface Loss extends Serializable { 8 | void backward(Matrix actualOutput, Matrix targetOutput) throws Exception; 9 | double measure(Matrix actualOutput, Matrix targetOutput) throws Exception; 10 | } 11 | -------------------------------------------------------------------------------- /src/loss/LossArgMax.java: -------------------------------------------------------------------------------- 1 | package loss; 2 | 3 | import matrix.Matrix; 4 | 5 | public class LossArgMax implements Loss { 6 | 7 | /** 8 | * 9 | */ 10 | private static final long serialVersionUID = 1L; 11 | 12 | @Override 13 | public void backward(Matrix actualOutput, Matrix targetOutput) throws Exception { 14 | throw new Exception("not implemented"); 15 | 16 | } 17 | 18 | @Override 19 | public double measure(Matrix actualOutput, Matrix targetOutput) throws Exception { 20 | if (actualOutput.w.length != targetOutput.w.length) { 21 | throw new Exception("mismatch"); 22 | } 23 | double maxActual = Double.NEGATIVE_INFINITY; 24 | double maxTarget = Double.NEGATIVE_INFINITY; 25 | int indxMaxActual = -1; 26 | int indxMaxTarget = -1; 27 | for (int i = 0; i < actualOutput.w.length; i++) { 28 | if (actualOutput.w[i] > maxActual) { 29 | maxActual = actualOutput.w[i]; 30 | indxMaxActual = i; 31 | } 32 | if (targetOutput.w[i] > maxTarget) { 33 | maxTarget = targetOutput.w[i]; 34 | indxMaxTarget = i; 35 | } 36 | } 37 | if (indxMaxActual == indxMaxTarget) { 38 | return 0; 39 | } 40 | else { 41 | return 1; 42 | } 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/loss/LossMultiDimensionalBinary.java: -------------------------------------------------------------------------------- 1 | package loss; 2 | 3 | import matrix.Matrix; 4 | 5 | public class LossMultiDimensionalBinary implements Loss { 6 | 7 | /** 8 | * 9 | */ 10 | private static final long serialVersionUID = 1L; 11 | 12 | @Override 13 | public void backward(Matrix actualOutput, Matrix targetOutput) throws Exception { 14 | throw new Exception("not implemented"); 15 | } 16 | 17 | @Override 18 | public double measure(Matrix actualOutput, Matrix targetOutput) throws Exception { 19 | if (actualOutput.w.length != targetOutput.w.length) { 20 | throw new Exception("mismatch"); 21 | } 22 | 23 | for (int i = 0; i < targetOutput.w.length; i++) { 24 | if (targetOutput.w[i] >= 0.5 && actualOutput.w[i] < 0.5) { 25 | return 1; 26 | } 27 | if (targetOutput.w[i] < 0.5 && actualOutput.w[i] >= 0.5) { 28 | return 1; 29 | } 30 | } 31 | return 0; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/loss/LossSoftmax.java: -------------------------------------------------------------------------------- 1 | package loss; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | 5 | import autodiff.Graph; 6 | import matrix.Matrix; 7 | import model.Model; 8 | import datastructs.DataSequence; 9 | import datastructs.DataStep; 10 | import util.Util; 11 | 12 | 13 | public class LossSoftmax implements Loss { 14 | 15 | /** 16 | * 17 | */ 18 | private static final long serialVersionUID = 1L; 19 | 20 | @Override 21 | public void backward(Matrix logprobs, Matrix targetOutput) throws Exception { 22 | int targetIndex = getTargetIndex(targetOutput); 23 | Matrix probs = getSoftmaxProbs(logprobs, 1.0); 24 | for (int i = 0; i < probs.w.length; i++) { 25 | logprobs.dw[i] = probs.w[i]; 26 | } 27 | logprobs.dw[targetIndex] -= 1; 28 | } 29 | 30 | @Override 31 | public double measure(Matrix logprobs, Matrix targetOutput) throws Exception { 32 | int targetIndex = getTargetIndex(targetOutput); 33 | Matrix probs = getSoftmaxProbs(logprobs, 1.0); 34 | double cost = -Math.log(probs.w[targetIndex]); 35 | return cost; 36 | } 37 | 38 | public static double calculateMedianPerplexity(Model model, List sequences) throws Exception { 39 | double temperature = 1.0; 40 | List ppls = new ArrayList<>(); 41 | for (DataSequence seq : sequences) { 42 | double n = 0; 43 | double neglog2ppl = 0; 44 | 45 | Graph g = new Graph(false); 46 | model.resetState(); 47 | for (DataStep step : seq.steps) { 48 | Matrix logprobs = model.forward(step.input, g); 49 | Matrix probs = getSoftmaxProbs(logprobs, temperature); 50 | int targetIndex = getTargetIndex(step.targetOutput); 51 | double probOfCorrect = probs.w[targetIndex]; 52 | double log2prob = Math.log(probOfCorrect)/Math.log(2); //change-of-base 53 | neglog2ppl += -log2prob; 54 | n += 1; 55 | } 56 | 57 | n -= 1; //don't count first symbol of sentence 58 | double ppl = Math.pow(2, (neglog2ppl/(n-1))); 59 | ppls.add(ppl); 60 | } 61 | return Util.median(ppls); 62 | } 63 | 64 | public static Matrix getSoftmaxProbs(Matrix logprobs, double temperature) throws Exception { 65 | Matrix probs = new Matrix(logprobs.w.length); 66 | if (temperature != 1.0) { 67 | for (int i = 0; i < logprobs.w.length; i++) { 68 | logprobs.w[i] /= temperature; 69 | } 70 | } 71 | double maxval = Double.NEGATIVE_INFINITY; 72 | for (int i = 0; i < logprobs.w.length; i++) { 73 | if (logprobs.w[i] > maxval) { 74 | maxval = logprobs.w[i]; 75 | } 76 | } 77 | double sum = 0; 78 | for (int i = 0; i < logprobs.w.length; i++) { 79 | probs.w[i] = Math.exp(logprobs.w[i] - maxval); //all inputs to exp() are non-positive 80 | sum += probs.w[i]; 81 | } 82 | for (int i = 0; i < probs.w.length; i++) { 83 | probs.w[i] /= sum; 84 | } 85 | return probs; 86 | } 87 | 88 | private static int getTargetIndex(Matrix targetOutput) throws Exception { 89 | for (int i = 0; i < targetOutput.w.length; i++) { 90 | if (targetOutput.w[i] == 1.0) { 91 | return i; 92 | } 93 | } 94 | throw new Exception("no target index selected"); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/loss/LossSumOfSquares.java: -------------------------------------------------------------------------------- 1 | package loss; 2 | import matrix.Matrix; 3 | 4 | public class LossSumOfSquares implements Loss { 5 | 6 | /** 7 | * 8 | */ 9 | private static final long serialVersionUID = 1L; 10 | 11 | @Override 12 | public void backward(Matrix actualOutput, Matrix targetOutput) throws Exception { 13 | for (int i = 0; i < targetOutput.w.length; i++) { 14 | double errDelta = actualOutput.w[i] - targetOutput.w[i]; 15 | actualOutput.dw[i] += errDelta; 16 | } 17 | } 18 | 19 | @Override 20 | public double measure(Matrix actualOutput, Matrix targetOutput) throws Exception { 21 | double sum = 0; 22 | for (int i = 0; i < targetOutput.w.length; i++) { 23 | double errDelta = actualOutput.w[i] - targetOutput.w[i]; 24 | sum += 0.5 * errDelta * errDelta; 25 | } 26 | return sum; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/matrix/Matrix.java: -------------------------------------------------------------------------------- 1 | package matrix; 2 | import java.io.Serializable; 3 | import java.util.Random; 4 | 5 | 6 | public class Matrix implements Serializable { 7 | 8 | private static final long serialVersionUID = 1L; 9 | public int rows; 10 | public int cols; 11 | public double[] w; 12 | public double[] dw; 13 | public double[] stepCache; 14 | 15 | @Override 16 | public String toString() { 17 | String result = ""; 18 | for (int r = 0; r < rows; r++) { 19 | for (int c = 0; c < cols; c++) { 20 | result += String.format("%.4f",getW(r, c)) + "\t"; 21 | } 22 | result += "\n"; 23 | } 24 | return result; 25 | } 26 | 27 | public Matrix clone() { 28 | Matrix result = new Matrix(rows, cols); 29 | for (int i = 0; i < w.length; i++) { 30 | result.w[i] = w[i]; 31 | result.dw[i] = dw[i]; 32 | result.stepCache[i] = stepCache[i]; 33 | } 34 | return result; 35 | } 36 | 37 | public void resetDw() { 38 | for (int i = 0; i < dw.length; i++) { 39 | dw[i] = 0; 40 | } 41 | } 42 | 43 | public void resetStepCache() { 44 | for (int i = 0; i < stepCache.length; i++) { 45 | stepCache[i] = 0; 46 | } 47 | } 48 | 49 | public static Matrix transpose(Matrix m) { 50 | Matrix result = new Matrix(m.cols, m.rows); 51 | for (int r = 0; r < m.rows; r++) { 52 | for (int c = 0; c < m.cols; c++) { 53 | result.setW(c, r, m.getW(r, c)); 54 | } 55 | } 56 | return result; 57 | } 58 | 59 | public static Matrix rand(int rows, int cols, double initParamsStdDev, Random rng) { 60 | Matrix result = new Matrix(rows, cols); 61 | for (int i = 0; i < result.w.length; i++) { 62 | result.w[i] = rng.nextGaussian() * initParamsStdDev; 63 | } 64 | return result; 65 | } 66 | 67 | public static Matrix ident(int dim) { 68 | Matrix result = new Matrix(dim, dim); 69 | for (int i = 0; i < dim; i++) { 70 | result.setW(i, i, 1.0); 71 | } 72 | return result; 73 | } 74 | 75 | public static Matrix uniform(int rows, int cols, double s) { 76 | Matrix result = new Matrix(rows, cols); 77 | for (int i = 0; i < result.w.length; i++) { 78 | result.w[i] = s; 79 | } 80 | return result; 81 | } 82 | 83 | public static Matrix ones(int rows, int cols) { 84 | return uniform(rows, cols, 1.0); 85 | } 86 | 87 | public static Matrix negones(int rows, int cols) { 88 | return uniform(rows, cols, -1.0); 89 | } 90 | 91 | public Matrix(int dim) { 92 | this.rows = dim; 93 | this.cols = 1; 94 | this.w = new double[rows * cols]; 95 | this.dw = new double[rows * cols]; 96 | this.stepCache = new double[rows * cols]; 97 | } 98 | 99 | public Matrix(int rows, int cols) { 100 | this.rows = rows; 101 | this.cols = cols; 102 | this.w = new double[rows * cols]; 103 | this.dw = new double[rows * cols]; 104 | this.stepCache = new double[rows * cols]; 105 | } 106 | 107 | public Matrix(double[] vector) { 108 | this.rows = vector.length; 109 | this.cols = 1; 110 | this.w = vector; 111 | this.dw = new double[vector.length]; 112 | this.stepCache = new double[vector.length]; 113 | } 114 | 115 | private int index(int row, int col) { 116 | int ix = cols * row + col; 117 | return ix; 118 | } 119 | 120 | private double getW(int row, int col) { 121 | return w[index(row, col)]; 122 | } 123 | 124 | private void setW(int row, int col, double val) { 125 | w[index(row, col)] = val; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/model/FeedForwardLayer.java: -------------------------------------------------------------------------------- 1 | package model; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import matrix.Matrix; 7 | import autodiff.Graph; 8 | 9 | 10 | public class FeedForwardLayer implements Model { 11 | 12 | private static final long serialVersionUID = 1L; 13 | Matrix W; 14 | Matrix b; 15 | Nonlinearity f; 16 | 17 | public FeedForwardLayer(int inputDimension, int outputDimension, Nonlinearity f, double initParamsStdDev, Random rng) { 18 | W = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 19 | b = new Matrix(outputDimension); 20 | this.f = f; 21 | } 22 | 23 | @Override 24 | public Matrix forward(Matrix input, Graph g) throws Exception { 25 | Matrix sum = g.add(g.mul(W, input), b); 26 | Matrix out = g.nonlin(f, sum); 27 | return out; 28 | } 29 | 30 | @Override 31 | public void resetState() { 32 | 33 | } 34 | 35 | @Override 36 | public List getParameters() { 37 | List result = new ArrayList<>(); 38 | result.add(W); 39 | result.add(b); 40 | return result; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/model/GruLayer.java: -------------------------------------------------------------------------------- 1 | package model; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import matrix.Matrix; 7 | import autodiff.Graph; 8 | 9 | /* 10 | * As described in: 11 | * "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation" 12 | * http://arxiv.org/abs/1406.1078 13 | */ 14 | 15 | public class GruLayer implements Model { 16 | 17 | private static final long serialVersionUID = 1L; 18 | int inputDimension; 19 | int outputDimension; 20 | 21 | Matrix IHmix, HHmix, Bmix; 22 | Matrix IHnew, HHnew, Bnew; 23 | Matrix IHreset, HHreset, Breset; 24 | 25 | Matrix context; 26 | 27 | Nonlinearity fMix = new SigmoidUnit(); 28 | Nonlinearity fReset = new SigmoidUnit(); 29 | Nonlinearity fNew = new TanhUnit(); 30 | 31 | public GruLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) { 32 | this.inputDimension = inputDimension; 33 | this.outputDimension = outputDimension; 34 | IHmix = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 35 | HHmix = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng); 36 | Bmix = new Matrix(outputDimension); 37 | IHnew = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 38 | HHnew = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng); 39 | Bnew = new Matrix(outputDimension); 40 | IHreset = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 41 | HHreset = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng); 42 | Breset= new Matrix(outputDimension); 43 | } 44 | 45 | @Override 46 | public Matrix forward(Matrix input, Graph g) throws Exception { 47 | 48 | Matrix sum0 = g.mul(IHmix, input); 49 | Matrix sum1 = g.mul(HHmix, context); 50 | Matrix actMix = g.nonlin(fMix, g.add(g.add(sum0, sum1), Bmix)); 51 | 52 | Matrix sum2 = g.mul(IHreset, input); 53 | Matrix sum3 = g.mul(HHreset, context); 54 | Matrix actReset = g.nonlin(fReset, g.add(g.add(sum2, sum3), Breset)); 55 | 56 | Matrix sum4 = g.mul(IHnew, input); 57 | Matrix gatedContext = g.elmul(actReset, context); 58 | Matrix sum5 = g.mul(HHnew, gatedContext); 59 | Matrix actNewPlusGatedContext = g.nonlin(fNew, g.add(g.add(sum4, sum5), Bnew)); 60 | 61 | Matrix memvals = g.elmul(actMix, context); 62 | Matrix newvals = g.elmul(g.oneMinus(actMix), actNewPlusGatedContext); 63 | Matrix output = g.add(memvals, newvals); 64 | 65 | //rollover activations for next iteration 66 | context = output; 67 | 68 | return output; 69 | } 70 | 71 | @Override 72 | public void resetState() { 73 | context = new Matrix(outputDimension); 74 | } 75 | 76 | @Override 77 | public List getParameters() { 78 | List result = new ArrayList<>(); 79 | result.add(IHmix); 80 | result.add(HHmix); 81 | result.add(Bmix); 82 | result.add(IHnew); 83 | result.add(HHnew); 84 | result.add(Bnew); 85 | result.add(IHreset); 86 | result.add(HHreset); 87 | result.add(Breset); 88 | return result; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/model/LinearLayer.java: -------------------------------------------------------------------------------- 1 | package model; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import matrix.Matrix; 7 | import autodiff.Graph; 8 | 9 | 10 | public class LinearLayer implements Model { 11 | 12 | private static final long serialVersionUID = 1L; 13 | Matrix W; 14 | //no biases 15 | 16 | public LinearLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) { 17 | W = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 18 | } 19 | 20 | @Override 21 | public Matrix forward(Matrix input, Graph g) throws Exception { 22 | Matrix out = g.mul(W, input); 23 | return out; 24 | } 25 | 26 | @Override 27 | public void resetState() { 28 | 29 | } 30 | 31 | @Override 32 | public List getParameters() { 33 | List result = new ArrayList<>(); 34 | result.add(W); 35 | return result; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/model/LinearUnit.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | 4 | public class LinearUnit implements Nonlinearity { 5 | 6 | 7 | private static final long serialVersionUID = 1L; 8 | 9 | @Override 10 | public double forward(double x) { 11 | return x; 12 | } 13 | 14 | @Override 15 | public double backward(double x) { 16 | return 1.0; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/model/LstmLayer.java: -------------------------------------------------------------------------------- 1 | package model; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import matrix.Matrix; 7 | import autodiff.Graph; 8 | 9 | public class LstmLayer implements Model { 10 | 11 | private static final long serialVersionUID = 1L; 12 | int inputDimension; 13 | int outputDimension; 14 | 15 | Matrix Wix, Wih, bi; 16 | Matrix Wfx, Wfh, bf; 17 | Matrix Wox, Woh, bo; 18 | Matrix Wcx, Wch, bc; 19 | 20 | Matrix hiddenContext; 21 | Matrix cellContext; 22 | 23 | Nonlinearity fInputGate = new SigmoidUnit(); 24 | Nonlinearity fForgetGate = new SigmoidUnit(); 25 | Nonlinearity fOutputGate = new SigmoidUnit(); 26 | Nonlinearity fCellInput = new TanhUnit(); 27 | Nonlinearity fCellOutput = new TanhUnit(); 28 | 29 | public LstmLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) { 30 | this.inputDimension = inputDimension; 31 | this.outputDimension = outputDimension; 32 | Wix = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 33 | Wih = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng); 34 | bi = new Matrix(outputDimension); 35 | Wfx = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 36 | Wfh = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng); 37 | //set forget bias to 1.0, as described here: http://jmlr.org/proceedings/papers/v37/jozefowicz15.pdf 38 | bf = Matrix.ones(outputDimension, 1); 39 | Wox = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 40 | Woh = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng); 41 | bo = new Matrix(outputDimension); 42 | Wcx = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng); 43 | Wch = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng); 44 | bc = new Matrix(outputDimension); 45 | } 46 | 47 | @Override 48 | public Matrix forward(Matrix input, Graph g) throws Exception { 49 | 50 | //input gate 51 | Matrix sum0 = g.mul(Wix, input); 52 | Matrix sum1 = g.mul(Wih, hiddenContext); 53 | Matrix inputGate = g.nonlin(fInputGate, g.add(g.add(sum0, sum1), bi)); 54 | 55 | //forget gate 56 | Matrix sum2 = g.mul(Wfx, input); 57 | Matrix sum3 = g.mul(Wfh, hiddenContext); 58 | Matrix forgetGate = g.nonlin(fForgetGate, g.add(g.add(sum2, sum3), bf)); 59 | 60 | //output gate 61 | Matrix sum4 = g.mul(Wox, input); 62 | Matrix sum5 = g.mul(Woh, hiddenContext); 63 | Matrix outputGate = g.nonlin(fOutputGate, g.add(g.add(sum4, sum5), bo)); 64 | 65 | //write operation on cells 66 | Matrix sum6 = g.mul(Wcx, input); 67 | Matrix sum7 = g.mul(Wch, hiddenContext); 68 | Matrix cellInput = g.nonlin(fCellInput, g.add(g.add(sum6, sum7), bc)); 69 | 70 | //compute new cell activation 71 | Matrix retainCell = g.elmul(forgetGate, cellContext); 72 | Matrix writeCell = g.elmul(inputGate, cellInput); 73 | Matrix cellAct = g.add(retainCell, writeCell); 74 | 75 | //compute hidden state as gated, saturated cell activations 76 | Matrix output = g.elmul(outputGate, g.nonlin(fCellOutput, cellAct)); 77 | 78 | //rollover activations for next iteration 79 | hiddenContext = output; 80 | cellContext = cellAct; 81 | 82 | return output; 83 | } 84 | 85 | @Override 86 | public void resetState() { 87 | hiddenContext = new Matrix(outputDimension); 88 | cellContext = new Matrix(outputDimension); 89 | } 90 | 91 | @Override 92 | public List getParameters() { 93 | List result = new ArrayList<>(); 94 | result.add(Wix); 95 | result.add(Wih); 96 | result.add(bi); 97 | result.add(Wfx); 98 | result.add(Wfh); 99 | result.add(bf); 100 | result.add(Wox); 101 | result.add(Woh); 102 | result.add(bo); 103 | result.add(Wcx); 104 | result.add(Wch); 105 | result.add(bc); 106 | return result; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/model/Model.java: -------------------------------------------------------------------------------- 1 | package model; 2 | import java.io.Serializable; 3 | import java.util.List; 4 | 5 | import matrix.Matrix; 6 | import autodiff.Graph; 7 | 8 | 9 | public interface Model extends Serializable { 10 | Matrix forward(Matrix input, Graph g) throws Exception; 11 | void resetState(); 12 | List getParameters(); 13 | } 14 | -------------------------------------------------------------------------------- /src/model/NeuralNetwork.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import matrix.Matrix; 7 | import autodiff.Graph; 8 | 9 | public class NeuralNetwork implements Model { 10 | 11 | private static final long serialVersionUID = 1L; 12 | List layers = new ArrayList<>(); 13 | 14 | public NeuralNetwork(List layers) { 15 | this.layers = layers; 16 | } 17 | 18 | @Override 19 | public Matrix forward(Matrix input, Graph g) throws Exception { 20 | Matrix prev = input; 21 | for (Model layer : layers) { 22 | prev = layer.forward(prev, g); 23 | } 24 | return prev; 25 | } 26 | 27 | @Override 28 | public void resetState() { 29 | for (Model layer : layers) { 30 | layer.resetState(); 31 | } 32 | } 33 | 34 | @Override 35 | public List getParameters() { 36 | List result = new ArrayList<>(); 37 | for (Model layer : layers) { 38 | result.addAll(layer.getParameters()); 39 | } 40 | return result; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/model/Nonlinearity.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | import java.io.Serializable; 4 | 5 | 6 | public interface Nonlinearity extends Serializable { 7 | double forward(double x); 8 | double backward(double x); 9 | } 10 | -------------------------------------------------------------------------------- /src/model/RectifiedLinearUnit.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | 4 | public class RectifiedLinearUnit implements Nonlinearity { 5 | 6 | private static final long serialVersionUID = 1L; 7 | private double slope; 8 | 9 | public RectifiedLinearUnit() { 10 | this.slope = 0; 11 | } 12 | 13 | public RectifiedLinearUnit(double slope) { 14 | this.slope = slope; 15 | } 16 | 17 | @Override 18 | public double forward(double x) { 19 | if (x >= 0) { 20 | return x; 21 | } 22 | else { 23 | return x * slope; 24 | } 25 | } 26 | 27 | @Override 28 | public double backward(double x) { 29 | if (x >= 0) { 30 | return 1.0; 31 | } 32 | else { 33 | return slope; 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/model/RnnLayer.java: -------------------------------------------------------------------------------- 1 | package model; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import matrix.Matrix; 7 | import autodiff.Graph; 8 | 9 | 10 | public class RnnLayer implements Model { 11 | 12 | private static final long serialVersionUID = 1L; 13 | int inputDimension; 14 | int outputDimension; 15 | 16 | Matrix W, b; 17 | 18 | Matrix context; 19 | 20 | Nonlinearity f; 21 | 22 | public RnnLayer(int inputDimension, int outputDimension, Nonlinearity hiddenUnit, double initParamsStdDev, Random rng) { 23 | this.inputDimension = inputDimension; 24 | this.outputDimension = outputDimension; 25 | this.f = hiddenUnit; 26 | W = Matrix.rand(outputDimension, inputDimension+outputDimension, initParamsStdDev, rng); 27 | b = new Matrix(outputDimension); 28 | } 29 | 30 | @Override 31 | public Matrix forward(Matrix input, Graph g) throws Exception { 32 | 33 | Matrix concat = g.concatVectors(input, context); 34 | 35 | Matrix sum = g.mul(W, concat); 36 | sum = g.add(sum, b); 37 | Matrix output = g.nonlin(f, sum); 38 | 39 | //rollover activations for next iteration 40 | context = output; 41 | 42 | return output; 43 | } 44 | 45 | @Override 46 | public void resetState() { 47 | context = new Matrix(outputDimension); 48 | } 49 | 50 | @Override 51 | public List getParameters() { 52 | List result = new ArrayList<>(); 53 | result.add(W); 54 | result.add(b); 55 | return result; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/model/SigmoidUnit.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | 4 | 5 | public class SigmoidUnit implements Nonlinearity { 6 | 7 | private static final long serialVersionUID = 1L; 8 | 9 | @Override 10 | public double forward(double x) { 11 | return 1 / (1 + Math.exp(-x)); 12 | } 13 | 14 | @Override 15 | public double backward(double x) { 16 | double act = forward(x); 17 | return act * (1 - act); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/model/SineUnit.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | 4 | 5 | public class SineUnit implements Nonlinearity { 6 | 7 | private static final long serialVersionUID = 1L; 8 | 9 | @Override 10 | public double forward(double x) { 11 | return Math.sin(x); 12 | } 13 | 14 | @Override 15 | public double backward(double x) { 16 | return Math.cos(x); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/model/TanhUnit.java: -------------------------------------------------------------------------------- 1 | package model; 2 | 3 | 4 | 5 | public class TanhUnit implements Nonlinearity { 6 | 7 | private static final long serialVersionUID = 1L; 8 | 9 | @Override 10 | public double forward(double x) { 11 | return Math.tanh(x); 12 | } 13 | 14 | @Override 15 | public double backward(double x) { 16 | double coshx = Math.cosh(x); 17 | double denom = (Math.cosh(2*x) + 1); 18 | return 4 * coshx * coshx / (denom * denom); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/trainer/Trainer.java: -------------------------------------------------------------------------------- 1 | package trainer; 2 | import java.util.ArrayList; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import util.FileIO; 7 | import autodiff.Graph; 8 | import datastructs.DataSequence; 9 | import datastructs.DataSet; 10 | import datastructs.DataStep; 11 | import loss.Loss; 12 | import matrix.Matrix; 13 | import model.Model; 14 | 15 | public class Trainer { 16 | 17 | public static double decayRate = 0.999; 18 | public static double smoothEpsilon = 1e-8; 19 | public static double gradientClipValue = 5; 20 | public static double regularization = 0.000001; // L2 regularization strength 21 | 22 | public static double train(int trainingEpochs, double learningRate, Model model, DataSet data, int reportEveryNthEpoch, Random rng) throws Exception { 23 | return train(trainingEpochs, learningRate, model, data, reportEveryNthEpoch, false, false, null, rng); 24 | } 25 | 26 | public static double train(int trainingEpochs, double learningRate, Model model, DataSet data, int reportEveryNthEpoch, boolean initFromSaved, boolean overwriteSaved, String savePath, Random rng) throws Exception { 27 | System.out.println("--------------------------------------------------------------"); 28 | if (initFromSaved) { 29 | System.out.println("initializing model from saved state..."); 30 | try { 31 | model = (Model)FileIO.deserialize(savePath); 32 | data.DisplayReport(model, rng); 33 | } 34 | catch (Exception e) { 35 | System.out.println("Oops. Unable to load from a saved state."); 36 | System.out.println("WARNING: " + e.getMessage()); 37 | System.out.println("Continuing from freshly initialized model instead."); 38 | } 39 | } 40 | double result = 1.0; 41 | for (int epoch = 0; epoch < trainingEpochs; epoch++) { 42 | 43 | String show = "epoch["+(epoch+1)+"/"+trainingEpochs+"]"; 44 | 45 | double reportedLossTrain = pass(learningRate, model, data.training, true, data.lossTraining, data.lossReporting); 46 | result = reportedLossTrain; 47 | if (Double.isNaN(reportedLossTrain) || Double.isInfinite(reportedLossTrain)) { 48 | throw new Exception("WARNING: invalid value for training loss. Try lowering learning rate."); 49 | } 50 | double reportedLossValidation = 0; 51 | double reportedLossTesting = 0; 52 | if (data.validation != null) { 53 | reportedLossValidation = pass(learningRate, model, data.validation, false, data.lossTraining, data.lossReporting); 54 | result = reportedLossValidation; 55 | } 56 | if (data.testing != null) { 57 | reportedLossTesting = pass(learningRate, model, data.testing, false, data.lossTraining, data.lossReporting); 58 | result = reportedLossTesting; 59 | } 60 | show += "\ttrain loss = "+String.format("%.5f", reportedLossTrain); 61 | if (data.validation != null) { 62 | show += "\tvalid loss = "+String.format("%.5f", reportedLossValidation); 63 | } 64 | if (data.testing != null) { 65 | show += "\ttest loss = "+String.format("%.5f", reportedLossTesting); 66 | } 67 | System.out.println(show); 68 | 69 | if (epoch % reportEveryNthEpoch == reportEveryNthEpoch - 1) { 70 | data.DisplayReport(model, rng); 71 | } 72 | 73 | if (overwriteSaved) { 74 | FileIO.serialize(savePath, model); 75 | } 76 | 77 | if (reportedLossTrain == 0 && reportedLossValidation == 0) { 78 | System.out.println("--------------------------------------------------------------"); 79 | System.out.println("\nDONE."); 80 | break; 81 | } 82 | } 83 | return result; 84 | } 85 | 86 | public static double pass(double learningRate, Model model, List sequences, boolean applyTraining, Loss lossTraining, Loss lossReporting) throws Exception { 87 | 88 | double numerLoss = 0; 89 | double denomLoss = 0; 90 | 91 | for (DataSequence seq : sequences) { 92 | model.resetState(); 93 | Graph g = new Graph(applyTraining); 94 | for (DataStep step : seq.steps) { 95 | Matrix output = model.forward(step.input, g); 96 | if (step.targetOutput != null) { 97 | double loss = lossReporting.measure(output, step.targetOutput); 98 | if (Double.isNaN(loss) || Double.isInfinite(loss)) { 99 | return loss; 100 | } 101 | numerLoss += loss; 102 | denomLoss++; 103 | if (applyTraining) { 104 | lossTraining.backward(output, step.targetOutput); 105 | } 106 | } 107 | } 108 | List thisSequence = new ArrayList<>(); 109 | thisSequence.add(seq); 110 | if (applyTraining) { 111 | g.backward(); //backprop dw values 112 | updateModelParams(model, learningRate); //update params 113 | } 114 | } 115 | return numerLoss/denomLoss; 116 | } 117 | 118 | public static void updateModelParams(Model model, double stepSize) throws Exception { 119 | for (Matrix m : model.getParameters()) { 120 | for (int i = 0; i < m.w.length; i++) { 121 | 122 | // rmsprop adaptive learning rate 123 | double mdwi = m.dw[i]; 124 | m.stepCache[i] = m.stepCache[i] * decayRate + (1 - decayRate) * mdwi * mdwi; 125 | 126 | // gradient clip 127 | if (mdwi > gradientClipValue) { 128 | mdwi = gradientClipValue; 129 | } 130 | if (mdwi < -gradientClipValue) { 131 | mdwi = -gradientClipValue; 132 | } 133 | 134 | // update (and regularize) 135 | m.w[i] += - stepSize * mdwi / Math.sqrt(m.stepCache[i] + smoothEpsilon) - regularization * m.w[i]; 136 | m.dw[i] = 0; 137 | } 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/util/FileIO.java: -------------------------------------------------------------------------------- 1 | package util; 2 | 3 | 4 | import java.io.FileInputStream; 5 | import java.io.FileOutputStream; 6 | import java.io.ObjectInputStream; 7 | import java.io.ObjectOutputStream; 8 | 9 | public class FileIO { 10 | 11 | public static void serialize(String path, Object obj) throws Exception { 12 | FileOutputStream fileOut = new FileOutputStream(path); 13 | ObjectOutputStream out = new ObjectOutputStream(fileOut); 14 | out.writeObject(obj); 15 | out.close(); 16 | fileOut.close(); 17 | } 18 | 19 | public static Object deserialize(String path) throws Exception { 20 | FileInputStream fileIn = new FileInputStream(path); 21 | ObjectInputStream in = new ObjectInputStream(fileIn); 22 | Object result = in.readObject(); 23 | in.close(); 24 | fileIn.close(); 25 | return result; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/util/NeuralNetworkHelper.java: -------------------------------------------------------------------------------- 1 | package util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Random; 6 | 7 | import model.FeedForwardLayer; 8 | import model.GruLayer; 9 | import model.LinearLayer; 10 | import model.LstmLayer; 11 | import model.Model; 12 | import model.NeuralNetwork; 13 | import model.Nonlinearity; 14 | import model.RnnLayer; 15 | 16 | public class NeuralNetworkHelper { 17 | 18 | public static NeuralNetwork makeLstm(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, Nonlinearity decoderUnit, double initParamsStdDev, Random rng) { 19 | List layers = new ArrayList<>(); 20 | for (int h = 0; h < hiddenLayers; h++) { 21 | if (h == 0) { 22 | layers.add(new LstmLayer(inputDimension, hiddenDimension, initParamsStdDev, rng)); 23 | } 24 | else { 25 | layers.add(new LstmLayer(hiddenDimension, hiddenDimension, initParamsStdDev, rng)); 26 | } 27 | } 28 | layers.add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 29 | return new NeuralNetwork(layers); 30 | } 31 | 32 | public static NeuralNetwork makeLstmWithInputBottleneck(int inputDimension, int bottleneckDimension, int hiddenDimension, int hiddenLayers, int outputDimension, Nonlinearity decoderUnit, double initParamsStdDev, Random rng) { 33 | List layers = new ArrayList<>(); 34 | layers.add(new LinearLayer(inputDimension, bottleneckDimension, initParamsStdDev, rng)); 35 | for (int h = 0; h < hiddenLayers; h++) { 36 | if (h == 0) { 37 | layers.add(new LstmLayer(bottleneckDimension, hiddenDimension, initParamsStdDev, rng)); 38 | } 39 | else { 40 | layers.add(new LstmLayer(hiddenDimension, hiddenDimension, initParamsStdDev, rng)); 41 | } 42 | } 43 | layers.add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 44 | return new NeuralNetwork(layers); 45 | } 46 | 47 | public static NeuralNetwork makeFeedForward(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, Nonlinearity hiddenUnit, Nonlinearity decoderUnit, double initParamsStdDev, Random rng) { 48 | List layers = new ArrayList<>(); 49 | if (hiddenLayers == 0) { 50 | layers.add(new FeedForwardLayer(inputDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 51 | return new NeuralNetwork(layers); 52 | } 53 | else { 54 | for (int h = 0; h < hiddenLayers; h++) { 55 | if (h == 0) { 56 | layers.add(new FeedForwardLayer(inputDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 57 | } 58 | else { 59 | layers.add(new FeedForwardLayer(hiddenDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 60 | } 61 | } 62 | layers.add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 63 | return new NeuralNetwork(layers); 64 | } 65 | } 66 | 67 | public static NeuralNetwork makeGru(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, Nonlinearity decoderUnit, double initParamsStdDev, Random rng) { 68 | List layers = new ArrayList<>(); 69 | for (int h = 0; h < hiddenLayers; h++) { 70 | if (h == 0) { 71 | layers.add(new GruLayer(inputDimension, hiddenDimension, initParamsStdDev, rng)); 72 | } 73 | else { 74 | layers.add(new GruLayer(hiddenDimension, hiddenDimension, initParamsStdDev, rng)); 75 | } 76 | } 77 | layers.add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 78 | return new NeuralNetwork(layers); 79 | } 80 | 81 | public static NeuralNetwork makeRnn(int inputDimension, int hiddenDimension, int hiddenLayers, int outputDimension, Nonlinearity hiddenUnit, Nonlinearity decoderUnit, double initParamsStdDev, Random rng) { 82 | List layers = new ArrayList<>(); 83 | for (int h = 0; h < hiddenLayers; h++) { 84 | if (h == 0) { 85 | layers.add(new RnnLayer(inputDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 86 | } 87 | else { 88 | layers.add(new RnnLayer(hiddenDimension, hiddenDimension, hiddenUnit, initParamsStdDev, rng)); 89 | } 90 | } 91 | layers.add(new FeedForwardLayer(hiddenDimension, outputDimension, decoderUnit, initParamsStdDev, rng)); 92 | return new NeuralNetwork(layers); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/util/Util.java: -------------------------------------------------------------------------------- 1 | package util; 2 | import java.util.Collections; 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import matrix.Matrix; 7 | 8 | public class Util { 9 | 10 | public static int pickIndexFromRandomVector(Matrix probs, Random r) throws Exception { 11 | double mass = 1.0; 12 | for (int i = 0; i < probs.w.length; i++) { 13 | double prob = probs.w[i] / mass; 14 | if (r.nextDouble() < prob) { 15 | return i; 16 | } 17 | mass -= probs.w[i]; 18 | } 19 | throw new Exception("no target index selected"); 20 | } 21 | 22 | public static double median(List vals) { 23 | Collections.sort(vals); 24 | int mid = vals.size()/2; 25 | if (vals.size() % 2 == 1) { 26 | return vals.get(mid); 27 | } 28 | else { 29 | return (vals.get(mid-1) + vals.get(mid)) / 2; 30 | } 31 | } 32 | 33 | public static String timeString(double milliseconds) { 34 | String result = ""; 35 | 36 | int m = (int) milliseconds; 37 | 38 | int hours = 0; 39 | while (m >= 1000*60*60) { 40 | m -= 1000*60*60; 41 | hours++; 42 | } 43 | int minutes = 0; 44 | while (m >= 1000*60) { 45 | m -= 1000*60; 46 | minutes++; 47 | } 48 | if (hours > 0) { 49 | result += hours + " hours, "; 50 | } 51 | int seconds = 0; 52 | while (m >= 1000) { 53 | m -= 1000; 54 | seconds ++; 55 | } 56 | result += minutes + " minutes and "; 57 | result += seconds + " seconds."; 58 | return result; 59 | } 60 | } 61 | --------------------------------------------------------------------------------