├── .gitignore ├── LICENSE ├── README.md ├── _config.yml ├── config ├── DCD.config ├── DEMIParallelDCD.config ├── ParallelDCD.config ├── StructuredPerceptron.config ├── kushman │ ├── config1.json │ ├── config2.json │ ├── config3.json │ ├── config4.json │ ├── config5.json │ ├── config6.json │ ├── config7.json │ ├── config8.json │ └── config9.json ├── lemma.config ├── ner-conll.config ├── pipeline.basic.config └── pipeline.config ├── data ├── aggregate │ ├── fold0.txt │ ├── fold1.txt │ ├── fold2.txt │ ├── fold3.txt │ ├── fold4.txt │ └── info.txt ├── aggregateLex │ ├── fold0.txt │ ├── fold1.txt │ ├── fold2.txt │ ├── fold3.txt │ ├── fold4.txt │ └── info.txt ├── aggregateTmpl │ ├── fold0.txt │ ├── fold1.txt │ ├── fold2.txt │ ├── fold3.txt │ ├── fold4.txt │ └── info.txt ├── allArith │ ├── fold0.txt │ ├── fold1.txt │ ├── fold2.txt │ ├── fold3.txt │ ├── fold4.txt │ └── info.txt ├── allArithLex │ ├── fold0.txt │ ├── fold1.txt │ ├── fold2.txt │ ├── fold3.txt │ ├── fold4.txt │ └── info.txt ├── allArithTmpl │ ├── fold0.txt │ ├── fold1.txt │ ├── fold2.txt │ ├── fold3.txt │ ├── fold4.txt │ └── info.txt ├── glove.6B.300d.verbs.txt ├── handcrafted │ ├── fold.txt │ └── questions.json ├── patterns.txt ├── perturb │ ├── new.txt │ └── old.txt ├── questions.json └── simple_interest │ ├── fold0.txt │ ├── fold1.txt │ ├── fold2.txt │ ├── fold3.txt │ ├── fold4.txt │ ├── questions.json │ ├── si.json │ └── si_kushman.json ├── pom.xml ├── run.sh ├── runAllExpts1.sh ├── runAllExpts2.sh ├── runAllExpts3.sh ├── scripts └── kushman │ ├── runSubhro.sh │ ├── runSubhroAgg.sh │ ├── runSubhroAllArith.sh │ └── runSubhroHandcraft.sh └── src └── main └── java ├── constraints ├── ConsDriver.java ├── ConsInfSolver.java ├── Constraints.java ├── GraphDriver.java └── GraphInfSolver.java ├── coref ├── CorefDriver.java ├── CorefFeatGen.java ├── CorefInfSolver.java ├── CorefX.java └── CorefY.java ├── demo ├── Demo.java ├── Derivation.java ├── Driver.java └── Server.java ├── graph ├── GraphDriver.java ├── GraphFeatGen.java ├── GraphInfSolver.java ├── GraphX.java └── GraphY.java ├── logic ├── Driver.java ├── Logic.java ├── LogicDriver.java ├── LogicFeatGen.java ├── LogicInfSolver.java ├── LogicX.java ├── LogicY.java ├── Relevance.java └── Verbs.java ├── pair ├── PairDriver.java ├── PairFeatGen.java ├── PairInfSolver.java ├── PairX.java └── PairY.java ├── rate ├── RateDriver.java ├── RateFeatGen.java ├── RateInfSolver.java ├── RateX.java └── RateY.java ├── reader ├── CrowdFlower.java ├── Dataset.java └── Reader.java ├── relevance ├── RelDriver.java ├── RelFeatGen.java ├── RelInfSolver.java ├── RelX.java └── RelY.java ├── run ├── Annotations.java ├── RunDriver.java ├── RunFeatGen.java ├── RunInfSolver.java ├── RunX.java └── RunY.java ├── structure ├── DataFormat.java ├── Node.java ├── PairComparator.java ├── Problem.java ├── QuantSpan.java ├── QuantitySchema.java ├── Schema.java ├── SimpleQuantifier.java ├── StanfordProblem.java └── StanfordSchema.java └── utils ├── FeatGen.java ├── Folds.java ├── Params.java └── Tools.java /.gitignore: -------------------------------------------------------------------------------- 1 | target/* 2 | .* 3 | log/* 4 | log_* 5 | models* 6 | data/vector* 7 | data/train* 8 | data/test* 9 | /target 10 | *~ 11 | models/* 12 | *cache* 13 | dependencyOld/* 14 | src/main/resources/* 15 | data_nn/* 16 | *.pyc 17 | *.iml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 CogComp 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Arithmetic Word Problem Solver 2 | 3 | This repository contains the source code and data for automatic math 4 | word problem solving. It contains the implementation of 3 distinct 5 | arithmetic word problem solvers. This has been developed by Subhro Roy 6 | at the Cognitive Computation Group at the University of Illinois, 7 | Urbana Champaign. 8 | 9 | ### Publication 10 | 11 | This repository has been used in the following publications. Please 12 | cite them if you use the code or data. 13 | 14 | ~~~~ 15 | Subhro Roy and Dan Roth. 16 | Mapping to Declarative Knowledge for Word Problem Solving. 17 | TACL 2018. 18 | ~~~~ 19 | ~~~~ 20 | Subhro Roy and Dan Roth. 21 | Unit Dependency Graph and its Application to Arithmetic Word Problem Solving. 22 | AAAI 2017. 23 | ~~~~ 24 | ~~~~ 25 | Subhro Roy and Dan Roth. 26 | Solving General Arithmetic Word Problems. 27 | EMNLP 2015. 28 | ~~~~ 29 | 30 | 31 | 32 | ### Data 33 | 34 | Data can be found in the file data/questions.json. This file 35 | constitutes our largest dataset Aggregate. All other released 36 | datasets, namely allArith, allArithLex, allArithTmpl, aggregateLex, 37 | and aggregateTmpl, are subsets of Aggregate. 38 | 39 | The data file has a json format. Each datapoint is represented as 40 | ~~~~ 41 | [ 42 | { 43 | "iIndex": 1232857298, 44 | "sQuestion": "A large bag of balls was kept under Haley’s bed. Her mom placed the balls in bags for children in foster homes. If every bag can contain 4.0 balls, and Haley has 36.0 bags, how many balls total will be donated?", 45 | "quants": [ 46 | 4.0, 47 | 36.0 48 | ], 49 | "lAlignments": [ 50 | 1, 51 | 0 52 | ], 53 | "lEquations": [ 54 | "X=(36.0 * 4.0)" 55 | ], 56 | "lSolutions": [ 57 | 144.0 58 | ], 59 | "rates": [0] 60 | }, 61 | ... 62 | ] 63 | ~~~~ 64 | 65 | "iIndex" is a unique integer identifier, "sQuestion" is the problem 66 | text, quants is the ordered list of numbers detected from the text, 67 | "lAlignments" refer to the alignment of numbers from the text to the 68 | equation (if i-th position has j, then the i-th number in the equation 69 | is mapped to the j-th number from the quants), "lEquations" is the 70 | list of equations, and lSolutions is a list of solutions. "rates" 71 | indicate the indices of quantities which represent a rate 72 | relationship. If the question represents a rate, "rate" will have -1. 73 | 74 | The data/ directory has several folders, each representing a dataset. 75 | A dataset directory usually has a few fold files, each fold file 76 | contains indices of the problems belonging to the fold. The dataset 77 | comprises the union of all the folds present in the folder. 78 | 79 | 80 | ### Instructions to run the code 81 | 82 | You will need to have maven installed in your system. To download the 83 | dependencies, run 84 | 85 | mvn dependency:copy-dependencies 86 | 87 | Next compile using : 88 | 89 | mvn compile 90 | 91 | The main interface to the code is the script run.sh. It has the 92 | following arguments: 93 | 94 | * --data : Data file containing all the questions (uses data/questions.json if not provided) 95 | * --mode < mode > : Mode takes one of the following options 96 | * Rel : Trains and tests the relevance classifier. Output file log/Rel.out 97 | * Pair : Trains and tests the LCA classifier. Output file log/Pair.out 98 | * Vertex : Trains and tests the vertex label classifier. Output file log/Vertex.out 99 | * Edge : Trains and tests the edge label classifier. Output file log/Edge.out 100 | * GraphDecompose : Trains and tests the decomposed UDG prediction model. Output file log/GraphDecompose.out 101 | * GraphJoint : Trains and tests the joint UDG prediction model. Output file log/GraphJoint.out 102 | * LCA : Trains and tests the LCA++ solver (EMNLP 2015). Output file log/LCA.out 103 | * UnitDep : Trains and tests the UnitDep solver (AAAI 2017). Output file log/UnitDep.out 104 | * Coref : Trains and tests the coreference model or the rule selection model. Output file log/Coref.out 105 | * Logic : Trains and tests the logic model with gold rule selection and relevance. Output file log/Logic.out 106 | * E2ELogic : Trains and tests the end to end logic system. Output file log/E2ELogic.out 107 | * --train ... : List of fold files for training. 108 | * --test ... : List of fold files for testing. 109 | * --cv ... : List of fold files for n-fold cross validation. Will not 110 | work when train and test are provided. 111 | * --model_dir < dir > : Location to save model files (uses models/ if not provided). 112 | * --print_mistakes : Option to print test questions the system got wrong. 113 | * --print_correct : Option to print test questions the system got correct. 114 | * --demo : Option to run terminal based demo of the system. 115 | * --demo_server : Starts the web demo, only for Cogcomp users. 116 | 117 | Have a look at some of the scripts under scripts/ for examples of 118 | commands used to generates results in the papers. 119 | 120 | ### Other issues 121 | 122 | Please send any suggestions, comments, issues to subhro@csail.mit.edu . 123 | 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /config/DCD.config: -------------------------------------------------------------------------------- 1 | # {L2LossSSVM, StructuredPerceptron} 2 | LEARNING_MODEL = L2LossSSVM 3 | 4 | # {DCDSolver, ParallelDCDSolver, DEMIParallelDCDSolver}; 5 | L2_LOSS_SSVM_SOLVER_TYPE = DCDSolver 6 | 7 | NUMBER_OF_THREADS = 16 8 | C_FOR_STRUCTURE = 1.0 9 | TRAINMINI = false 10 | TRAINMINI_SIZE = 1000 11 | STOP_CONDITION = 0.1 12 | CHECK_INFERENCE_OPT = false 13 | MAX_NUM_ITER = 5 14 | PROGRESS_REPORT_ITER = 1 15 | INNER_STOP_CONDITION = 0.1 16 | MAX_ITER_INNER = 250 17 | MAX_ITER_INNER_FINAL = 2500 18 | TOTAL_NUMBER_FEATURE = -1 19 | CLEAN_CACHE = true 20 | CLEAN_CACHE_ITER = 5 21 | DEMIDCD_NUMBER_OF_UPDATES_BEFORE_UPDATE_BUFFER = 100 22 | DEMIDCD_NUMBER_OF_INF_PARSE_BEFORE_UPDATE_WV = 10 23 | LEARNING_RATE = 0.01 24 | DECAY_LEARNING_RATE = false 25 | -------------------------------------------------------------------------------- /config/DEMIParallelDCD.config: -------------------------------------------------------------------------------- 1 | # {L2LossSSVM, StructuredPerceptron} 2 | LEARNING_MODEL = L2LossSSVM 3 | 4 | # {DCDSolver, ParallelDCDSolver, DEMIParallelDCDSolver}; 5 | L2_LOSS_SSVM_SOLVER_TYPE = DEMIParallelDCDSolver 6 | 7 | MAX_NUM_ITER = 1000 8 | STOP_CONDITION = 0.1 9 | 10 | NUMBER_OF_THREADS = 16 11 | C_FOR_STRUCTURE = 1.0 12 | TRAINMINI = false 13 | TRAINMINI_SIZE = 1000 14 | CHECK_INFERENCE_OPT = true 15 | PROGRESS_REPORT_ITER = 10 16 | INNER_STOP_CONDITION = 0.1 17 | MAX_ITER_INNER = 250 18 | MAX_ITER_INNER_FINAL = 2500 19 | TOTAL_NUMBER_FEATURE = -1 20 | CLEAN_CACHE = true 21 | CLEAN_CACHE_ITER = 5 22 | DEMIDCD_NUMBER_OF_UPDATES_BEFORE_UPDATE_BUFFER = 100 23 | DEMIDCD_NUMBER_OF_INF_PARSE_BEFORE_UPDATE_WV = 10 24 | LEARNING_RATE = 0.01 25 | DECAY_LEARNING_RATE = false 26 | -------------------------------------------------------------------------------- /config/ParallelDCD.config: -------------------------------------------------------------------------------- 1 | # {L2LossSSVM, StructuredPerceptron} 2 | LEARNING_MODEL = L2LossSSVM 3 | 4 | # {DCDSolver, ParallelDCDSolver, DEMIParallelDCDSolver}; 5 | L2_LOSS_SSVM_SOLVER_TYPE = ParallelDCDSolver 6 | 7 | NUMBER_OF_THREADS = 16 8 | C_FOR_STRUCTURE = 1.0 9 | TRAINMINI = false 10 | TRAINMINI_SIZE = 1000 11 | STOP_CONDITION = 0.1 12 | CHECK_INFERENCE_OPT = true 13 | MAX_NUM_ITER = 250 14 | PROGRESS_REPORT_ITER = 10 15 | INNER_STOP_CONDITION = 0.1 16 | MAX_ITER_INNER = 250 17 | MAX_ITER_INNER_FINAL = 2500 18 | TOTAL_NUMBER_FEATURE = -1 19 | CLEAN_CACHE = true 20 | CLEAN_CACHE_ITER = 5 21 | DEMIDCD_NUMBER_OF_UPDATES_BEFORE_UPDATE_BUFFER = 100 22 | DEMIDCD_NUMBER_OF_INF_PARSE_BEFORE_UPDATE_WV = 10 23 | LEARNING_RATE = 0.01 24 | DECAY_LEARNING_RATE = false 25 | -------------------------------------------------------------------------------- /config/StructuredPerceptron.config: -------------------------------------------------------------------------------- 1 | # {L2LossSSVM, StructuredPerceptron} 2 | LEARNING_MODEL = StructuredPerceptron 3 | 4 | # {DCDSolver, ParallelDCDSolver, DEMIParallelDCDSolver}; 5 | L2_LOSS_SSVM_SOLVER_TYPE = DCDSolver 6 | 7 | NUMBER_OF_THREADS = 16 8 | C_FOR_STRUCTURE = 1.0 9 | TRAINMINI = false 10 | TRAINMINI_SIZE = 1000 11 | STOP_CONDITION = 0.1 12 | CHECK_INFERENCE_OPT = true 13 | MAX_NUM_ITER = 250 14 | PROGRESS_REPORT_ITER = 1 15 | INNER_STOP_CONDITION = 0.1 16 | MAX_ITER_INNER = 250 17 | MAX_ITER_INNER_FINAL = 2500 18 | TOTAL_NUMBER_FEATURE = -1 19 | CLEAN_CACHE = true 20 | CLEAN_CACHE_ITER = 5 21 | DEMIDCD_NUMBER_OF_UPDATES_BEFORE_UPDATE_BUFFER = 100 22 | DEMIDCD_NUMBER_OF_INF_PARSE_BEFORE_UPDATE_WV = 10 23 | LEARNING_RATE = 0.01 24 | DECAY_LEARNING_RATE = false 25 | -------------------------------------------------------------------------------- /config/kushman/config1.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/allArith/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=5, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config2.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/allArithLex/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=5, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config3.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/allArithTmpl/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=5, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config4.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/aggregate/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=5, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config5.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/aggregateLex/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=5, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config6.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/aggregateTmpl/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=5, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config7.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/allHandcraft/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=2, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config8.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/aggHandcraft/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=2, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/kushman/config9.json: -------------------------------------------------------------------------------- 1 | { 2 | "sFullQuestionFile"="data/questions.json", 3 | "sStanfordParseDirectory"="stanford-parser/parses/", 4 | "sFoldFilePrefix"="data/perturb/fold", 5 | "iPercentageEquationAnnotations"=100, 6 | "iNumFolds"=2, 7 | "iNumThreads"=64, 8 | "iMaxBeamSize"=1000, 9 | "iMaxNumPerSystemInBeam"=100 10 | } 11 | -------------------------------------------------------------------------------- /config/lemma.config: -------------------------------------------------------------------------------- 1 | useStanfordConvention true -------------------------------------------------------------------------------- /config/ner-conll.config: -------------------------------------------------------------------------------- 1 | # Conll config file 2 | 3 | # Required fields 4 | configFilename finalSystemBILOU 5 | pathToModelFile data/Models/CoNLL 6 | taggingEncodingScheme BILOU 7 | tokenizationScheme DualTokenizationScheme 8 | 9 | # Optional fields 10 | beamSize 5 11 | forceNewSentenceOnLineBreaks true 12 | labelTypes PER ORG LOC MISC 13 | logging false 14 | # debuggingLogPath irrelevant 15 | inferenceMethod GREEDY 16 | normalizeTitleText false 17 | pathToTokenNormalizationData brown-clusters/brown-english-wikitext.case-intact.txt-c1000-freq10-v3.txt 18 | predictionConfidenceThreshold -1 19 | sortLexicallyFilesInFolders true 20 | thresholdPrediction false 21 | treatAllFilesInFolderAsOneBigDocument true 22 | debug true 23 | 24 | # Features 25 | Forms 1 26 | Capitalization 1 27 | WordTypeInformation 1 28 | Affixes 1 29 | PreviousTag1 1 30 | PreviousTag2 1 31 | PreviousTagPatternLevel1 1 32 | PreviousTagPatternLevel2 1 33 | AggregateContext 0 34 | AggregateGazetteerMatches 0 35 | PrevTagsForContext 1 36 | PredictionsLevel1 0 37 | 38 | # Feature groups 39 | BrownClusterPaths 1 40 | isLowercaseBrownClusters false false false 41 | pathsToBrownClusters brown-clusters/brown-english-wikitext.case-intact.txt-c1000-freq10-v3.txt brown-clusters/brownBllipClusters brown-clusters/brown-rcv1.clean.tokenized-CoNLL03.txt-c1000-freq1.txt 42 | minWordAppThresholdsForBrownClusters 5 5 5 43 | 44 | GazetteersFeatures 1 45 | pathToGazetteersLists ner-ext/KnownLists 46 | 47 | WordEmbeddings 0 48 | # pathsToWordEmbeddings WordEmbedding/model-2280000000.LEARNING_RATE=1e-08.EMBEDDING_LEARNING_RATE=1e-07.EMBEDDING_SIZE=50.gz 49 | # embeddingDimensionalities 50 50 | # minWordAppThresholdsForEmbeddings 0 51 | # normalizationConstantsForEmbeddings 1.0 52 | # normalizationMethodsForEmbeddings OVERALL 53 | # isLowercaseWordEmbeddings false -------------------------------------------------------------------------------- /config/pipeline.basic.config: -------------------------------------------------------------------------------- 1 | forceCacheUpdate false 2 | cacheDirectory annotation-cache-test 3 | throwExceptionIfNotCached false 4 | cacheHeapSizeInMegabytes 100 5 | cacheDiskSizeInMegabytes 200 6 | setCacheShutdownHook true -------------------------------------------------------------------------------- /config/pipeline.config: -------------------------------------------------------------------------------- 1 | usePos true 2 | useChunker true 3 | useLemmatizer true 4 | useNer true 5 | useNerExt false 6 | useStanfordParse true 7 | forceCacheUpdate false 8 | lemmaConfig config/lemma.config 9 | cacheDirectory annotation-cache-test 10 | throwExceptionIfNotCached false 11 | cacheHeapSizeInMegabytes 100 12 | cacheDiskSizeInMegabytes 200 13 | setCacheShutdownHook true 14 | nerConllConfig config/ner-conll.config -------------------------------------------------------------------------------- /data/aggregate/fold0.txt: -------------------------------------------------------------------------------- 1 | 1232856534 2 | 1232857116 3 | 1232856969 4 | 1344 5 | 1232857076 6 | 1232857271 7 | 1232856562 8 | 1232856943 9 | 1232857188 10 | 1232856574 11 | 1107 12 | 824 13 | 230 14 | 316 15 | 184 16 | 1232856908 17 | 1232856893 18 | 770 19 | 1232857267 20 | 1232857297 21 | 233 22 | 1232856538 23 | 1232856537 24 | 1232856512 25 | 262 26 | 1232856746 27 | 424 28 | 564 29 | 31 30 | 1232856707 31 | 855 32 | 260 33 | 659 34 | 1111 35 | 1232856812 36 | 1232856750 37 | 1232856419 38 | 1232856855 39 | 1232856575 40 | 650 41 | 1141 42 | 165 43 | 1232857013 44 | 1232856880 45 | 1232856571 46 | 1232856396 47 | 311 48 | 1555 49 | 1232856475 50 | 476 51 | 110 52 | 867 53 | 104 54 | 1232856832 55 | 1232856445 56 | 1232857005 57 | 1232856532 58 | 1375 59 | 1232856814 60 | 1232856418 61 | 1232856649 62 | 1232856444 63 | 116 64 | 796 65 | 962 66 | 1232856670 67 | 266 68 | 1232856783 69 | 1426 70 | 1027 71 | 786 72 | 1232856504 73 | 236 74 | 805 75 | 615 76 | 1232856484 77 | 873 78 | 422 79 | 833 80 | 174 81 | 1232857160 82 | 417 83 | 1597 84 | 1232856657 85 | 1401 86 | 1232856868 87 | 119 88 | 1232856590 89 | 1557 90 | 1232856591 91 | 253 92 | 1232856496 93 | 678 94 | 1232856922 95 | 1232856776 96 | 1232856417 97 | 1232856584 98 | 638 99 | 1232857278 100 | 1232856939 101 | 1232857292 102 | 1356 103 | 1447 104 | 1232857004 105 | 1232856809 106 | 1232856992 107 | 1232856773 108 | 835 109 | 357 110 | 212 111 | 1232856395 112 | 1232856542 113 | 1232856632 114 | 711 115 | 1232856761 116 | 854 117 | 1379 118 | 654 119 | 1232856841 120 | 1232856454 121 | 943 122 | 1232857215 123 | 1232856791 124 | 1232856877 125 | 1232857187 126 | 1232856469 127 | 57 128 | 857 129 | 1232856823 130 | 1232856693 131 | 9 132 | 1232856866 133 | 1232856425 134 | 1232856548 135 | 695 136 | 755 137 | 1232856410 138 | 845 139 | 1100 140 | 789 141 | 1232857319 142 | 715 143 | 677 144 | 1232856423 145 | 1577 146 | 1232856796 147 | 1563 148 | 479 149 | 1232856610 150 | 862 151 | 874 152 | 893 153 | 1232856557 154 | 869 155 | 1232856979 156 | 203 157 | 117 158 | 1232856433 159 | 1232856732 160 | 1232856625 161 | 1184 162 | 841 163 | 1232856829 164 | 1232856631 165 | 1462 166 | 858 167 | 1232856972 168 | 838 169 | 1232856822 170 | 1112 171 | 976 172 | 1232856516 173 | 495 174 | 1232856955 175 | 1449 176 | 1576 177 | 250 178 | 1232856565 179 | 1232856651 180 | 223 181 | 780 182 | 1232857286 183 | 1578 184 | 207 185 | 892 186 | 856 187 | 859 188 | 1232856876 189 | 1314 190 | 452 191 | 1232856918 192 | 916 193 | 723 194 | 1232856644 195 | 1232856662 196 | 604 197 | 1610 198 | 651 199 | 717 200 | 1232856928 201 | 1174 202 | 774 203 | 995 204 | 719 205 | 1232857209 206 | 1232856795 207 | 1099 208 | 528 209 | 1232856757 210 | 1232856694 211 | 214 212 | 1232856886 213 | 1232856786 214 | 1026 215 | 843 216 | 1232857162 217 | 889 218 | 1570 219 | 1232856455 220 | 1232857192 221 | 347 222 | 1232856701 223 | 742 224 | 1232856619 225 | 718 226 | 1232856718 227 | 1232856461 228 | 1232856623 229 | 1232856793 230 | 1223 231 | 1232856719 232 | 1547 233 | 1232857006 234 | 197 235 | 794 236 | 1232856422 237 | 1232856681 238 | 764 239 | 1232856472 240 | 645 241 | 1232856553 242 | 231 243 | 761 244 | 1232856802 245 | 865 246 | 793 247 | 577 248 | 1232856736 249 | 132 250 | 1232856700 251 | 885 252 | 1232857021 253 | 870 254 | 853 255 | 1448 256 | 1232856854 257 | 974 258 | 1232856800 259 | 1232856847 260 | 273 261 | 737 262 | 1232856554 263 | 832 264 | 1232856787 265 | 900 266 | 373 267 | 1224 268 | 1608 269 | 668 270 | 1232857166 271 | 745 272 | 758 273 | 1232856769 274 | 527 275 | 1232856853 276 | 1600 277 | 123 278 | 1559 279 | 1232856630 280 | 1232857221 281 | 220 282 | 821 283 | 1341 284 | 1232856824 285 | 1232856617 286 | 1385 287 | 1232856774 288 | 1232856683 289 | 1232856560 290 | 1232856998 291 | 1232857295 292 | 969 293 | 1422 294 | 163 295 | 1562 296 | 722 297 | 1232856507 298 | 897 299 | -------------------------------------------------------------------------------- /data/aggregate/fold1.txt: -------------------------------------------------------------------------------- 1 | 1232856685 2 | 970 3 | 1232857030 4 | 1364 5 | 1232857143 6 | 1232856398 7 | 1232856947 8 | 1232856415 9 | 1181 10 | 1466 11 | 712 12 | 690 13 | 1232856826 14 | 1568 15 | 1232856446 16 | 1232857134 17 | 1232856439 18 | 189 19 | 1232856420 20 | 1 21 | 1232856613 22 | 1232857000 23 | 643 24 | 1232856672 25 | 159 26 | 210 27 | 1579 28 | 1620 29 | 1232857308 30 | 1232856723 31 | 561 32 | 1232856894 33 | 1232856536 34 | 173 35 | 1232856524 36 | 1232856655 37 | 804 38 | 171 39 | 138 40 | 1232857211 41 | 1333 42 | 181 43 | 1232856541 44 | 1232856957 45 | 1226 46 | 822 47 | 733 48 | 622 49 | 830 50 | 265 51 | 1232856690 52 | 1232857092 53 | 1232856535 54 | 8 55 | 878 56 | 1232856540 57 | 877 58 | 1232857132 59 | 209 60 | 1232856692 61 | 637 62 | 751 63 | 1232856573 64 | 692 65 | 1232856821 66 | 1435 67 | 1183 68 | 234 69 | 1232856948 70 | 1031 71 | 1232856759 72 | 814 73 | 684 74 | 1232856479 75 | 1232856830 76 | 633 77 | 2 78 | 1232856789 79 | 1232856891 80 | 1430 81 | 87 82 | 1232856860 83 | 179 84 | 259 85 | 726 86 | 1232857118 87 | 1320 88 | 597 89 | 1232856406 90 | 152 91 | 1030 92 | 852 93 | 828 94 | 1229 95 | 657 96 | 264 97 | 1232856734 98 | 1232856742 99 | 1594 100 | 1232856730 101 | 208 102 | 1232857140 103 | 1232857317 104 | 1232856820 105 | 1232856845 106 | 1232857313 107 | 1232857066 108 | 1232856778 109 | 1232857274 110 | 1232856394 111 | 1232857133 112 | 1232856952 113 | 144 114 | 65 115 | 451 116 | 183 117 | 1117 118 | 756 119 | 1232856815 120 | 1232856443 121 | 1453 122 | 1232856602 123 | 1232856594 124 | 891 125 | 847 126 | 1232856529 127 | 1232856582 128 | 1232856495 129 | 568 130 | 271 131 | 1232856691 132 | 1232856840 133 | 1232856848 134 | 1232856477 135 | 732 136 | 1232857086 137 | 1232856973 138 | 1550 139 | 364 140 | 407 141 | 1232856668 142 | 1232856764 143 | 1232856621 144 | 205 145 | 1232856744 146 | 1232856476 147 | 1232856457 148 | 1232857270 149 | 107 150 | 1232856715 151 | 708 152 | 1232856486 153 | 798 154 | 1043 155 | 1232856811 156 | 1232856449 157 | 1232856813 158 | 1456 159 | 689 160 | 1232856885 161 | 1542 162 | 1232856552 163 | 133 164 | 1232856456 165 | 879 166 | 1232856919 167 | 1232856661 168 | 261 169 | 1232857173 170 | 658 171 | 1232856788 172 | 876 173 | 605 174 | 1553 175 | 194 176 | 213 177 | 754 178 | 1233 179 | 290 180 | 1055 181 | 1232856834 182 | 1248 183 | 1545 184 | 1232856810 185 | 1232857206 186 | 1219 187 | 1232856720 188 | 1543 189 | 598 190 | 33 191 | 1232856735 192 | 1459 193 | 1569 194 | 1232857263 195 | 655 196 | 456 197 | 341 198 | 1232856675 199 | 403 200 | 1232856936 201 | 328 202 | 1029 203 | 1232857064 204 | 1232856578 205 | 1232857115 206 | 648 207 | 13 208 | 667 209 | 1232857220 210 | 1232856915 211 | 849 212 | 640 213 | 1420 214 | 1232857238 215 | 1340 216 | 1232856502 217 | 1232856580 218 | 652 219 | 1232856618 220 | 1232857198 221 | 457 222 | 1232856879 223 | 1232856963 224 | 811 225 | 585 226 | 1047 227 | 1232856603 228 | 783 229 | 562 230 | 776 231 | 1232856940 232 | 1232857088 233 | 1232857303 234 | 905 235 | 1590 236 | 840 237 | 1232856777 238 | 1232856450 239 | 1232857269 240 | 1463 241 | 1232856914 242 | 1232856870 243 | 1225 244 | 1202 245 | 1232857281 246 | 1232857043 247 | 1232856608 248 | 1232856645 249 | 1380 250 | 17 251 | 1232856664 252 | 720 253 | 390 254 | 242 255 | 1230 256 | 801 257 | 1232856587 258 | 1232857219 259 | 1332 260 | 1232856490 261 | 202 262 | 195 263 | 1232856852 264 | 1232856843 265 | 471 266 | 1232856702 267 | 221 268 | 1232856754 269 | 1232856652 270 | 280 271 | 67 272 | 788 273 | 6 274 | 1232856611 275 | 235 276 | 1232856673 277 | 1232856990 278 | 196 279 | 1232856614 280 | 1427 281 | 1232857084 282 | 1232856907 283 | 1232857100 284 | 973 285 | 51 286 | 1102 287 | 778 288 | 1232856470 289 | 644 290 | 1232856505 291 | 890 292 | 1232857148 293 | 1175 294 | 1232857125 295 | 1232856798 296 | 1432 297 | 1232856607 298 | 1232856660 299 | -------------------------------------------------------------------------------- /data/aggregate/fold2.txt: -------------------------------------------------------------------------------- 1 | 283 2 | 1417 3 | 125 4 | 323 5 | 1232856547 6 | 1232857231 7 | 795 8 | 1232856739 9 | 1232856762 10 | 1232856539 11 | 1403 12 | 1232856401 13 | 1232856404 14 | 1232856906 15 | 1232857222 16 | 1232856819 17 | 58 18 | 1232856464 19 | 779 20 | 1232856643 21 | 1232856481 22 | 567 23 | 1028 24 | 1445 25 | 964 26 | 588 27 | 1232856887 28 | 1232857217 29 | 1446 30 | 172 31 | 815 32 | 1232856659 33 | 506 34 | 1232856509 35 | 1232856709 36 | 1232857172 37 | 343 38 | 1101 39 | 837 40 | 1336 41 | 1232856954 42 | 1232856785 43 | 1232856447 44 | 1232857144 45 | 1232856970 46 | 1587 47 | 255 48 | 131 49 | 1232857232 50 | 1232856927 51 | 1171 52 | 647 53 | 200 54 | 753 55 | 190 56 | 1232856561 57 | 1232856498 58 | 1232856748 59 | 1232856583 60 | 707 61 | 1232856489 62 | 1601 63 | 1232857147 64 | 1232856432 65 | 481 66 | 1232857300 67 | 817 68 | 83 69 | 1232857001 70 | 1232857104 71 | 329 72 | 1232856863 73 | 1232856680 74 | 1024 75 | 685 76 | 3 77 | 1231 78 | 43 79 | 810 80 | 1235 81 | 1564 82 | 145 83 | 1232856424 84 | 1232856463 85 | 1232856862 86 | 688 87 | 665 88 | 1232857229 89 | 1560 90 | 1232856658 91 | 14 92 | 380 93 | 454 94 | 177 95 | 812 96 | 219 97 | 1232856589 98 | 1232856803 99 | 295 100 | 725 101 | 1232857326 102 | 1232856549 103 | 1458 104 | 1025 105 | 617 106 | 246 107 | 1232856865 108 | 1232856434 109 | 913 110 | 1232857208 111 | 1232856604 112 | 1236 113 | 109 114 | 1232856833 115 | 1232856522 116 | 319 117 | 1232856851 118 | 987 119 | 254 120 | 1232856520 121 | 1232856985 122 | 1232857177 123 | 1232856987 124 | 1232856448 125 | 1566 126 | 135 127 | 868 128 | 686 129 | 834 130 | 1232856752 131 | 1232856483 132 | 1232856600 133 | 60 134 | 191 135 | 860 136 | 894 137 | 760 138 | 603 139 | 29 140 | 1232856962 141 | 1232856510 142 | 150 143 | 1232856641 144 | 1232856838 145 | 180 146 | 11 147 | 392 148 | 1232856807 149 | 227 150 | 534 151 | 245 152 | 1572 153 | 709 154 | 1232856440 155 | 1232856755 156 | 1232856980 157 | 1232857128 158 | 252 159 | 555 160 | 1232856616 161 | 687 162 | 721 163 | 1232856799 164 | 399 165 | 215 166 | 1232857272 167 | 487 168 | 1232856556 169 | 1232856711 170 | 1548 171 | 1232856714 172 | 1232857171 173 | 1232856612 174 | 1232856902 175 | 967 176 | 1232856414 177 | 727 178 | 1232856794 179 | 972 180 | 1232856429 181 | 636 182 | 1387 183 | 1232856869 184 | 489 185 | 653 186 | 781 187 | 1232856559 188 | 646 189 | 1571 190 | 1232856881 191 | 1232856766 192 | 1232856753 193 | 1232856601 194 | 1023 195 | 1232857093 196 | 1169 197 | 1232857060 198 | 896 199 | 1232856867 200 | 78 201 | 809 202 | 1232856508 203 | 106 204 | 19 205 | 1232856503 206 | 842 207 | 1232856650 208 | 134 209 | 414 210 | 1232857186 211 | 268 212 | 1436 213 | 1232857165 214 | 1450 215 | 1035 216 | 917 217 | 1362 218 | 1232857289 219 | 1452 220 | 1232856808 221 | 1232856568 222 | 1232856646 223 | 803 224 | 675 225 | 1168 226 | 1461 227 | 54 228 | 1232856724 229 | 1232857034 230 | 884 231 | 1232856828 232 | 1232856466 233 | 1323 234 | 1232856816 235 | 671 236 | 1232856713 237 | 1232857018 238 | 563 239 | 127 240 | 1037 241 | 1232857259 242 | 747 243 | 275 244 | 140 245 | 1232856474 246 | 1232856493 247 | 1108 248 | 192 249 | 34 250 | 557 251 | 1232856687 252 | 816 253 | 147 254 | 1232856678 255 | 1614 256 | 249 257 | 1232857096 258 | 1232856689 259 | 97 260 | 1464 261 | 1177 262 | 1195 263 | 1232856756 264 | 1232856674 265 | 1232856895 266 | 148 267 | 1232856430 268 | 1232856965 269 | 1220 270 | 1232857150 271 | 882 272 | 634 273 | 1232856827 274 | 239 275 | 289 276 | 1232856857 277 | 1232856453 278 | 839 279 | 1232857131 280 | 1164 281 | 240 282 | 310 283 | 1370 284 | 1232856763 285 | 1232857122 286 | 1232856858 287 | 1232856784 288 | 572 289 | 1232857007 290 | 530 291 | 1307 292 | 1232856779 293 | 1232857167 294 | 807 295 | 886 296 | 1232856564 297 | 829 298 | 1376 299 | 806 300 | -------------------------------------------------------------------------------- /data/aggregate/fold3.txt: -------------------------------------------------------------------------------- 1 | 1287 2 | 1232856576 3 | 703 4 | 1232856491 5 | 827 6 | 1232856478 7 | 1232856403 8 | 1232856551 9 | 170 10 | 168 11 | 300 12 | 1232856792 13 | 257 14 | 1232856772 15 | 819 16 | 791 17 | 1556 18 | 746 19 | 1232856721 20 | 385 21 | 297 22 | 1232857288 23 | 1232857099 24 | 1232857241 25 | 792 26 | 1460 27 | 1232857194 28 | 1232856725 29 | 1232856624 30 | 327 31 | 1232857158 32 | 1232856882 33 | 965 34 | 1232856530 35 | 901 36 | 1232857023 37 | 1232856733 38 | 1232856804 39 | 237 40 | 846 41 | 1232856480 42 | 1232856780 43 | 282 44 | 151 45 | 1232857252 46 | 121 47 | 728 48 | 1232856775 49 | 1329 50 | 662 51 | 596 52 | 228 53 | 1232857170 54 | 1232856910 55 | 1232856405 56 | 1232856408 57 | 850 58 | 1232856818 59 | 825 60 | 1232856716 61 | 949 62 | 1232856737 63 | 1232856978 64 | 291 65 | 1232857102 66 | 363 67 | 887 68 | 1232857168 69 | 784 70 | 35 71 | 1444 72 | 787 73 | 1232856442 74 | 1247 75 | 130 76 | 393 77 | 1232856699 78 | 1232856566 79 | 1232857235 80 | 157 81 | 826 82 | 1603 83 | 217 84 | 206 85 | 86 86 | 775 87 | 1232856676 88 | 696 89 | 1232857197 90 | 716 91 | 679 92 | 1232856467 93 | 926 94 | 1232857183 95 | 844 96 | 1232856667 97 | 1232857045 98 | 1228 99 | 1232857212 100 | 1232856518 101 | 1232856545 102 | 1232856482 103 | 1232856872 104 | 169 105 | 1189 106 | 1232856654 107 | 238 108 | 274 109 | 899 110 | 1232856781 111 | 1232856703 112 | 702 113 | 188 114 | 224 115 | 1232857119 116 | 52 117 | 279 118 | 1339 119 | 198 120 | 762 121 | 1232856642 122 | 1232857253 123 | 418 124 | 614 125 | 1232856546 126 | 1549 127 | 1232856740 128 | 226 129 | 872 130 | 1232856558 131 | 1232856435 132 | 105 133 | 1552 134 | 124 135 | 1232856638 136 | 759 137 | 1232856899 138 | 88 139 | 1232856890 140 | 730 141 | 1232856609 142 | 963 143 | 175 144 | 1167 145 | 335 146 | 1232856629 147 | 1232856704 148 | 559 149 | 1232856679 150 | 1232856519 151 | 682 152 | 1434 153 | 898 154 | 1232856817 155 | 586 156 | 649 157 | 683 158 | 632 159 | 968 160 | 440 161 | 1114 162 | 36 163 | 864 164 | 1561 165 | 293 166 | 1232856569 167 | 1232856760 168 | 934 169 | 1232857094 170 | 1232857273 171 | 977 172 | 292 173 | 48 174 | 1232856517 175 | 1232857226 176 | 1232856665 177 | 1468 178 | 740 179 | 251 180 | 1232857268 181 | 1232856677 182 | 1232856525 183 | 1232856555 184 | 162 185 | 802 186 | 1232856705 187 | 1232856492 188 | 526 189 | 47 190 | 1335 191 | 641 192 | 1232856671 193 | 978 194 | 1165 195 | 1465 196 | 1232856758 197 | 1232856622 198 | 666 199 | 1232856639 200 | 904 201 | 426 202 | 565 203 | 1232857330 204 | 1120 205 | 1232856592 206 | 56 207 | 693 208 | 741 209 | 871 210 | 1232856397 211 | 587 212 | 1222 213 | 391 214 | 1232856636 215 | 301 216 | 881 217 | 531 218 | 1623 219 | 1232856634 220 | 269 221 | 880 222 | 137 223 | 68 224 | 1232856988 225 | 222 226 | 1588 227 | 1232856797 228 | 1544 229 | 1232856637 230 | 744 231 | 1188 232 | 748 233 | 1232856666 234 | 1036 235 | 516 236 | 158 237 | 749 238 | 1310 239 | 149 240 | 768 241 | 820 242 | 731 243 | 1232856452 244 | 1232856501 245 | 1558 246 | 1232857318 247 | 1227 248 | 1232857223 249 | 697 250 | 1232856462 251 | 218 252 | 1574 253 | 1421 254 | 906 255 | 1457 256 | 178 257 | 1232856751 258 | 1232856837 259 | 122 260 | 1163 261 | 317 262 | 1232857049 263 | 1575 264 | 229 265 | 143 266 | 394 267 | 1232857025 268 | 447 269 | 971 270 | 1232856459 271 | 1232856697 272 | 674 273 | 12 274 | 777 275 | 7 276 | 167 277 | 769 278 | 1551 279 | 146 280 | 25 281 | 773 282 | 312 283 | 267 284 | 706 285 | 1232856577 286 | 1232856620 287 | 1232856904 288 | 1232857195 289 | 15 290 | 1232856451 291 | 1232856770 292 | 1232856912 293 | 1232856765 294 | 1232856626 295 | 1232856959 296 | 1051 297 | 1166 298 | 176 299 | -------------------------------------------------------------------------------- /data/aggregate/fold4.txt: -------------------------------------------------------------------------------- 1 | 1232856597 2 | 757 3 | 1232857298 4 | 642 5 | 1232856741 6 | 1232856884 7 | 20 8 | 153 9 | 164 10 | 507 11 | 445 12 | 1232856615 13 | 523 14 | 910 15 | 694 16 | 1232856767 17 | 73 18 | 782 19 | 1232856412 20 | 425 21 | 75 22 | 672 23 | 480 24 | 823 25 | 156 26 | 1232856806 27 | 1232856983 28 | 861 29 | 1232856953 30 | 980 31 | 1232856500 32 | 1232856585 33 | 613 34 | 161 35 | 836 36 | 1232857321 37 | 1232856835 38 | 1232856544 39 | 664 40 | 1232856892 41 | 225 42 | 1232856570 43 | 442 44 | 1232856421 45 | 1232856431 46 | 1232857182 47 | 1232856437 48 | 1232857123 49 | 1232857163 50 | 1232856698 51 | 1232856896 52 | 676 53 | 1232856581 54 | 799 55 | 1232856458 56 | 397 57 | 1232856426 58 | 1232856964 59 | 1232856506 60 | 1232856917 61 | 1232856921 62 | 713 63 | 1232856511 64 | 120 65 | 1451 66 | 1221 67 | 142 68 | 1232856871 69 | 1232857225 70 | 1232856441 71 | 182 72 | 851 73 | 848 74 | 1565 75 | 1232856738 76 | 771 77 | 1232856633 78 | 1034 79 | 241 80 | 154 81 | 1232856497 82 | 698 83 | 1232856402 84 | 1232856889 85 | 1022 86 | 367 87 | 201 88 | 1554 89 | 1232856929 90 | 111 91 | 1232856640 92 | 1232856400 93 | 136 94 | 735 95 | 1232857277 96 | 1106 97 | 1232857107 98 | 766 99 | 1232856695 100 | 1232856995 101 | 1232857230 102 | 1433 103 | 831 104 | 432 105 | 1589 106 | 1232857242 107 | 1232856935 108 | 248 109 | 166 110 | 1232856749 111 | 1541 112 | 1232856550 113 | 339 114 | 1103 115 | 1232857020 116 | 752 117 | 1232856839 118 | 1232856488 119 | 895 120 | 1232856706 121 | 1342 122 | 1115 123 | 1232857129 124 | 401 125 | 619 126 | 1232856727 127 | 247 128 | 1232856514 129 | 1567 130 | 1232856411 131 | 569 132 | 1232857085 133 | 10 134 | 1232856513 135 | 1243 136 | 1232856515 137 | 705 138 | 1232857106 139 | 1377 140 | 1232856663 141 | 350 142 | 1232856635 143 | 1546 144 | 1232856688 145 | 914 146 | 951 147 | 4 148 | 1232856627 149 | 1232856653 150 | 1232857037 151 | 438 152 | 96 153 | 1232856944 154 | 673 155 | 966 156 | 1232856932 157 | 704 158 | 1613 159 | 1039 160 | 808 161 | 663 162 | 1238 163 | 1038 164 | 700 165 | 1232856708 166 | 216 167 | 1232856428 168 | 1232856836 169 | 903 170 | 797 171 | 1232857331 172 | 1442 173 | 1232857257 174 | 1232856586 175 | 69 176 | 739 177 | 1105 178 | 1232856543 179 | 1232856521 180 | 464 181 | 26 182 | 883 183 | 1232856728 184 | 907 185 | 701 186 | 743 187 | 1232856471 188 | 1232856856 189 | 256 190 | 1232856977 191 | 1232856465 192 | 790 193 | 1232856913 194 | 16 195 | 1232857095 196 | 155 197 | 908 198 | 535 199 | 1232856528 200 | 211 201 | 1232856717 202 | 763 203 | 139 204 | 724 205 | 1581 206 | 670 207 | 628 208 | 1232856588 209 | 573 210 | 1232856743 211 | 1232856722 212 | 1232856647 213 | 1232856686 214 | 342 215 | 187 216 | 185 217 | 1495 218 | 1232856782 219 | 326 220 | 630 221 | 1582 222 | 1232856745 223 | 349 224 | 1232856712 225 | 1232857130 226 | 1232856427 227 | 631 228 | 660 229 | 1232856485 230 | 1232856473 231 | 1232856526 232 | 199 233 | 1232856438 234 | 1232856849 235 | 1232857114 236 | 22 237 | 21 238 | 1232857032 239 | 1232857239 240 | 1232856628 241 | 322 242 | 1232856499 243 | 772 244 | 1596 245 | 656 246 | 986 247 | 1232856567 248 | 1098 249 | 1573 250 | 669 251 | 888 252 | 1232856801 253 | 330 254 | 1232856460 255 | 1232856413 256 | 70 257 | 324 258 | 681 259 | 1104 260 | 750 261 | 800 262 | 1232856771 263 | 1232856656 264 | 108 265 | 1232856864 266 | 160 267 | 1110 268 | 1232856805 269 | 81 270 | 734 271 | 1232856888 272 | 1232857120 273 | 1232857080 274 | 1232857179 275 | 1232856533 276 | 1232856527 277 | 5 278 | 1109 279 | 258 280 | 82 281 | 1232857203 282 | 1232856407 283 | 505 284 | 1232856875 285 | 1232856598 286 | 186 287 | 1232857236 288 | 1424 289 | 244 290 | 1172 291 | 875 292 | 691 293 | 1170 294 | 902 295 | 1232856844 296 | 511 297 | 680 298 | 1232856468 299 | 141 300 | -------------------------------------------------------------------------------- /data/aggregate/info.txt: -------------------------------------------------------------------------------- 1 | Kushman: 2 | 3 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5536912751677853 MargCorrEquations: 0.540268456375839 4 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5134228187919463 MargCorrEquations: 0.5067114093959731 5 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5819397993311036 MargCorrEquations: 0.5652173913043478 6 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5134228187919463 MargCorrEquations: 0.5067114093959731 7 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5585284280936454 MargCorrEquations: 0.5484949832775919 8 | -------------------------------------------------------------------------------- /data/aggregateLex/fold0.txt: -------------------------------------------------------------------------------- 1 | 1232857259 2 | 1232856929 3 | 1232856712 4 | 837 5 | 680 6 | 1232857129 7 | 139 8 | 1335 9 | 535 10 | 1232856686 11 | 1232856632 12 | 1110 13 | 1232856541 14 | 781 15 | 364 16 | 1435 17 | 1232856895 18 | 1232857317 19 | 1232856690 20 | 343 21 | 1232856548 22 | 247 23 | 1232857132 24 | 1232856591 25 | 1457 26 | 1232856888 27 | 780 28 | 1232857049 29 | 1566 30 | 1232856641 31 | 1232856547 32 | 339 33 | 1232856828 34 | 1232856793 35 | 1164 36 | 1027 37 | 1232856884 38 | 487 39 | 615 40 | 269 41 | 142 42 | 1232856505 43 | 1232856522 44 | 1446 45 | 716 46 | 190 47 | 1557 48 | 658 49 | 565 50 | 187 51 | 236 52 | 1232857330 53 | 328 54 | 888 55 | 1232857187 56 | 776 57 | 893 58 | 1232856559 59 | 1232857179 60 | 399 61 | 1232857170 62 | 1232857269 63 | 210 64 | 614 65 | 1232856801 66 | 870 67 | 1232857241 68 | 1620 69 | 1232857203 70 | 1232856734 71 | 587 72 | 1424 73 | 640 74 | 1232856608 75 | 241 76 | 718 77 | 1232856907 78 | 159 79 | 1100 80 | 862 81 | 1051 82 | 1232857096 83 | 1232857274 84 | 1232856410 85 | 795 86 | 821 87 | 1163 88 | 897 89 | 943 90 | 1232856630 91 | 1232857263 92 | 1232857198 93 | 1577 94 | 107 95 | 1565 96 | 1589 97 | 1232856784 98 | 1232856851 99 | 613 100 | 317 101 | 205 102 | 224 103 | 889 104 | 1232856557 105 | 1232856844 106 | 1232857086 107 | 5 108 | 1232857177 109 | 152 110 | 316 111 | 1232856499 112 | 1232857102 113 | 1232856411 114 | 1232856872 115 | 1232856485 116 | 239 117 | 1364 118 | 1232856512 119 | 1168 120 | 158 121 | 619 122 | 1232857100 123 | 143 124 | 695 125 | 1232857099 126 | 713 127 | 1195 128 | 1232857060 129 | 1232857298 130 | 1232856569 131 | 228 132 | 1232856635 133 | 898 134 | 259 135 | 643 136 | 799 137 | 603 138 | 1232857208 139 | 1104 140 | 1232856537 141 | 1573 142 | 1232856418 143 | 319 144 | 1232856571 145 | 134 146 | 189 147 | 1232856904 148 | 782 149 | 170 150 | -------------------------------------------------------------------------------- /data/aggregateLex/fold1.txt: -------------------------------------------------------------------------------- 1 | 257 2 | 1442 3 | 1232857045 4 | 1590 5 | 842 6 | 282 7 | 872 8 | 432 9 | 701 10 | 685 11 | 1232856840 12 | 1232856459 13 | 1232856957 14 | 1551 15 | 874 16 | 703 17 | 1232857020 18 | 227 19 | 1603 20 | 1232856488 21 | 1232856455 22 | 531 23 | 1569 24 | 1596 25 | 1232856927 26 | 1587 27 | 1421 28 | 1232857222 29 | 1120 30 | 910 31 | 1232856716 32 | 1232856816 33 | 972 34 | 1232856970 35 | 660 36 | 1232856417 37 | 1460 38 | 1232856556 39 | 1232856483 40 | 221 41 | 1232857297 42 | 179 43 | 794 44 | 1570 45 | 1232856802 46 | 763 47 | 1232856502 48 | 163 49 | 233 50 | 659 51 | 1232857123 52 | 1232856943 53 | 1232856475 54 | 1232856721 55 | 1232856552 56 | 1232857286 57 | 177 58 | 1232856852 59 | 1232856882 60 | 1232856426 61 | 1232856798 62 | 1232857235 63 | 1232857162 64 | 693 65 | 1236 66 | 1232856592 67 | 1175 68 | 1232856955 69 | 1238 70 | 1543 71 | 489 72 | 1232856818 73 | 1232856685 74 | 1232856607 75 | 183 76 | 1232856953 77 | 237 78 | 1232856581 79 | 764 80 | 196 81 | 1232856769 82 | 1232856691 83 | 1232856486 84 | 839 85 | 1232857217 86 | 628 87 | 4 88 | 1181 89 | 13 90 | 815 91 | 1232856879 92 | 1232857018 93 | 779 94 | 1055 95 | 1451 96 | 125 97 | 1232856515 98 | 1232856940 99 | 684 100 | 1232856497 101 | 194 102 | 1232856973 103 | 1232856397 104 | 758 105 | 1232856783 106 | 201 107 | 1232857270 108 | 1232856444 109 | 820 110 | 881 111 | 310 112 | 1112 113 | 1422 114 | 1225 115 | 1232856672 116 | 851 117 | 1232856758 118 | 199 119 | 1232856694 120 | 1232856746 121 | 1232857225 122 | 1560 123 | 182 124 | 1387 125 | 173 126 | 974 127 | 891 128 | 1546 129 | 1232856766 130 | 1232856503 131 | 1232856889 132 | 1232856493 133 | 865 134 | 1564 135 | 1232856687 136 | 1232856395 137 | 1232857268 138 | 1232856777 139 | 1453 140 | 824 141 | 1232857197 142 | 1600 143 | 1232857000 144 | 1232856765 145 | 849 146 | 1232857206 147 | 1223 148 | 1229 149 | 1232857025 150 | -------------------------------------------------------------------------------- /data/aggregateLex/fold2.txt: -------------------------------------------------------------------------------- 1 | 892 2 | 1232856408 3 | 1098 4 | 1232857209 5 | 1 6 | 1232857084 7 | 1232856704 8 | 1232856826 9 | 1232856683 10 | 1444 11 | 1232857006 12 | 1232856594 13 | 638 14 | 798 15 | 1232856773 16 | 1232856405 17 | 1232856645 18 | 1232856841 19 | 1232857106 20 | 1581 21 | 1232857107 22 | 668 23 | 1232856944 24 | 253 25 | 774 26 | 1232856858 27 | 1232856441 28 | 148 29 | 130 30 | 1232856477 31 | 133 32 | 1232856724 33 | 868 34 | 1114 35 | 172 36 | 242 37 | 1232857182 38 | 292 39 | 1232857271 40 | 506 41 | 191 42 | 1575 43 | 1232856886 44 | 1232856836 45 | 1232856754 46 | 1232856680 47 | 1232856402 48 | 385 49 | 1232856527 50 | 157 51 | 1232856715 52 | 1036 53 | 1232856589 54 | 1232856908 55 | 244 56 | 884 57 | 141 58 | 1232857092 59 | 1232856811 60 | 1232856988 61 | 1232856526 62 | 60 63 | 1232856797 64 | 137 65 | 797 66 | 1232856706 67 | 1228 68 | 1232856693 69 | 330 70 | 212 71 | 1232856663 72 | 1232857066 73 | 707 74 | 847 75 | 1232856932 76 | 1232857114 77 | 323 78 | 1232857326 79 | 681 80 | 1323 81 | 1232856702 82 | 1466 83 | 391 84 | 1232856692 85 | 1232857288 86 | 1232856848 87 | 1232856803 88 | 1232856891 89 | 1232857277 90 | 454 91 | 1232856627 92 | 1169 93 | 796 94 | 116 95 | 817 96 | 1232856854 97 | 1232856681 98 | 6 99 | 1232856570 100 | 1544 101 | 1105 102 | 1232857125 103 | 1232857212 104 | 1232856580 105 | 1232856890 106 | 150 107 | 740 108 | 787 109 | 1232856626 110 | 1202 111 | 1232857168 112 | 1232856506 113 | 1232857172 114 | 1232856460 115 | 184 116 | 748 117 | 1232857013 118 | 808 119 | 176 120 | 1232857331 121 | 1232856549 122 | 1232857115 123 | 788 124 | 1232856540 125 | 809 126 | 1232856703 127 | 636 128 | 1232857239 129 | 637 130 | 174 131 | 1232857289 132 | 631 133 | 557 134 | 1232856534 135 | 890 136 | 905 137 | 198 138 | 819 139 | 451 140 | 791 141 | 1232856584 142 | 1232856838 143 | 162 144 | 1226 145 | 1232856473 146 | 268 147 | 1232857215 148 | 1232856675 149 | 1232857023 150 | -------------------------------------------------------------------------------- /data/aggregateLex/fold3.txt: -------------------------------------------------------------------------------- 1 | 1232856491 2 | 1025 3 | 57 4 | 1232856659 5 | 123 6 | 390 7 | 1232856574 8 | 1232856431 9 | 1547 10 | 1232856737 11 | 652 12 | 1248 13 | 149 14 | 1232857085 15 | 1047 16 | 1235 17 | 1456 18 | 1232856442 19 | 1232856513 20 | 1232856928 21 | 1559 22 | 1232856835 23 | 879 24 | 1232856875 25 | 617 26 | 58 27 | 1232856398 28 | 1571 29 | 140 30 | 1232856796 31 | 1232856498 32 | 1232856536 33 | 1232857242 34 | 686 35 | 886 36 | 1370 37 | 1232856833 38 | 445 39 | 1039 40 | 203 41 | 1232857001 42 | 1232856902 43 | 1184 44 | 630 45 | 850 46 | 577 47 | 1232856495 48 | 1558 49 | 188 50 | 1232856500 51 | 895 52 | 564 53 | 235 54 | 902 55 | 1108 56 | 1550 57 | 706 58 | 279 59 | 1232857037 60 | 175 61 | 1232857150 62 | 256 63 | 1232856952 64 | 861 65 | 900 66 | 1232857319 67 | 1232857043 68 | 1167 69 | 223 70 | 250 71 | 1561 72 | 622 73 | 969 74 | 1542 75 | 251 76 | 1232856819 77 | 407 78 | 597 79 | 131 80 | 1232856954 81 | 688 82 | 1403 83 | 1232856602 84 | 138 85 | 1432 86 | 206 87 | 1232857226 88 | 1227 89 | 214 90 | 132 91 | 530 92 | 634 93 | 1232856699 94 | 1232857005 95 | 1232856470 96 | 1232856492 97 | 1232856590 98 | 252 99 | 1232856921 100 | 894 101 | 1141 102 | 1232856800 103 | 679 104 | 1232856509 105 | 1232856521 106 | 1232856508 107 | 1232856947 108 | 1232856730 109 | 1375 110 | 835 111 | 869 112 | 1170 113 | 217 114 | 1339 115 | 1232856573 116 | 812 117 | 1232857122 118 | 230 119 | 822 120 | 846 121 | 1555 122 | 1232856865 123 | 481 124 | 1232857318 125 | 1232856689 126 | 1232857238 127 | 966 128 | 1459 129 | 153 130 | 1232857167 131 | 871 132 | 1232857116 133 | 1232856707 134 | 1232856474 135 | 802 136 | 311 137 | 186 138 | 1232856542 139 | 1232857223 140 | 197 141 | 135 142 | 1232856598 143 | 261 144 | 1232856757 145 | 1232856519 146 | 1232856437 147 | 1037 148 | 33 149 | 1556 150 | -------------------------------------------------------------------------------- /data/aggregateLex/fold4.txt: -------------------------------------------------------------------------------- 1 | 1232856642 2 | 730 3 | 1232856674 4 | 1232856913 5 | 747 6 | 1232856484 7 | 1232856815 8 | 1232856735 9 | 495 10 | 164 11 | 1232856695 12 | 1232856670 13 | 1232857034 14 | 1232857165 15 | 1232856917 16 | 596 17 | 258 18 | 1232856740 19 | 951 20 | 1232856532 21 | 166 22 | 195 23 | 843 24 | 949 25 | 1332 26 | 1232856603 27 | 1232857221 28 | 1545 29 | 1232856480 30 | 1102 31 | 1232856936 32 | 151 33 | 1232856575 34 | 209 35 | 1232856624 36 | 1232856792 37 | 1232857272 38 | 585 39 | 1232856538 40 | 756 41 | 1232856749 42 | 248 43 | 265 44 | 1594 45 | 1232857131 46 | 1232856433 47 | 1613 48 | 1103 49 | 917 50 | 1232856543 51 | 1623 52 | 1563 53 | 260 54 | 240 55 | 1232857183 56 | 1232856478 57 | 145 58 | 245 59 | 1232857278 60 | 1232857231 61 | 1232856795 62 | 322 63 | 1579 64 | 1232856576 65 | 1232856764 66 | 858 67 | 1232856720 68 | 1450 69 | 793 70 | 1232856434 71 | 202 72 | 1117 73 | 1232857292 74 | 207 75 | 327 76 | 442 77 | 1232857021 78 | 136 79 | 1232856821 80 | 1232856450 81 | 896 82 | 1458 83 | 1232856420 84 | 1232856415 85 | 464 86 | 11 87 | 838 88 | 1232856992 89 | 414 90 | 122 91 | 1232856588 92 | 1608 93 | 885 94 | 1232857143 95 | 567 96 | 867 97 | 181 98 | 816 99 | 899 100 | 859 101 | 1232856514 102 | 752 103 | 665 104 | 1232857119 105 | 1232856516 106 | 692 107 | 1221 108 | 1232857229 109 | 1578 110 | 632 111 | 1232856507 112 | 773 113 | 1232857308 114 | 1232856476 115 | 768 116 | 1232856640 117 | 1232856528 118 | 1232856919 119 | 832 120 | 671 121 | 633 122 | 1232856688 123 | 1222 124 | 1232856440 125 | 1232857188 126 | 1111 127 | 1567 128 | 1232857236 129 | 841 130 | 882 131 | 1232856678 132 | 1232856546 133 | 1232856469 134 | 1232856568 135 | 559 136 | 1232856880 137 | 1232856432 138 | 1232856774 139 | 1232857186 140 | 1232856820 141 | 1232856910 142 | 864 143 | 586 144 | 1232856539 145 | 1232857195 146 | 1232856456 147 | 1106 148 | 1232856489 149 | 1232856658 150 | 1232857273 151 | -------------------------------------------------------------------------------- /data/aggregateLex/info.txt: -------------------------------------------------------------------------------- 1 | Kushman: 2 | 3 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.37583892617449666 MargCorrEquations: 0.37583892617449666 4 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.4899328859060403 MargCorrEquations: 0.47651006711409394 5 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5100671140939598 MargCorrEquations: 0.5033557046979866 6 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5167785234899329 MargCorrEquations: 0.5369127516778524 7 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.42 MargCorrEquations: 0.41333333333333333 8 | -------------------------------------------------------------------------------- /data/aggregateTmpl/fold0.txt: -------------------------------------------------------------------------------- 1 | 1342 2 | 615 3 | 215 4 | 1339 5 | 1232856965 6 | 78 7 | 1565 8 | 156 9 | 1029 10 | 195 11 | 795 12 | 1232856474 13 | 29 14 | 563 15 | 903 16 | 971 17 | 868 18 | 1 19 | 1232856954 20 | 636 21 | 1568 22 | 523 23 | 1574 24 | 837 25 | 1232856952 26 | 1229 27 | 898 28 | 632 29 | 1109 30 | 1232856875 31 | 1226 32 | 159 33 | 838 34 | 67 35 | 1232857221 36 | 1453 37 | 68 38 | 25 39 | 87 40 | 1430 41 | 747 42 | 1112 43 | 870 44 | 1232857084 45 | 1024 46 | 1232856428 47 | 70 48 | 168 49 | 1047 50 | 1340 51 | 1432 52 | 5 53 | 108 54 | 268 55 | 11 56 | 857 57 | 1231 58 | 1232857096 59 | 1232856432 60 | 1232857034 61 | 107 62 | 2 63 | 1232857217 64 | 1099 65 | 750 66 | 319 67 | 1174 68 | 1232857226 69 | 781 70 | 644 71 | 342 72 | 1164 73 | 1232857076 74 | 865 75 | 73 76 | 1232856427 77 | 347 78 | 867 79 | 637 80 | 1362 81 | 1232856944 82 | 808 83 | 1232856589 84 | 16 85 | 1232856980 86 | 1401 87 | 1232856609 88 | 1039 89 | 1232856420 90 | 633 91 | 1232856964 92 | 1462 93 | 198 94 | 131 95 | 1466 96 | 116 97 | 343 98 | 191 99 | 1232856983 100 | 164 101 | 1171 102 | 202 103 | 1447 104 | 1232857093 105 | 1232857092 106 | 978 107 | 9 108 | 1232856419 109 | 872 110 | 1563 111 | 282 112 | 897 113 | 1170 114 | 792 115 | 854 116 | 1037 117 | 1232856963 118 | 1573 119 | 1232856935 120 | 1232857192 121 | 167 122 | 1232857043 123 | 1232856414 124 | 1232857006 125 | 794 126 | 506 127 | 876 128 | 322 129 | 907 130 | 1232856586 131 | 233 132 | 1232856947 133 | 1232856410 134 | 1564 135 | 835 136 | 787 137 | 1232856939 138 | 1025 139 | 1434 140 | 81 141 | 86 142 | 1232856570 143 | 132 144 | 1232857223 145 | 968 146 | 1445 147 | 212 148 | 859 149 | 1232856955 150 | -------------------------------------------------------------------------------- /data/aggregateTmpl/fold1.txt: -------------------------------------------------------------------------------- 1 | 1232857230 2 | 105 3 | 163 4 | 242 5 | 1232856962 6 | 526 7 | 1232857088 8 | 181 9 | 559 10 | 238 11 | 605 12 | 1055 13 | 802 14 | 217 15 | 1232857187 16 | 853 17 | 240 18 | 184 19 | 1221 20 | 162 21 | 1232857085 22 | 622 23 | 1232857231 24 | 934 25 | 122 26 | 1202 27 | 1571 28 | 1228 29 | 862 30 | 189 31 | 236 32 | 119 33 | 1232856417 34 | 1023 35 | 1232856849 36 | 926 37 | 264 38 | 150 39 | 1572 40 | 1232856936 41 | 628 42 | 1375 43 | 1232857198 44 | 1427 45 | 1247 46 | 246 47 | 219 48 | 1341 49 | 1232857130 50 | 1232857102 51 | 1232856603 52 | 631 53 | 1232856987 54 | 31 55 | 244 56 | 1232857049 57 | 1172 58 | 106 59 | 171 60 | 221 61 | 1232857235 62 | 261 63 | 1232856418 64 | 155 65 | 1223 66 | 1232857163 67 | 511 68 | 1232857095 69 | 1232857132 70 | 1022 71 | 908 72 | 864 73 | 980 74 | 814 75 | 1232856429 76 | 1336 77 | 598 78 | 1232856592 79 | 218 80 | 1232856477 81 | 1101 82 | 784 83 | 7 84 | 1232857267 85 | 161 86 | 235 87 | 1450 88 | 1380 89 | 977 90 | 1232856415 91 | 986 92 | 1566 93 | 1376 94 | 34 95 | 507 96 | 124 97 | 36 98 | 151 99 | 1232857018 100 | 1232857206 101 | 779 102 | 1232856733 103 | 1356 104 | 271 105 | 804 106 | 913 107 | 206 108 | 1461 109 | 786 110 | 123 111 | 828 112 | 1232857045 113 | 234 114 | 1452 115 | 216 116 | 20 117 | 657 118 | 1435 119 | 1232857239 120 | 1232856597 121 | 1459 122 | 1232857099 123 | 836 124 | 806 125 | 1238 126 | 905 127 | 52 128 | 26 129 | 1106 130 | 1232856431 131 | 166 132 | 790 133 | 1224 134 | 614 135 | 1165 136 | 1232857236 137 | 48 138 | 172 139 | 603 140 | 1433 141 | 1181 142 | 777 143 | 152 144 | 773 145 | 1594 146 | 901 147 | 800 148 | 914 149 | 275 150 | -------------------------------------------------------------------------------- /data/aggregateTmpl/fold2.txt: -------------------------------------------------------------------------------- 1 | 1232856979 2 | 1420 3 | 1098 4 | 949 5 | 797 6 | 809 7 | 1141 8 | 364 9 | 1232856411 10 | 1038 11 | 323 12 | 916 13 | 534 14 | 1570 15 | 1232856985 16 | 1177 17 | 56 18 | 848 19 | 963 20 | 10 21 | 1446 22 | 1232856576 23 | 135 24 | 619 25 | 1232857030 26 | 1232856977 27 | 653 28 | 555 29 | 904 30 | 188 31 | 1232856587 32 | 535 33 | 758 34 | 1232856433 35 | 1188 36 | 165 37 | 902 38 | 799 39 | 1370 40 | 178 41 | 1051 42 | 815 43 | 1232856732 44 | 1232857208 45 | 1329 46 | 223 47 | 851 48 | 142 49 | 531 50 | 121 51 | 1232857106 52 | 60 53 | 967 54 | 228 55 | 257 56 | 1232857005 57 | 1110 58 | 834 59 | 1232856953 60 | 367 61 | 1105 62 | 1232856413 63 | 139 64 | 1103 65 | 35 66 | 1232856604 67 | 125 68 | 1232857060 69 | 1117 70 | 1232856600 71 | 226 72 | 1233 73 | 253 74 | 1232857114 75 | 192 76 | 1560 77 | 1232856571 78 | 136 79 | 910 80 | 82 81 | 154 82 | 8 83 | 1379 84 | 1232856575 85 | 969 86 | 1232857211 87 | 791 88 | 1167 89 | 964 90 | 1232857203 91 | 185 92 | 1555 93 | 43 94 | 1168 95 | 1232857197 96 | 789 97 | 179 98 | 220 99 | 22 100 | 169 101 | 1232856972 102 | 1232857001 103 | 1232857252 104 | 310 105 | 1232857177 106 | 229 107 | 846 108 | 972 109 | 207 110 | 1232857013 111 | 1232857259 112 | 109 113 | 1232856602 114 | 1232857212 115 | 1232857131 116 | 110 117 | 1232856434 118 | 874 119 | 1111 120 | 617 121 | 793 122 | 1026 123 | 1232856998 124 | 751 125 | 753 126 | 821 127 | 829 128 | 1344 129 | 12 130 | 1232857007 131 | 782 132 | 974 133 | 906 134 | 778 135 | 951 136 | 1556 137 | 760 138 | 1232856940 139 | 860 140 | 1232856830 141 | 1600 142 | 1108 143 | 210 144 | 111 145 | 175 146 | 1561 147 | 153 148 | 811 149 | 1102 150 | -------------------------------------------------------------------------------- /data/aggregateTmpl/fold3.txt: -------------------------------------------------------------------------------- 1 | 1613 2 | 267 3 | 250 4 | 225 5 | 141 6 | 1232857086 7 | 54 8 | 197 9 | 1232857220 10 | 1232856435 11 | 561 12 | 962 13 | 203 14 | 1232856437 15 | 147 16 | 1562 17 | 1232857107 18 | 1232857238 19 | 145 20 | 1558 21 | 1232856607 22 | 117 23 | 861 24 | 1232856574 25 | 987 26 | 889 27 | 1232856978 28 | 177 29 | 871 30 | 1232857004 31 | 788 32 | 1104 33 | 1232856943 34 | 1184 35 | 1115 36 | 1610 37 | 208 38 | 1456 39 | 170 40 | 350 41 | 1310 42 | 1232856957 43 | 1230 44 | 1232856988 45 | 1559 46 | 13 47 | 205 48 | 530 49 | 1232857115 50 | 230 51 | 1031 52 | 1232856573 53 | 19 54 | 807 55 | 1232857125 56 | 69 57 | 265 58 | 1232857094 59 | 1243 60 | 1114 61 | 130 62 | 262 63 | 1232857118 64 | 183 65 | 774 66 | 1468 67 | 1232856590 68 | 249 69 | 1465 70 | 199 71 | 1028 72 | 528 73 | 4 74 | 772 75 | 1225 76 | 158 77 | 832 78 | 516 79 | 1232857128 80 | 1232857241 81 | 1220 82 | 654 83 | 1100 84 | 1457 85 | 222 86 | 1232856608 87 | 160 88 | 869 89 | 1232857229 90 | 1120 91 | 17 92 | 237 93 | 227 94 | 1027 95 | 363 96 | 33 97 | 730 98 | 254 99 | 1232857263 100 | 1387 101 | 1163 102 | 1448 103 | 211 104 | 656 105 | 1232857165 106 | 138 107 | 1232857232 108 | 1364 109 | 1232856969 110 | 1232857120 111 | 1232856995 112 | 680 113 | 1232856990 114 | 1335 115 | 140 116 | 201 117 | 1107 118 | 1248 119 | 1232857032 120 | 849 121 | 812 122 | 564 123 | 266 124 | 148 125 | 1232857104 126 | 1426 127 | 174 128 | 752 129 | 638 130 | 182 131 | 688 132 | 1232857253 133 | 826 134 | 1232857023 135 | 1232856970 136 | 1232856478 137 | 1458 138 | 47 139 | 196 140 | 231 141 | 200 142 | 349 143 | 1232856829 144 | 120 145 | 776 146 | 214 147 | 1034 148 | 1169 149 | 251 150 | -------------------------------------------------------------------------------- /data/aggregateTmpl/fold4.txt: -------------------------------------------------------------------------------- 1 | 97 2 | 1569 3 | 1232857215 4 | 847 5 | 1232857066 6 | 248 7 | 1189 8 | 15 9 | 88 10 | 1232856959 11 | 1232856598 12 | 875 13 | 852 14 | 877 15 | 209 16 | 1567 17 | 1385 18 | 850 19 | 831 20 | 858 21 | 65 22 | 176 23 | 1436 24 | 96 25 | 527 26 | 1195 27 | 1183 28 | 255 29 | 562 30 | 505 31 | 1232857122 32 | 311 33 | 1232856948 34 | 134 35 | 1232856430 36 | 1232857064 37 | 976 38 | 149 39 | 1232856594 40 | 557 41 | 1232857326 42 | 239 43 | 973 44 | 190 45 | 820 46 | 1232857021 47 | 1232857000 48 | 259 49 | 1236 50 | 1232856473 51 | 137 52 | 1232857319 53 | 51 54 | 1332 55 | 1232857123 56 | 1232857129 57 | 180 58 | 247 59 | 1232857173 60 | 1232856591 61 | 775 62 | 133 63 | 187 64 | 805 65 | 104 66 | 798 67 | 856 68 | 173 69 | 1222 70 | 855 71 | 1449 72 | 1232857222 73 | 213 74 | 1333 75 | 14 76 | 127 77 | 1166 78 | 1036 79 | 241 80 | 1219 81 | 783 82 | 801 83 | 1377 84 | 21 85 | 1232857100 86 | 630 87 | 1235 88 | 1232856973 89 | 260 90 | 780 91 | 966 92 | 57 93 | 1232856992 94 | 917 95 | 943 96 | 1232856412 97 | 1557 98 | 1030 99 | 845 100 | 1232857037 101 | 1232856588 102 | 1232857119 103 | 824 104 | 995 105 | 1451 106 | 1232857219 107 | 256 108 | 1232857020 109 | 144 110 | 640 111 | 6 112 | 965 113 | 258 114 | 143 115 | 283 116 | 1043 117 | 1175 118 | 970 119 | 1232856475 120 | 224 121 | 194 122 | 796 123 | 816 124 | 186 125 | 1232857080 126 | 83 127 | 269 128 | 1227 129 | 1232857025 130 | 899 131 | 3 132 | 1232857209 133 | 803 134 | 1232857188 135 | 1460 136 | 827 137 | 1232856476 138 | 75 139 | 157 140 | 822 141 | 900 142 | 881 143 | 245 144 | 146 145 | 1035 146 | 58 147 | 252 148 | 634 149 | 1232857116 150 | 1444 151 | -------------------------------------------------------------------------------- /data/aggregateTmpl/info.txt: -------------------------------------------------------------------------------- 1 | Kushman: 2 | 3 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5637583892617449 MargCorrEquations: 0.5570469798657718 4 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5369127516778524 MargCorrEquations: 0.5234899328859061 5 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.610738255033557 MargCorrEquations: 0.6040268456375839 6 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5302013422818792 MargCorrEquations: 0.5100671140939598 7 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5533333333333333 MargCorrEquations: 0.56 8 | -------------------------------------------------------------------------------- /data/allArith/fold0.txt: -------------------------------------------------------------------------------- 1 | 1314 2 | 319 3 | 1466 4 | 1221 5 | 511 6 | 153 7 | 1320 8 | 755 9 | 19 10 | 1333 11 | 208 12 | 68 13 | 557 14 | 526 15 | 839 16 | 1 17 | 489 18 | 848 19 | 828 20 | 852 21 | 26 22 | 833 23 | 834 24 | 796 25 | 1039 26 | 880 27 | 622 28 | 1163 29 | 1248 30 | 216 31 | 758 32 | 149 33 | 246 34 | 1101 35 | 803 36 | 109 37 | 1577 38 | 335 39 | 1370 40 | 1590 41 | 70 42 | 349 43 | 1579 44 | 1571 45 | 860 46 | 1546 47 | 902 48 | 1544 49 | 1165 50 | 267 51 | 655 52 | 762 53 | 789 54 | 279 55 | 1451 56 | 1356 57 | 662 58 | 646 59 | 739 60 | 1448 61 | 454 62 | 685 63 | 1620 64 | 1603 65 | 392 66 | 401 67 | 1442 68 | 794 69 | 778 70 | 226 71 | 887 72 | 615 73 | 697 74 | 691 75 | 236 76 | 707 77 | 162 78 | 1588 79 | 1569 80 | 168 81 | 1452 82 | 807 83 | 154 84 | 155 85 | 1457 86 | 407 87 | 367 88 | 837 89 | 121 90 | 33 91 | 530 92 | 67 93 | 506 94 | 756 95 | 1377 96 | 1562 97 | 385 98 | 326 99 | 577 100 | 29 101 | 588 102 | 244 103 | 144 104 | 130 105 | 220 106 | 188 107 | 759 108 | 896 109 | 559 110 | 425 111 | 140 112 | 457 113 | 633 114 | 868 115 | 21 116 | 1613 117 | 634 118 | 177 119 | 703 120 | 795 121 | 1420 122 | 1495 123 | 649 124 | 106 125 | 1545 126 | 201 127 | 886 128 | 451 129 | 572 130 | 166 131 | 1403 132 | 213 133 | 1568 134 | 202 135 | 971 136 | 708 137 | 773 138 | 432 139 | 1376 140 | 888 141 | 881 142 | 690 143 | 861 144 | 152 145 | 1551 146 | 69 147 | 147 148 | 688 149 | 1549 150 | 87 151 | 733 152 | 316 153 | 700 154 | 686 155 | 1559 156 | 251 157 | 598 158 | 1099 159 | 718 160 | 980 161 | 1446 162 | 235 163 | 1235 164 | 1225 165 | 1342 166 | 882 167 | -------------------------------------------------------------------------------- /data/allArith/fold1.txt: -------------------------------------------------------------------------------- 1 | 605 2 | 883 3 | 749 4 | 120 5 | 704 6 | 876 7 | 1219 8 | 746 9 | 1102 10 | 905 11 | 1141 12 | 1552 13 | 783 14 | 1463 15 | 894 16 | 1034 17 | 872 18 | 176 19 | 25 20 | 734 21 | 968 22 | 228 23 | 256 24 | 914 25 | 817 26 | 698 27 | 1031 28 | 904 29 | 645 30 | 12 31 | 951 32 | 1166 33 | 934 34 | 780 35 | 838 36 | 664 37 | 730 38 | 36 39 | 587 40 | 191 41 | 1570 42 | 1557 43 | 1379 44 | 148 45 | 806 46 | 452 47 | 694 48 | 209 49 | 339 50 | 657 51 | 906 52 | 884 53 | 175 54 | 183 55 | 295 56 | 269 57 | 870 58 | 167 59 | 1236 60 | 1168 61 | 858 62 | 516 63 | 641 64 | 132 65 | 75 66 | 227 67 | 289 68 | 1461 69 | 1465 70 | 1224 71 | 43 72 | 821 73 | 242 74 | 237 75 | 1175 76 | 865 77 | 1542 78 | 1573 79 | 141 80 | 804 81 | 1550 82 | 1184 83 | 1332 84 | 111 85 | 871 86 | 674 87 | 293 88 | 51 89 | 107 90 | 531 91 | 829 92 | 234 93 | 1181 94 | 390 95 | 347 96 | 619 97 | 424 98 | 809 99 | 14 100 | 447 101 | 586 102 | 962 103 | 1167 104 | 856 105 | 712 106 | 185 107 | 716 108 | 851 109 | 1362 110 | 667 111 | 252 112 | 3 113 | 222 114 | 900 115 | 163 116 | 719 117 | 223 118 | 822 119 | 96 120 | 814 121 | 573 122 | 1385 123 | 752 124 | 1375 125 | 916 126 | 908 127 | 6 128 | 917 129 | 770 130 | 643 131 | 1445 132 | 397 133 | 562 134 | 628 135 | 1335 136 | 652 137 | 659 138 | 137 139 | 174 140 | 81 141 | 596 142 | 505 143 | 380 144 | 1103 145 | 180 146 | 160 147 | 847 148 | 1558 149 | 1464 150 | 1170 151 | 534 152 | 445 153 | 1578 154 | 78 155 | 264 156 | 249 157 | 650 158 | 903 159 | 210 160 | 825 161 | 31 162 | 16 163 | 456 164 | 819 165 | 1450 166 | 597 167 | -------------------------------------------------------------------------------- /data/allArith/fold2.txt: -------------------------------------------------------------------------------- 1 | 1340 2 | 978 3 | 1227 4 | 1564 5 | 892 6 | 108 7 | 178 8 | 857 9 | 774 10 | 217 11 | 238 12 | 1422 13 | 1098 14 | 245 15 | 1230 16 | 695 17 | 792 18 | 214 19 | 317 20 | 182 21 | 969 22 | 787 23 | 692 24 | 1226 25 | 261 26 | 343 27 | 965 28 | 1623 29 | 274 30 | 723 31 | 1563 32 | 648 33 | 728 34 | 793 35 | 666 36 | 1107 37 | 290 38 | 824 39 | 757 40 | 442 41 | 669 42 | 528 43 | 88 44 | 744 45 | 187 46 | 885 47 | 1024 48 | 747 49 | 1548 50 | 742 51 | 225 52 | 1177 53 | 995 54 | 1401 55 | 1433 56 | 322 57 | 671 58 | 684 59 | 569 60 | 835 61 | 766 62 | 873 63 | 300 64 | 1100 65 | 713 66 | 1287 67 | 1025 68 | 327 69 | 614 70 | 357 71 | 561 72 | 241 73 | 253 74 | 8 75 | 1030 76 | 1574 77 | 124 78 | 2 79 | 832 80 | 673 81 | 1553 82 | 297 83 | 899 84 | 555 85 | 35 86 | 58 87 | 1307 88 | 987 89 | 720 90 | 874 91 | 438 92 | 151 93 | 342 94 | 324 95 | 800 96 | 853 97 | 196 98 | 565 99 | 1600 100 | 1561 101 | 820 102 | 976 103 | 17 104 | 48 105 | 399 106 | 11 107 | 189 108 | 970 109 | 86 110 | 301 111 | 233 112 | 1380 113 | 810 114 | 414 115 | 259 116 | 677 117 | 393 118 | 850 119 | 83 120 | 207 121 | 330 122 | 1468 123 | 907 124 | 1172 125 | 262 126 | 875 127 | 797 128 | 275 129 | 1453 130 | 751 131 | 1105 132 | 479 133 | 199 134 | 771 135 | 637 136 | 138 137 | 1560 138 | 1435 139 | 811 140 | 854 141 | 977 142 | 680 143 | 1459 144 | 709 145 | 801 146 | 890 147 | 329 148 | 826 149 | 230 150 | 1174 151 | 638 152 | 632 153 | 1029 154 | 779 155 | 403 156 | 422 157 | 4 158 | 127 159 | 1608 160 | 913 161 | 373 162 | 203 163 | 781 164 | 1043 165 | 1567 166 | 34 167 | -------------------------------------------------------------------------------- /data/allArith/fold3.txt: -------------------------------------------------------------------------------- 1 | 146 2 | 893 3 | 194 4 | 670 5 | 1576 6 | 642 7 | 1035 8 | 280 9 | 426 10 | 777 11 | 631 12 | 1027 13 | 681 14 | 974 15 | 134 16 | 13 17 | 312 18 | 1427 19 | 841 20 | 282 21 | 1462 22 | 1575 23 | 292 24 | 480 25 | 1115 26 | 864 27 | 1587 28 | 1228 29 | 157 30 | 725 31 | 1112 32 | 1223 33 | 842 34 | 808 35 | 116 36 | 799 37 | 753 38 | 689 39 | 836 40 | 190 41 | 1028 42 | 1594 43 | 846 44 | 1364 45 | 711 46 | 665 47 | 949 48 | 164 49 | 341 50 | 258 51 | 889 52 | 830 53 | 56 54 | 764 55 | 786 56 | 1547 57 | 7 58 | 119 59 | 1556 60 | 136 61 | 535 62 | 1169 63 | 1541 64 | 1434 65 | 54 66 | 1555 67 | 845 68 | 1566 69 | 727 70 | 840 71 | 672 72 | 1106 73 | 668 74 | 97 75 | 702 76 | 1238 77 | 1188 78 | 630 79 | 1417 80 | 495 81 | 898 82 | 173 83 | 653 84 | 737 85 | 487 86 | 271 87 | 310 88 | 1458 89 | 717 90 | 802 91 | 1581 92 | 1447 93 | 973 94 | 391 95 | 585 96 | 192 97 | 215 98 | 812 99 | 1614 100 | 122 101 | 867 102 | 701 103 | 877 104 | 363 105 | 1164 106 | 1108 107 | 1229 108 | 679 109 | 640 110 | 732 111 | 179 112 | 1189 113 | 823 114 | 170 115 | 897 116 | 563 117 | 1421 118 | 926 119 | 735 120 | 224 121 | 724 122 | 603 123 | 1026 124 | 943 125 | 715 126 | 604 127 | 1424 128 | 523 129 | 1323 130 | 693 131 | 743 132 | 910 133 | 843 134 | 1344 135 | 722 136 | 760 137 | 22 138 | 784 139 | 221 140 | 440 141 | 617 142 | 726 143 | 394 144 | 125 145 | 1038 146 | 1247 147 | 816 148 | 1171 149 | 1436 150 | 247 151 | 1449 152 | 660 153 | 57 154 | 654 155 | 745 156 | 250 157 | 895 158 | 15 159 | 418 160 | 782 161 | 507 162 | 788 163 | 564 164 | 869 165 | 1183 166 | 687 167 | -------------------------------------------------------------------------------- /data/allArith/fold4.txt: -------------------------------------------------------------------------------- 1 | 1195 2 | 197 3 | 471 4 | 143 5 | 268 6 | 964 7 | 1572 8 | 1426 9 | 205 10 | 678 11 | 613 12 | 844 13 | 161 14 | 986 15 | 1601 16 | 1104 17 | 855 18 | 266 19 | 1114 20 | 255 21 | 248 22 | 966 23 | 1051 24 | 768 25 | 1387 26 | 798 27 | 740 28 | 1036 29 | 169 30 | 47 31 | 721 32 | 901 33 | 859 34 | 156 35 | 219 36 | 273 37 | 1310 38 | 682 39 | 159 40 | 104 41 | 1117 42 | 257 43 | 186 44 | 212 45 | 195 46 | 831 47 | 1456 48 | 683 49 | 328 50 | 211 51 | 476 52 | 1543 53 | 139 54 | 731 55 | 1339 56 | 206 57 | 1202 58 | 1589 59 | 142 60 | 65 61 | 879 62 | 73 63 | 647 64 | 636 65 | 963 66 | 5 67 | 52 68 | 748 69 | 1120 70 | 658 71 | 761 72 | 663 73 | 1610 74 | 1432 75 | 1231 76 | 644 77 | 706 78 | 172 79 | 568 80 | 1109 81 | 754 82 | 323 83 | 1243 84 | 750 85 | 283 86 | 171 87 | 772 88 | 1047 89 | 651 90 | 60 91 | 1554 92 | 741 93 | 364 94 | 849 95 | 481 96 | 1037 97 | 135 98 | 311 99 | 967 100 | 815 101 | 527 102 | 110 103 | 229 104 | 82 105 | 464 106 | 254 107 | 158 108 | 763 109 | 775 110 | 1111 111 | 1110 112 | 827 113 | 1233 114 | 150 115 | 20 116 | 260 117 | 776 118 | 805 119 | 675 120 | 145 121 | 1336 122 | 769 123 | 790 124 | 878 125 | 231 126 | 181 127 | 10 128 | 105 129 | 265 130 | 891 131 | 696 132 | 1022 133 | 567 134 | 972 135 | 1597 136 | 133 137 | 1444 138 | 184 139 | 1220 140 | 117 141 | 862 142 | 1582 143 | 239 144 | 198 145 | 291 146 | 1596 147 | 1460 148 | 1565 149 | 218 150 | 200 151 | 1023 152 | 1341 153 | 1222 154 | 240 155 | 1430 156 | 123 157 | 131 158 | 705 159 | 1055 160 | 676 161 | 350 162 | 791 163 | 165 164 | 1329 165 | 417 166 | 9 167 | 656 168 | -------------------------------------------------------------------------------- /data/allArith/info.txt: -------------------------------------------------------------------------------- 1 | Kushman: 2 | 3 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7590361445783133 MargCorrEquations: 0.7469879518072289 4 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.6927710843373494 MargCorrEquations: 0.6867469879518072 5 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7289156626506024 MargCorrEquations: 0.7048192771084337 6 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7771084337349398 MargCorrEquations: 0.7650602409638554 7 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.6766467065868264 MargCorrEquations: 0.6646706586826348 8 | -------------------------------------------------------------------------------- /data/allArithLex/fold0.txt: -------------------------------------------------------------------------------- 1 | 1603 2 | 188 3 | 692 4 | 385 5 | 166 6 | 614 7 | 660 8 | 1098 9 | 809 10 | 899 11 | 640 12 | 1195 13 | 259 14 | 130 15 | 894 16 | 820 17 | 889 18 | 35 19 | 874 20 | 871 21 | 891 22 | 763 23 | 872 24 | 1229 25 | 1453 26 | 779 27 | 758 28 | 969 29 | 163 30 | 1589 31 | 905 32 | 1575 33 | 1222 34 | 330 35 | 782 36 | 1573 37 | 107 38 | 819 39 | 139 40 | 622 41 | 260 42 | 701 43 | 535 44 | 893 45 | 328 46 | 1451 47 | 881 48 | 1170 49 | 1023 50 | 1106 51 | 896 52 | 1120 53 | 1223 54 | 235 55 | 143 56 | 815 57 | 829 58 | 839 59 | 1105 60 | 634 61 | 1446 62 | 879 63 | 781 64 | 716 65 | 145 66 | 530 67 | 141 68 | 1456 69 | 603 70 | 1566 71 | 245 72 | 637 73 | 233 74 | 1578 75 | 1579 76 | 764 77 | 138 78 | 1403 79 | 895 80 | 587 81 | 835 82 | 1228 83 | 638 84 | -------------------------------------------------------------------------------- /data/allArithLex/fold1.txt: -------------------------------------------------------------------------------- 1 | 1466 2 | 1100 3 | 159 4 | 214 5 | 1552 6 | 730 7 | 283 8 | 822 9 | 850 10 | 1036 11 | 60 12 | 846 13 | 797 14 | 51 15 | 897 16 | 1108 17 | 1380 18 | 773 19 | 343 20 | 183 21 | 1458 22 | 636 23 | 1558 24 | 175 25 | 1594 26 | 665 27 | 442 28 | 252 29 | 135 30 | 153 31 | 794 32 | 885 33 | 179 34 | 1457 35 | 1037 36 | 851 37 | 1570 38 | 323 39 | 1184 40 | 57 41 | 1560 42 | 882 43 | 258 44 | 1561 45 | 173 46 | 842 47 | 187 48 | 194 49 | 862 50 | 658 51 | 1055 52 | 816 53 | 843 54 | 152 55 | 791 56 | 203 57 | 1039 58 | 282 59 | 951 60 | 1102 61 | 1556 62 | 202 63 | 847 64 | 184 65 | 121 66 | 837 67 | 917 68 | 1559 69 | 1202 70 | 1375 71 | 1620 72 | 615 73 | 688 74 | 65 75 | 838 76 | 1104 77 | 228 78 | 752 79 | 339 80 | 747 81 | 6 82 | 972 83 | 799 84 | -------------------------------------------------------------------------------- /data/allArithLex/fold2.txt: -------------------------------------------------------------------------------- 1 | 1600 2 | 633 3 | 172 4 | 52 5 | 841 6 | 793 7 | 157 8 | 1608 9 | 322 10 | 253 11 | 943 12 | 177 13 | 821 14 | 1164 15 | 223 16 | 186 17 | 196 18 | 1166 19 | 317 20 | 1221 21 | 123 22 | 868 23 | 768 24 | 217 25 | 58 26 | 1433 27 | 454 28 | 1332 29 | 241 30 | 630 31 | 1248 32 | 1422 33 | 884 34 | 155 35 | 212 36 | 22 37 | 1167 38 | 824 39 | 1544 40 | 1181 41 | 125 42 | 1110 43 | 1565 44 | 247 45 | 1581 46 | 198 47 | 966 48 | 199 49 | 963 50 | 407 51 | 133 52 | 176 53 | 1169 54 | 1577 55 | 390 56 | 242 57 | 1551 58 | 1596 59 | 489 60 | 1025 61 | 248 62 | 158 63 | 230 64 | 559 65 | 1163 66 | 681 67 | 1236 68 | 685 69 | 209 70 | 33 71 | 1335 72 | 149 73 | 206 74 | 1613 75 | 1227 76 | 706 77 | 148 78 | 680 79 | 1027 80 | 776 81 | 648 82 | 4 83 | 703 84 | -------------------------------------------------------------------------------- /data/allArithLex/fold3.txt: -------------------------------------------------------------------------------- 1 | 267 2 | 311 3 | 695 4 | 870 5 | 207 6 | 1435 7 | 1111 8 | 756 9 | 250 10 | 181 11 | 867 12 | 150 13 | 137 14 | 1557 15 | 1563 16 | 1434 17 | 686 18 | 849 19 | 189 20 | 1421 21 | 327 22 | 1103 23 | 151 24 | 414 25 | 910 26 | 116 27 | 713 28 | 134 29 | 1114 30 | 1545 31 | 631 32 | 191 33 | 694 34 | 1323 35 | 668 36 | 170 37 | 268 38 | 432 39 | 865 40 | 774 41 | 858 42 | 869 43 | 224 44 | 164 45 | 812 46 | 481 47 | 628 48 | 632 49 | 597 50 | 261 51 | 748 52 | 1587 53 | 190 54 | 864 55 | 1495 56 | 987 57 | 1168 58 | 883 59 | 1452 60 | 182 61 | 1590 62 | 707 63 | 798 64 | 1547 65 | 586 66 | 832 67 | 256 68 | 251 69 | 105 70 | 565 71 | 1141 72 | 892 73 | 257 74 | 659 75 | 788 76 | 817 77 | 142 78 | 898 79 | 1442 80 | 310 81 | 1555 82 | 619 83 | 1051 84 | -------------------------------------------------------------------------------- /data/allArithLex/fold4.txt: -------------------------------------------------------------------------------- 1 | 197 2 | 1542 3 | 1235 4 | 888 5 | 1461 6 | 808 7 | 1 8 | 1567 9 | 1387 10 | 319 11 | 240 12 | 1427 13 | 132 14 | 244 15 | 1543 16 | 617 17 | 1546 18 | 1364 19 | 684 20 | 210 21 | 886 22 | 900 23 | 1370 24 | 796 25 | 557 26 | 316 27 | 162 28 | 759 29 | 1550 30 | 880 31 | 495 32 | 652 33 | 1047 34 | 451 35 | 1444 36 | 1225 37 | 1238 38 | 237 39 | 585 40 | 1117 41 | 364 42 | 265 43 | 890 44 | 464 45 | 1459 46 | 487 47 | 1571 48 | 596 49 | 234 50 | 445 51 | 613 52 | 577 53 | 643 54 | 136 55 | 11 56 | 391 57 | 1450 58 | 201 59 | 671 60 | 1175 61 | 1432 62 | 974 63 | 279 64 | 269 65 | 1569 66 | 1623 67 | 780 68 | 718 69 | 567 70 | 787 71 | 122 72 | 1564 73 | 1424 74 | 902 75 | 292 76 | 5 77 | 679 78 | 236 79 | 221 80 | 1112 81 | 131 82 | 1339 83 | 506 84 | -------------------------------------------------------------------------------- /data/allArithLex/info.txt: -------------------------------------------------------------------------------- 1 | Kushman: 2 | 3 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5783132530120482 MargCorrEquations: 0.5783132530120482 4 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.6746987951807228 MargCorrEquations: 0.6867469879518072 5 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.5662650602409639 MargCorrEquations: 0.5662650602409639 6 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.6626506024096386 MargCorrEquations: 0.6506024096385542 7 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7108433734939759 MargCorrEquations: 0.7108433734939759 8 | -------------------------------------------------------------------------------- /data/allArithTmpl/fold0.txt: -------------------------------------------------------------------------------- 1 | 60 2 | 654 3 | 899 4 | 1600 5 | 127 6 | 1034 7 | 14 8 | 1055 9 | 1220 10 | 347 11 | 1243 12 | 908 13 | 816 14 | 789 15 | 870 16 | 603 17 | 1026 18 | 136 19 | 507 20 | 322 21 | 75 22 | 1184 23 | 164 24 | 802 25 | 1101 26 | 155 27 | 1029 28 | 133 29 | 1226 30 | 563 31 | 628 32 | 638 33 | 65 34 | 36 35 | 342 36 | 971 37 | 1451 38 | 914 39 | 157 40 | 778 41 | 856 42 | 221 43 | 622 44 | 783 45 | 1459 46 | 56 47 | 227 48 | 969 49 | 903 50 | 615 51 | 58 52 | 178 53 | 107 54 | 633 55 | 1456 56 | 130 57 | 511 58 | 637 59 | 631 60 | 1110 61 | 134 62 | 634 63 | 104 64 | 526 65 | 1221 66 | 1427 67 | 965 68 | 799 69 | 210 70 | 824 71 | 1022 72 | 1453 73 | 987 74 | 120 75 | 781 76 | 1177 77 | 34 78 | 815 79 | 1108 80 | 1165 81 | 52 82 | 150 83 | 1166 84 | -------------------------------------------------------------------------------- /data/allArithTmpl/fold1.txt: -------------------------------------------------------------------------------- 1 | 792 2 | 110 3 | 69 4 | 531 5 | 26 6 | 19 7 | 1248 8 | 1233 9 | 220 10 | 1051 11 | 169 12 | 535 13 | 146 14 | 11 15 | 1434 16 | 143 17 | 598 18 | 1031 19 | 811 20 | 1458 21 | 505 22 | 821 23 | 906 24 | 122 25 | 1037 26 | 5 27 | 163 28 | 1112 29 | 926 30 | 43 31 | 162 32 | 796 33 | 156 34 | 913 35 | 12 36 | 1435 37 | 123 38 | 47 39 | 1235 40 | 20 41 | 1172 42 | 182 43 | 617 44 | 916 45 | 70 46 | 152 47 | 750 48 | 86 49 | 323 50 | 1436 51 | 149 52 | 1175 53 | 1610 54 | 1103 55 | 4 56 | 82 57 | 798 58 | 125 59 | 809 60 | 57 61 | 17 62 | 972 63 | 51 64 | 73 65 | 1461 66 | 753 67 | 214 68 | 1171 69 | 1181 70 | 1195 71 | 1169 72 | 167 73 | 1457 74 | 970 75 | 1236 76 | 788 77 | 980 78 | 555 79 | 1114 80 | 138 81 | 986 82 | 943 83 | 774 84 | -------------------------------------------------------------------------------- /data/allArithTmpl/fold2.txt: -------------------------------------------------------------------------------- 1 | 562 2 | 559 3 | 962 4 | 350 5 | 855 6 | 1100 7 | 154 8 | 319 9 | 1446 10 | 1107 11 | 1188 12 | 48 13 | 1043 14 | 1460 15 | 1224 16 | 557 17 | 807 18 | 219 19 | 1426 20 | 776 21 | 976 22 | 751 23 | 653 24 | 806 25 | 153 26 | 158 27 | 534 28 | 1183 29 | 854 30 | 1105 31 | 966 32 | 995 33 | 54 34 | 814 35 | 1168 36 | 183 37 | 1227 38 | 97 39 | 773 40 | 145 41 | 217 42 | 827 43 | 1106 44 | 211 45 | 151 46 | 168 47 | 142 48 | 904 49 | 109 50 | 10 51 | 175 52 | 910 53 | 1027 54 | 801 55 | 812 56 | 822 57 | 632 58 | 797 59 | 137 60 | 1432 61 | 564 62 | 619 63 | 1231 64 | 25 65 | 614 66 | 1099 67 | 829 68 | 800 69 | 790 70 | 144 71 | 172 72 | 367 73 | 7 74 | 8 75 | 523 76 | 901 77 | 907 78 | 1444 79 | 1238 80 | 787 81 | 116 82 | 13 83 | 1452 84 | -------------------------------------------------------------------------------- /data/allArithTmpl/fold3.txt: -------------------------------------------------------------------------------- 1 | 124 2 | 3 3 | 791 4 | 1222 5 | 87 6 | 1174 7 | 105 8 | 1310 9 | 1202 10 | 747 11 | 1164 12 | 605 13 | 967 14 | 121 15 | 977 16 | 527 17 | 905 18 | 1447 19 | 343 20 | 161 21 | 1448 22 | 808 23 | 226 24 | 951 25 | 1028 26 | 1109 27 | 561 28 | 786 29 | 1225 30 | 516 31 | 16 32 | 777 33 | 782 34 | 179 35 | 1189 36 | 9 37 | 1024 38 | 1167 39 | 111 40 | 67 41 | 160 42 | 780 43 | 1025 44 | 900 45 | 229 46 | 1035 47 | 1466 48 | 1047 49 | 852 50 | 973 51 | 184 52 | 15 53 | 794 54 | 528 55 | 963 56 | 364 57 | 81 58 | 165 59 | 22 60 | 132 61 | 96 62 | 1030 63 | 978 64 | 88 65 | 775 66 | 530 67 | 1450 68 | 656 69 | 1223 70 | 1141 71 | 1111 72 | 1449 73 | 636 74 | 1120 75 | 106 76 | 644 77 | 968 78 | 141 79 | 31 80 | 224 81 | 1613 82 | 902 83 | 1228 84 | -------------------------------------------------------------------------------- /data/allArithTmpl/fold4.txt: -------------------------------------------------------------------------------- 1 | 1036 2 | 752 3 | 803 4 | 1098 5 | 225 6 | 964 7 | 1420 8 | 1468 9 | 1230 10 | 218 11 | 131 12 | 147 13 | 21 14 | 1102 15 | 1115 16 | 853 17 | 657 18 | 835 19 | 33 20 | 917 21 | 166 22 | 688 23 | 140 24 | 228 25 | 68 26 | 1433 27 | 779 28 | 2 29 | 216 30 | 1170 31 | 805 32 | 1163 33 | 828 34 | 230 35 | 135 36 | 212 37 | 363 38 | 213 39 | 820 40 | 784 41 | 1117 42 | 760 43 | 826 44 | 1 45 | 159 46 | 1229 47 | 934 48 | 1430 49 | 117 50 | 223 51 | 795 52 | 83 53 | 1023 54 | 506 55 | 1104 56 | 758 57 | 1219 58 | 215 59 | 831 60 | 29 61 | 6 62 | 148 63 | 804 64 | 78 65 | 772 66 | 108 67 | 1594 68 | 222 69 | 35 70 | 1462 71 | 1445 72 | 1039 73 | 974 74 | 139 75 | 949 76 | 730 77 | 1465 78 | 1247 79 | 171 80 | 119 81 | 1038 82 | 630 83 | 793 84 | -------------------------------------------------------------------------------- /data/allArithTmpl/info.txt: -------------------------------------------------------------------------------- 1 | Kushman: 2 | 3 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7108433734939759 MargCorrEquations: 0.6867469879518072 4 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7108433734939759 MargCorrEquations: 0.6987951807228916 5 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7108433734939759 MargCorrEquations: 0.6987951807228916 6 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7228915662650602 MargCorrEquations: 0.7228915662650602 7 | FINAL TOTAL TEST STATS: All Total MargCorrNumerical: 0.7349397590361446 MargCorrEquations: 0.6987951807228916 8 | -------------------------------------------------------------------------------- /data/handcrafted/fold.txt: -------------------------------------------------------------------------------- 1 | 100001 2 | 100002 3 | 100003 4 | 100004 5 | 100005 6 | 100006 7 | 100007 8 | 100008 9 | 100009 10 | 100010 11 | 100011 12 | 100012 13 | 100013 14 | 100014 15 | 100015 16 | 100016 17 | 100017 18 | 100018 19 | 100019 20 | 100020 21 | 100021 22 | -------------------------------------------------------------------------------- /data/patterns.txt: -------------------------------------------------------------------------------- 1 | // Addition patterns 2 | 3 | EXPR1 added EXPR2 --> EXPR1 + EXPR2 4 | EXPR1 add EXPR2 --> EXPR1 + EXPR2 5 | sum EXPR1 EXPR2 --> EXPR1 + EXPR2 REPEAT 6 | added EXPR1 EXPR2 --> EXPR1 + EXPR2 REPEAT 7 | add EXPR1 EXPR2 --> EXPR1 + EXPR2 REPEAT 8 | EXPR1 plus EXPR2 --> EXPR1 + EXPR2 9 | 10 | 11 | // Subtraction 12 | 13 | difference EXPR1 EXPR2 --> EXPR1 - EXPR2 14 | EXPR1 minus EXPR2 --> EXPR1 - EXPR2 15 | subtracted EXPR1 EXPR2 --> EXPR2 - EXPR1 16 | subtract EXPR1 EXPR2 --> EXPR2 - EXPR1 17 | EXPR1 subtracted EXPR2 --> EXPR2 + EXPR1 18 | EXPR1 subtract EXPR2 --> EXPR2 + EXPR1 19 | 20 | 21 | // Multiplication 22 | 23 | product EXPR1 EXPR2 --> EXPR1 * EXPR2 REPEAT 24 | multiplied EXPR1 EXPR2 --> EXPR1 * EXPR2 REPEAT 25 | multiply EXPR1 EXPR2 --> EXPR1 * EXPR2 REPEAT 26 | EXPR1 times EXPR2 --> EXPR1 * EXPR2 27 | EXPR1 multiplied EXPR2 --> EXPR1 * EXPR2 28 | EXPR1 multiply EXPR2 --> EXPR1 * EXPR2 29 | 30 | 31 | // Division 32 | 33 | divided EXPR1 EXPR2 --> EXPR1 / EXPR2 34 | divide EXPR1 EXPR2 --> EXPR1 / EXPR2 35 | EXPR1 divided EXPR2 --> EXPR1 / EXPR2 36 | EXPR1 fraction EXPR2 --> EXPR1 / EXPR2 37 | fraction EXPR1 EXPR2 --> EXPR2 / EXPR1 38 | EXPR1 percentage EXPR2 --> EXPR1 / EXPR2 39 | percentage EXPR1 EXPR2 --> EXPR2 / EXPR1 40 | ratio EXPR1 EXPR2 --> EXPR1 / EXPR2 41 | 42 | 43 | -------------------------------------------------------------------------------- /data/simple_interest/fold0.txt: -------------------------------------------------------------------------------- 1 | 1232856534 2 | 1232857116 3 | 1232856969 4 | 1344 5 | 1232857076 6 | 1232857271 7 | 1232856562 8 | 1232856943 9 | 1232857188 10 | 1232856574 11 | 1107 12 | 824 13 | 230 14 | 316 15 | 184 16 | 1232856908 17 | 1232856893 18 | 770 19 | 1232857267 20 | 1232857297 21 | 233 22 | 1232856538 23 | 1232856537 24 | 1232856512 25 | 262 26 | 1232856746 27 | 424 28 | 564 29 | 31 30 | 1232856707 31 | 855 32 | 260 33 | 659 34 | 1111 35 | 1232856812 36 | 1232856750 37 | 1232856419 38 | 1232856855 39 | 1232856575 40 | 650 41 | 1141 42 | 165 43 | 1232857013 44 | 1232856880 45 | 1232856571 46 | 1232856396 47 | 311 48 | 1555 49 | 1232856475 50 | 476 51 | 110 52 | 867 53 | 104 54 | 1232856832 55 | 1232856445 56 | 1232857005 57 | 1232856532 58 | 1375 59 | 1232856814 60 | 1232856418 61 | 1232856649 62 | 1232856444 63 | 116 64 | 796 65 | 962 66 | 1232856670 67 | 266 68 | 1232856783 69 | 1426 70 | 1027 71 | 786 72 | 1232856504 73 | 236 74 | 805 75 | 615 76 | 1232856484 77 | 873 78 | 422 79 | 833 80 | 174 81 | 1232857160 82 | 417 83 | 1597 84 | 1232856657 85 | 1401 86 | 1232856868 87 | 119 88 | 1232856590 89 | 1557 90 | 1232856591 91 | 253 92 | 1232856496 93 | 678 94 | 1232856922 95 | 1232856776 96 | 1232856417 97 | 1232856584 98 | 638 99 | 1232857278 100 | 1232856939 101 | 1232857292 102 | 1356 103 | 1447 104 | 1232857004 105 | 1232856809 106 | 1232856992 107 | 1232856773 108 | 835 109 | 357 110 | 212 111 | 1232856395 112 | 1232856542 113 | 1232856632 114 | 711 115 | 1232856761 116 | 854 117 | 1379 118 | 654 119 | 1232856841 120 | 1232856454 121 | 943 122 | 1232857215 123 | 1232856791 124 | 1232856877 125 | 1232857187 126 | 1232856469 127 | 57 128 | 857 129 | 1232856823 130 | 1232856693 131 | 9 132 | 1232856866 133 | 1232856425 134 | 1232856548 135 | 695 136 | 755 137 | 1232856410 138 | 845 139 | 1100 140 | 789 141 | 1232857319 142 | 715 143 | 677 144 | 1232856423 145 | 1577 146 | 1232856796 147 | 1563 148 | 479 149 | 1232856610 150 | 862 151 | 874 152 | 893 153 | 1232856557 154 | 869 155 | 1232856979 156 | 203 157 | 117 158 | 1232856433 159 | 1232856732 160 | 1232856625 161 | 1184 162 | 841 163 | 1232856829 164 | 1232856631 165 | 1462 166 | 858 167 | 1232856972 168 | 838 169 | 1232856822 170 | 1112 171 | 976 172 | 1232856516 173 | 495 174 | 1232856955 175 | 1449 176 | 1576 177 | 250 178 | 1232856565 179 | 1232856651 180 | 223 181 | 780 182 | 1232857286 183 | 1578 184 | 207 185 | 892 186 | 856 187 | 859 188 | 1232856876 189 | 1314 190 | 452 191 | 1232856918 192 | 916 193 | 723 194 | 1232856644 195 | 1232856662 196 | 604 197 | 1610 198 | 651 199 | 717 200 | 1232856928 201 | 1174 202 | 774 203 | 995 204 | 719 205 | 1232857209 206 | 1232856795 207 | 1099 208 | 528 209 | 1232856757 210 | 1232856694 211 | 214 212 | 1232856886 213 | 1232856786 214 | 1026 215 | 843 216 | 1232857162 217 | 889 218 | 1570 219 | 1232856455 220 | 1232857192 221 | 347 222 | 1232856701 223 | 742 224 | 1232856619 225 | 718 226 | 1232856718 227 | 1232856461 228 | 1232856623 229 | 1232856793 230 | 1223 231 | 1232856719 232 | 1547 233 | 1232857006 234 | 197 235 | 794 236 | 1232856422 237 | 1232856681 238 | 764 239 | 1232856472 240 | 645 241 | 1232856553 242 | 231 243 | 761 244 | 1232856802 245 | 865 246 | 793 247 | 577 248 | 1232856736 249 | 132 250 | 1232856700 251 | 885 252 | 1232857021 253 | 870 254 | 853 255 | 1448 256 | 1232856854 257 | 974 258 | 1232856800 259 | 1232856847 260 | 273 261 | 737 262 | 1232856554 263 | 832 264 | 1232856787 265 | 900 266 | 373 267 | 1224 268 | 1608 269 | 668 270 | 1232857166 271 | 745 272 | 758 273 | 1232856769 274 | 527 275 | 1232856853 276 | 1600 277 | 123 278 | 1559 279 | 1232856630 280 | 1232857221 281 | 220 282 | 821 283 | 1341 284 | 1232856824 285 | 1232856617 286 | 1385 287 | 1232856774 288 | 1232856683 289 | 1232856560 290 | 1232856998 291 | 1232857295 292 | 969 293 | 1422 294 | 163 295 | 1562 296 | 722 297 | 1232856507 298 | 897 299 | 110000 300 | 110001 301 | 110003 302 | -------------------------------------------------------------------------------- /data/simple_interest/fold1.txt: -------------------------------------------------------------------------------- 1 | 1232856685 2 | 970 3 | 1232857030 4 | 1364 5 | 1232857143 6 | 1232856398 7 | 1232856947 8 | 1232856415 9 | 1181 10 | 1466 11 | 712 12 | 690 13 | 1232856826 14 | 1568 15 | 1232856446 16 | 1232857134 17 | 1232856439 18 | 189 19 | 1232856420 20 | 1 21 | 1232856613 22 | 1232857000 23 | 643 24 | 1232856672 25 | 159 26 | 210 27 | 1579 28 | 1620 29 | 1232857308 30 | 1232856723 31 | 561 32 | 1232856894 33 | 1232856536 34 | 173 35 | 1232856524 36 | 1232856655 37 | 804 38 | 171 39 | 138 40 | 1232857211 41 | 1333 42 | 181 43 | 1232856541 44 | 1232856957 45 | 1226 46 | 822 47 | 733 48 | 622 49 | 830 50 | 265 51 | 1232856690 52 | 1232857092 53 | 1232856535 54 | 8 55 | 878 56 | 1232856540 57 | 877 58 | 1232857132 59 | 209 60 | 1232856692 61 | 637 62 | 751 63 | 1232856573 64 | 692 65 | 1232856821 66 | 1435 67 | 1183 68 | 234 69 | 1232856948 70 | 1031 71 | 1232856759 72 | 814 73 | 684 74 | 1232856479 75 | 1232856830 76 | 633 77 | 2 78 | 1232856789 79 | 1232856891 80 | 1430 81 | 87 82 | 1232856860 83 | 179 84 | 259 85 | 726 86 | 1232857118 87 | 1320 88 | 597 89 | 1232856406 90 | 152 91 | 1030 92 | 852 93 | 828 94 | 1229 95 | 657 96 | 264 97 | 1232856734 98 | 1232856742 99 | 1594 100 | 1232856730 101 | 208 102 | 1232857140 103 | 1232857317 104 | 1232856820 105 | 1232856845 106 | 1232857313 107 | 1232857066 108 | 1232856778 109 | 1232857274 110 | 1232856394 111 | 1232857133 112 | 1232856952 113 | 144 114 | 65 115 | 451 116 | 183 117 | 1117 118 | 756 119 | 1232856815 120 | 1232856443 121 | 1453 122 | 1232856602 123 | 1232856594 124 | 891 125 | 847 126 | 1232856529 127 | 1232856582 128 | 1232856495 129 | 568 130 | 271 131 | 1232856691 132 | 1232856840 133 | 1232856848 134 | 1232856477 135 | 732 136 | 1232857086 137 | 1232856973 138 | 1550 139 | 364 140 | 407 141 | 1232856668 142 | 1232856764 143 | 1232856621 144 | 205 145 | 1232856744 146 | 1232856476 147 | 1232856457 148 | 1232857270 149 | 107 150 | 1232856715 151 | 708 152 | 1232856486 153 | 798 154 | 1043 155 | 1232856811 156 | 1232856449 157 | 1232856813 158 | 1456 159 | 689 160 | 1232856885 161 | 1542 162 | 1232856552 163 | 133 164 | 1232856456 165 | 879 166 | 1232856919 167 | 1232856661 168 | 261 169 | 1232857173 170 | 658 171 | 1232856788 172 | 876 173 | 605 174 | 1553 175 | 194 176 | 213 177 | 754 178 | 1233 179 | 290 180 | 1055 181 | 1232856834 182 | 1248 183 | 1545 184 | 1232856810 185 | 1232857206 186 | 1219 187 | 1232856720 188 | 1543 189 | 598 190 | 33 191 | 1232856735 192 | 1459 193 | 1569 194 | 1232857263 195 | 655 196 | 456 197 | 341 198 | 1232856675 199 | 403 200 | 1232856936 201 | 328 202 | 1029 203 | 1232857064 204 | 1232856578 205 | 1232857115 206 | 648 207 | 13 208 | 667 209 | 1232857220 210 | 1232856915 211 | 849 212 | 640 213 | 1420 214 | 1232857238 215 | 1340 216 | 1232856502 217 | 1232856580 218 | 652 219 | 1232856618 220 | 1232857198 221 | 457 222 | 1232856879 223 | 1232856963 224 | 811 225 | 585 226 | 1047 227 | 1232856603 228 | 783 229 | 562 230 | 776 231 | 1232856940 232 | 1232857088 233 | 1232857303 234 | 905 235 | 1590 236 | 840 237 | 1232856777 238 | 1232856450 239 | 1232857269 240 | 1463 241 | 1232856914 242 | 1232856870 243 | 1225 244 | 1202 245 | 1232857281 246 | 1232857043 247 | 1232856608 248 | 1232856645 249 | 1380 250 | 17 251 | 1232856664 252 | 720 253 | 390 254 | 242 255 | 1230 256 | 801 257 | 1232856587 258 | 1232857219 259 | 1332 260 | 1232856490 261 | 202 262 | 195 263 | 1232856852 264 | 1232856843 265 | 471 266 | 1232856702 267 | 221 268 | 1232856754 269 | 1232856652 270 | 280 271 | 67 272 | 788 273 | 6 274 | 1232856611 275 | 235 276 | 1232856673 277 | 1232856990 278 | 196 279 | 1232856614 280 | 1427 281 | 1232857084 282 | 1232856907 283 | 1232857100 284 | 973 285 | 51 286 | 1102 287 | 778 288 | 1232856470 289 | 644 290 | 1232856505 291 | 890 292 | 1232857148 293 | 1175 294 | 1232857125 295 | 1232856798 296 | 1432 297 | 1232856607 298 | 1232856660 299 | 110005 300 | 110006 301 | 110008 302 | -------------------------------------------------------------------------------- /data/simple_interest/fold2.txt: -------------------------------------------------------------------------------- 1 | 283 2 | 1417 3 | 125 4 | 323 5 | 1232856547 6 | 1232857231 7 | 795 8 | 1232856739 9 | 1232856762 10 | 1232856539 11 | 1403 12 | 1232856401 13 | 1232856404 14 | 1232856906 15 | 1232857222 16 | 1232856819 17 | 58 18 | 1232856464 19 | 779 20 | 1232856643 21 | 1232856481 22 | 567 23 | 1028 24 | 1445 25 | 964 26 | 588 27 | 1232856887 28 | 1232857217 29 | 1446 30 | 172 31 | 815 32 | 1232856659 33 | 506 34 | 1232856509 35 | 1232856709 36 | 1232857172 37 | 343 38 | 1101 39 | 837 40 | 1336 41 | 1232856954 42 | 1232856785 43 | 1232856447 44 | 1232857144 45 | 1232856970 46 | 1587 47 | 255 48 | 131 49 | 1232857232 50 | 1232856927 51 | 1171 52 | 647 53 | 200 54 | 753 55 | 190 56 | 1232856561 57 | 1232856498 58 | 1232856748 59 | 1232856583 60 | 707 61 | 1232856489 62 | 1601 63 | 1232857147 64 | 1232856432 65 | 481 66 | 1232857300 67 | 817 68 | 83 69 | 1232857001 70 | 1232857104 71 | 329 72 | 1232856863 73 | 1232856680 74 | 1024 75 | 685 76 | 3 77 | 1231 78 | 43 79 | 810 80 | 1235 81 | 1564 82 | 145 83 | 1232856424 84 | 1232856463 85 | 1232856862 86 | 688 87 | 665 88 | 1232857229 89 | 1560 90 | 1232856658 91 | 14 92 | 380 93 | 454 94 | 177 95 | 812 96 | 219 97 | 1232856589 98 | 1232856803 99 | 295 100 | 725 101 | 1232857326 102 | 1232856549 103 | 1458 104 | 1025 105 | 617 106 | 246 107 | 1232856865 108 | 1232856434 109 | 913 110 | 1232857208 111 | 1232856604 112 | 1236 113 | 109 114 | 1232856833 115 | 1232856522 116 | 319 117 | 1232856851 118 | 987 119 | 254 120 | 1232856520 121 | 1232856985 122 | 1232857177 123 | 1232856987 124 | 1232856448 125 | 1566 126 | 135 127 | 868 128 | 686 129 | 834 130 | 1232856752 131 | 1232856483 132 | 1232856600 133 | 60 134 | 191 135 | 860 136 | 894 137 | 760 138 | 603 139 | 29 140 | 1232856962 141 | 1232856510 142 | 150 143 | 1232856641 144 | 1232856838 145 | 180 146 | 11 147 | 392 148 | 1232856807 149 | 227 150 | 534 151 | 245 152 | 1572 153 | 709 154 | 1232856440 155 | 1232856755 156 | 1232856980 157 | 1232857128 158 | 252 159 | 555 160 | 1232856616 161 | 687 162 | 721 163 | 1232856799 164 | 399 165 | 215 166 | 1232857272 167 | 487 168 | 1232856556 169 | 1232856711 170 | 1548 171 | 1232856714 172 | 1232857171 173 | 1232856612 174 | 1232856902 175 | 967 176 | 1232856414 177 | 727 178 | 1232856794 179 | 972 180 | 1232856429 181 | 636 182 | 1387 183 | 1232856869 184 | 489 185 | 653 186 | 781 187 | 1232856559 188 | 646 189 | 1571 190 | 1232856881 191 | 1232856766 192 | 1232856753 193 | 1232856601 194 | 1023 195 | 1232857093 196 | 1169 197 | 1232857060 198 | 896 199 | 1232856867 200 | 78 201 | 809 202 | 1232856508 203 | 106 204 | 19 205 | 1232856503 206 | 842 207 | 1232856650 208 | 134 209 | 414 210 | 1232857186 211 | 268 212 | 1436 213 | 1232857165 214 | 1450 215 | 1035 216 | 917 217 | 1362 218 | 1232857289 219 | 1452 220 | 1232856808 221 | 1232856568 222 | 1232856646 223 | 803 224 | 675 225 | 1168 226 | 1461 227 | 54 228 | 1232856724 229 | 1232857034 230 | 884 231 | 1232856828 232 | 1232856466 233 | 1323 234 | 1232856816 235 | 671 236 | 1232856713 237 | 1232857018 238 | 563 239 | 127 240 | 1037 241 | 1232857259 242 | 747 243 | 275 244 | 140 245 | 1232856474 246 | 1232856493 247 | 1108 248 | 192 249 | 34 250 | 557 251 | 1232856687 252 | 816 253 | 147 254 | 1232856678 255 | 1614 256 | 249 257 | 1232857096 258 | 1232856689 259 | 97 260 | 1464 261 | 1177 262 | 1195 263 | 1232856756 264 | 1232856674 265 | 1232856895 266 | 148 267 | 1232856430 268 | 1232856965 269 | 1220 270 | 1232857150 271 | 882 272 | 634 273 | 1232856827 274 | 239 275 | 289 276 | 1232856857 277 | 1232856453 278 | 839 279 | 1232857131 280 | 1164 281 | 240 282 | 310 283 | 1370 284 | 1232856763 285 | 1232857122 286 | 1232856858 287 | 1232856784 288 | 572 289 | 1232857007 290 | 530 291 | 1307 292 | 1232856779 293 | 1232857167 294 | 807 295 | 886 296 | 1232856564 297 | 829 298 | 1376 299 | 806 300 | 110010 301 | 110012 302 | 110016 303 | -------------------------------------------------------------------------------- /data/simple_interest/fold3.txt: -------------------------------------------------------------------------------- 1 | 1287 2 | 1232856576 3 | 703 4 | 1232856491 5 | 827 6 | 1232856478 7 | 1232856403 8 | 1232856551 9 | 170 10 | 168 11 | 300 12 | 1232856792 13 | 257 14 | 1232856772 15 | 819 16 | 791 17 | 1556 18 | 746 19 | 1232856721 20 | 385 21 | 297 22 | 1232857288 23 | 1232857099 24 | 1232857241 25 | 792 26 | 1460 27 | 1232857194 28 | 1232856725 29 | 1232856624 30 | 327 31 | 1232857158 32 | 1232856882 33 | 965 34 | 1232856530 35 | 901 36 | 1232857023 37 | 1232856733 38 | 1232856804 39 | 237 40 | 846 41 | 1232856480 42 | 1232856780 43 | 282 44 | 151 45 | 1232857252 46 | 121 47 | 728 48 | 1232856775 49 | 1329 50 | 662 51 | 596 52 | 228 53 | 1232857170 54 | 1232856910 55 | 1232856405 56 | 1232856408 57 | 850 58 | 1232856818 59 | 825 60 | 1232856716 61 | 949 62 | 1232856737 63 | 1232856978 64 | 291 65 | 1232857102 66 | 363 67 | 887 68 | 1232857168 69 | 784 70 | 35 71 | 1444 72 | 787 73 | 1232856442 74 | 1247 75 | 130 76 | 393 77 | 1232856699 78 | 1232856566 79 | 1232857235 80 | 157 81 | 826 82 | 1603 83 | 217 84 | 206 85 | 86 86 | 775 87 | 1232856676 88 | 696 89 | 1232857197 90 | 716 91 | 679 92 | 1232856467 93 | 926 94 | 1232857183 95 | 844 96 | 1232856667 97 | 1232857045 98 | 1228 99 | 1232857212 100 | 1232856518 101 | 1232856545 102 | 1232856482 103 | 1232856872 104 | 169 105 | 1189 106 | 1232856654 107 | 238 108 | 274 109 | 899 110 | 1232856781 111 | 1232856703 112 | 702 113 | 188 114 | 224 115 | 1232857119 116 | 52 117 | 279 118 | 1339 119 | 198 120 | 762 121 | 1232856642 122 | 1232857253 123 | 418 124 | 614 125 | 1232856546 126 | 1549 127 | 1232856740 128 | 226 129 | 872 130 | 1232856558 131 | 1232856435 132 | 105 133 | 1552 134 | 124 135 | 1232856638 136 | 759 137 | 1232856899 138 | 88 139 | 1232856890 140 | 730 141 | 1232856609 142 | 963 143 | 175 144 | 1167 145 | 335 146 | 1232856629 147 | 1232856704 148 | 559 149 | 1232856679 150 | 1232856519 151 | 682 152 | 1434 153 | 898 154 | 1232856817 155 | 586 156 | 649 157 | 683 158 | 632 159 | 968 160 | 440 161 | 1114 162 | 36 163 | 864 164 | 1561 165 | 293 166 | 1232856569 167 | 1232856760 168 | 934 169 | 1232857094 170 | 1232857273 171 | 977 172 | 292 173 | 48 174 | 1232856517 175 | 1232857226 176 | 1232856665 177 | 1468 178 | 740 179 | 251 180 | 1232857268 181 | 1232856677 182 | 1232856525 183 | 1232856555 184 | 162 185 | 802 186 | 1232856705 187 | 1232856492 188 | 526 189 | 47 190 | 1335 191 | 641 192 | 1232856671 193 | 978 194 | 1165 195 | 1465 196 | 1232856758 197 | 1232856622 198 | 666 199 | 1232856639 200 | 904 201 | 426 202 | 565 203 | 1232857330 204 | 1120 205 | 1232856592 206 | 56 207 | 693 208 | 741 209 | 871 210 | 1232856397 211 | 587 212 | 1222 213 | 391 214 | 1232856636 215 | 301 216 | 881 217 | 531 218 | 1623 219 | 1232856634 220 | 269 221 | 880 222 | 137 223 | 68 224 | 1232856988 225 | 222 226 | 1588 227 | 1232856797 228 | 1544 229 | 1232856637 230 | 744 231 | 1188 232 | 748 233 | 1232856666 234 | 1036 235 | 516 236 | 158 237 | 749 238 | 1310 239 | 149 240 | 768 241 | 820 242 | 731 243 | 1232856452 244 | 1232856501 245 | 1558 246 | 1232857318 247 | 1227 248 | 1232857223 249 | 697 250 | 1232856462 251 | 218 252 | 1574 253 | 1421 254 | 906 255 | 1457 256 | 178 257 | 1232856751 258 | 1232856837 259 | 122 260 | 1163 261 | 317 262 | 1232857049 263 | 1575 264 | 229 265 | 143 266 | 394 267 | 1232857025 268 | 447 269 | 971 270 | 1232856459 271 | 1232856697 272 | 674 273 | 12 274 | 777 275 | 7 276 | 167 277 | 769 278 | 1551 279 | 146 280 | 25 281 | 773 282 | 312 283 | 267 284 | 706 285 | 1232856577 286 | 1232856620 287 | 1232856904 288 | 1232857195 289 | 15 290 | 1232856451 291 | 1232856770 292 | 1232856912 293 | 1232856765 294 | 1232856626 295 | 1232856959 296 | 1051 297 | 1166 298 | 176 299 | 110017 300 | 110024 301 | 110031 302 | -------------------------------------------------------------------------------- /data/simple_interest/fold4.txt: -------------------------------------------------------------------------------- 1 | 1232856597 2 | 757 3 | 1232857298 4 | 642 5 | 1232856741 6 | 1232856884 7 | 20 8 | 153 9 | 164 10 | 507 11 | 445 12 | 1232856615 13 | 523 14 | 910 15 | 694 16 | 1232856767 17 | 73 18 | 782 19 | 1232856412 20 | 425 21 | 75 22 | 672 23 | 480 24 | 823 25 | 156 26 | 1232856806 27 | 1232856983 28 | 861 29 | 1232856953 30 | 980 31 | 1232856500 32 | 1232856585 33 | 613 34 | 161 35 | 836 36 | 1232857321 37 | 1232856835 38 | 1232856544 39 | 664 40 | 1232856892 41 | 225 42 | 1232856570 43 | 442 44 | 1232856421 45 | 1232856431 46 | 1232857182 47 | 1232856437 48 | 1232857123 49 | 1232857163 50 | 1232856698 51 | 1232856896 52 | 676 53 | 1232856581 54 | 799 55 | 1232856458 56 | 397 57 | 1232856426 58 | 1232856964 59 | 1232856506 60 | 1232856917 61 | 1232856921 62 | 713 63 | 1232856511 64 | 120 65 | 1451 66 | 1221 67 | 142 68 | 1232856871 69 | 1232857225 70 | 1232856441 71 | 182 72 | 851 73 | 848 74 | 1565 75 | 1232856738 76 | 771 77 | 1232856633 78 | 1034 79 | 241 80 | 154 81 | 1232856497 82 | 698 83 | 1232856402 84 | 1232856889 85 | 1022 86 | 367 87 | 201 88 | 1554 89 | 1232856929 90 | 111 91 | 1232856640 92 | 1232856400 93 | 136 94 | 735 95 | 1232857277 96 | 1106 97 | 1232857107 98 | 766 99 | 1232856695 100 | 1232856995 101 | 1232857230 102 | 1433 103 | 831 104 | 432 105 | 1589 106 | 1232857242 107 | 1232856935 108 | 248 109 | 166 110 | 1232856749 111 | 1541 112 | 1232856550 113 | 339 114 | 1103 115 | 1232857020 116 | 752 117 | 1232856839 118 | 1232856488 119 | 895 120 | 1232856706 121 | 1342 122 | 1115 123 | 1232857129 124 | 401 125 | 619 126 | 1232856727 127 | 247 128 | 1232856514 129 | 1567 130 | 1232856411 131 | 569 132 | 1232857085 133 | 10 134 | 1232856513 135 | 1243 136 | 1232856515 137 | 705 138 | 1232857106 139 | 1377 140 | 1232856663 141 | 350 142 | 1232856635 143 | 1546 144 | 1232856688 145 | 914 146 | 951 147 | 4 148 | 1232856627 149 | 1232856653 150 | 1232857037 151 | 438 152 | 96 153 | 1232856944 154 | 673 155 | 966 156 | 1232856932 157 | 704 158 | 1613 159 | 1039 160 | 808 161 | 663 162 | 1238 163 | 1038 164 | 700 165 | 1232856708 166 | 216 167 | 1232856428 168 | 1232856836 169 | 903 170 | 797 171 | 1232857331 172 | 1442 173 | 1232857257 174 | 1232856586 175 | 69 176 | 739 177 | 1105 178 | 1232856543 179 | 1232856521 180 | 464 181 | 26 182 | 883 183 | 1232856728 184 | 907 185 | 701 186 | 743 187 | 1232856471 188 | 1232856856 189 | 256 190 | 1232856977 191 | 1232856465 192 | 790 193 | 1232856913 194 | 16 195 | 1232857095 196 | 155 197 | 908 198 | 535 199 | 1232856528 200 | 211 201 | 1232856717 202 | 763 203 | 139 204 | 724 205 | 1581 206 | 670 207 | 628 208 | 1232856588 209 | 573 210 | 1232856743 211 | 1232856722 212 | 1232856647 213 | 1232856686 214 | 342 215 | 187 216 | 185 217 | 1495 218 | 1232856782 219 | 326 220 | 630 221 | 1582 222 | 1232856745 223 | 349 224 | 1232856712 225 | 1232857130 226 | 1232856427 227 | 631 228 | 660 229 | 1232856485 230 | 1232856473 231 | 1232856526 232 | 199 233 | 1232856438 234 | 1232856849 235 | 1232857114 236 | 22 237 | 21 238 | 1232857032 239 | 1232857239 240 | 1232856628 241 | 322 242 | 1232856499 243 | 772 244 | 1596 245 | 656 246 | 986 247 | 1232856567 248 | 1098 249 | 1573 250 | 669 251 | 888 252 | 1232856801 253 | 330 254 | 1232856460 255 | 1232856413 256 | 70 257 | 324 258 | 681 259 | 1104 260 | 750 261 | 800 262 | 1232856771 263 | 1232856656 264 | 108 265 | 1232856864 266 | 160 267 | 1110 268 | 1232856805 269 | 81 270 | 734 271 | 1232856888 272 | 1232857120 273 | 1232857080 274 | 1232857179 275 | 1232856533 276 | 1232856527 277 | 5 278 | 1109 279 | 258 280 | 82 281 | 1232857203 282 | 1232856407 283 | 505 284 | 1232856875 285 | 1232856598 286 | 186 287 | 1232857236 288 | 1424 289 | 244 290 | 1172 291 | 875 292 | 691 293 | 1170 294 | 902 295 | 1232856844 296 | 511 297 | 680 298 | 1232856468 299 | 141 300 | 110033 301 | 110039 302 | 110043 303 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | edu.illinois.cs.cogcomp 6 | arithmetic 7 | 0.0.1-SNAPSHOT 8 | jar 9 | 10 | arithmetic 11 | http://maven.apache.org 12 | 13 | 14 | 15 | CogcompSoftware 16 | CogcompSoftware 17 | http://cogcomp.cs.illinois.edu/m2repo/ 18 | 19 | 20 | 21 | UTF-8 22 | 23 | 24 | 25 | 26 | org.apache.xmlrpc 27 | xmlrpc-client 28 | 3.1.3 29 | 30 | 31 | org.apache.xmlrpc 32 | xmlrpc-server 33 | 3.1.3 34 | 35 | 36 | com.google.guava 37 | guava 38 | 18.0 39 | 40 | 41 | edu.illinois.cs.cogcomp 42 | illinois-nlp-pipeline 43 | 0.1.16 44 | compile 45 | 46 | 47 | edu.illinois.cs.cogcomp 48 | illinois-ner-model-ontonotes 49 | 50 | 51 | edu.illinois.cs.cogcomp 52 | illinois-srl 53 | 54 | 55 | 56 | 57 | commons-io 58 | commons-io 59 | 2.7 60 | 61 | 62 | edu.illinois.cs.cogcomp 63 | illinois-sl 64 | 1.3.6 65 | 66 | 67 | edu.illinois.cs.cogcomp 68 | big-data-utils 69 | 1.0.5 70 | 71 | 72 | javatools 73 | javatools 74 | 20120110 75 | 76 | 77 | edu.illinois.cs.cogcomp 78 | jwnl-prime 79 | 1.0.3 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | CogcompSoftware 97 | CogcompSoftware 98 | http://cogcomp.cs.illinois.edu/m2repo/ 99 | 100 | 101 | 102 | 103 | 104 | org.apache.maven.plugins 105 | maven-compiler-plugin 106 | 2.0.2 107 | 108 | 1.7 109 | 1.7 110 | 111 | 112 | 113 | org.apache.maven.plugins 114 | maven-source-plugin 115 | 2.2.1 116 | 117 | 118 | attach-sources 119 | 120 | jar 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | org.apache.maven.wagon 130 | wagon-ssh 131 | 2.4 132 | 133 | 134 | 135 | 136 | 137 | 138 | CogcompSoftware 139 | CogcompSoftware 140 | scp://bilbo.cs.uiuc.edu:/mounts/bilbo/disks/0/www/cogcomp/html/m2repo 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | mkdir -p models 2 | mkdir -p log 3 | java -cp target/classes/:target/dependency/* demo.Driver "$@" 4 | -------------------------------------------------------------------------------- /runAllExpts1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh run.sh --mode UnitDep --cv data/aggregate/fold0.txt data/aggregate/fold1.txt data/aggregate/fold2.txt data/aggregate/fold3.txt data/aggregate/fold4.txt --model_dir models1/ --print_mistakes > log/UnitDepAgg.out 4 | 5 | sh run.sh --mode UnitDep --cv data/aggregateLex/fold0.txt data/aggregateLex/fold1.txt data/aggregateLex/fold2.txt data/aggregateLex/fold3.txt data/aggregateLex/fold4.txt --model_dir models1/ --print_mistakes > log/UnitDepAggLex.out 6 | 7 | sh run.sh --mode UnitDep --cv data/aggregateTmpl/fold0.txt data/aggregateTmpl/fold1.txt data/aggregateTmpl/fold2.txt data/aggregateTmpl/fold3.txt data/aggregateTmpl/fold4.txt --model_dir models1/ --print_mistakes > log/UnitDepAggTmpl.out 8 | 9 | 10 | 11 | sh run.sh --mode LCA --cv data/aggregate/fold0.txt data/aggregate/fold1.txt data/aggregate/fold2.txt data/aggregate/fold3.txt data/aggregate/fold4.txt --model_dir models1/ --print_mistakes > log/LCAAgg.out 12 | 13 | sh run.sh --mode LCA --cv data/aggregateLex/fold0.txt data/aggregateLex/fold1.txt data/aggregateLex/fold2.txt data/aggregateLex/fold3.txt data/aggregateLex/fold4.txt --model_dir models1/ --print_mistakes > log/LCAAggLex.out 14 | 15 | sh run.sh --mode LCA --cv data/aggregateTmpl/fold0.txt data/aggregateTmpl/fold1.txt data/aggregateTmpl/fold2.txt data/aggregateTmpl/fold3.txt data/aggregateTmpl/fold4.txt --model_dir models1/ --print_mistakes > log/LCAAggTmpl.out 16 | 17 | 18 | 19 | sh run.sh --mode E2ELogic --cv data/aggregate/fold0.txt data/aggregate/fold1.txt data/aggregate/fold2.txt data/aggregate/fold3.txt data/aggregate/fold4.txt --model_dir models1/ --print_mistakes > log/E2ELogicAgg.out 20 | 21 | sh run.sh --mode E2ELogic --cv data/aggregateLex/fold0.txt data/aggregateLex/fold1.txt data/aggregateLex/fold2.txt data/aggregateLex/fold3.txt data/aggregateLex/fold4.txt --model_dir models1/ --print_mistakes > log/E2ELogicAggLex.out 22 | 23 | sh run.sh --mode E2ELogic --cv data/aggregateTmpl/fold0.txt data/aggregateTmpl/fold1.txt data/aggregateTmpl/fold2.txt data/aggregateTmpl/fold3.txt data/aggregateTmpl/fold4.txt --model_dir models1/ --print_mistakes > log/E2ELogicAggTmpl.out 24 | -------------------------------------------------------------------------------- /runAllExpts2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh run.sh --mode UnitDep --cv data/allArith/fold0.txt data/allArith/fold1.txt data/allArith/fold2.txt data/allArith/fold3.txt data/allArith/fold4.txt --model_dir models2/ --print_mistakes > log/UnitDepAll.out 4 | 5 | sh run.sh --mode UnitDep --cv data/allArithLex/fold0.txt data/allArithLex/fold1.txt data/allArithLex/fold2.txt data/allArithLex/fold3.txt data/allArithLex/fold4.txt --model_dir models2/ --print_mistakes > log/UnitDepAllLex.out 6 | 7 | sh run.sh --mode UnitDep --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/UnitDepAllTmpl.out 8 | 9 | sh run.sh --mode UnitDep --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/UnitDepPerturb.out 10 | 11 | 12 | 13 | sh run.sh --mode LCA --cv data/allArith/fold0.txt data/allArith/fold1.txt data/allArith/fold2.txt data/allArith/fold3.txt data/allArith/fold4.txt --model_dir models2/ --print_mistakes > log/LCAAll.out 14 | 15 | sh run.sh --mode LCA --cv data/allArithLex/fold0.txt data/allArithLex/fold1.txt data/allArithLex/fold2.txt data/allArithLex/fold3.txt data/allArithLex/fold4.txt --model_dir models2/ --print_mistakes > log/LCAAllLex.out 16 | 17 | sh run.sh --mode LCA --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/LCAAllTmpl.out 18 | 19 | sh run.sh --mode LCA --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/LCAPerturb.out 20 | 21 | 22 | 23 | sh run.sh --mode E2ELogic --cv data/allArith/fold0.txt data/allArith/fold1.txt data/allArith/fold2.txt data/allArith/fold3.txt data/allArith/fold4.txt --model_dir models2/ --print_mistakes > log/E2ELogicAll.out 24 | 25 | sh run.sh --mode E2ELogic --cv data/allArithLex/fold0.txt data/allArithLex/fold1.txt data/allArithLex/fold2.txt data/allArithLex/fold3.txt data/allArithLex/fold4.txt --model_dir models2/ --print_mistakes > log/E2ELogicAllLex.out 26 | 27 | sh run.sh --mode E2ELogic --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/E2ELogicAllTmpl.out 28 | 29 | sh run.sh --mode E2ELogic --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/E2ELogicPerturb.out 30 | -------------------------------------------------------------------------------- /runAllExpts3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh run.sh --data qAll.json --mode UnitDep --train data/perturb/old.txt --test data/handcrafted/fold.txt --model_dir models3/ --print_mistakes > log/UnitDepAllHandcraft.out 4 | 5 | sh run.sh --data qAll.json --mode UnitDep --train fAll.txt --test data/handcrafted/fold.txt --model_dir models3/ --print_mistakes > log/UnitDepAggHandcraft.out 6 | 7 | 8 | 9 | sh run.sh --data qAll.json --mode LCA --train data/perturb/old.txt --test data/handcrafted/fold.txt --model_dir models3/ --print_mistakes > log/LCAAllHandcraft.out 10 | 11 | sh run.sh --data qAll.json --mode LCA --train fAll.txt --test data/handcrafted/fold.txt --model_dir models3/ --print_mistakes > log/LCAAggHandcraft.out 12 | 13 | 14 | 15 | sh run.sh --data qAll.json --mode E2ELogic --train data/perturb/old.txt --test data/handcrafted/fold.txt --model_dir models3/ --print_mistakes > log/E2ELogicAllHandcraft.out 16 | 17 | sh run.sh --data qAll.json --mode E2ELogic --train fAll.txt --test data/handcrafted/fold.txt --model_dir models3/ --print_mistakes > log/E2ELogicAggHandcraft.out 18 | 19 | -------------------------------------------------------------------------------- /scripts/kushman/runSubhro.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh run.sh Main 0 4 | sh run.sh Main 1 5 | sh run.sh Main 2 6 | sh run.sh Main 3 7 | sh run.sh Main 4 8 | 9 | 10 | -------------------------------------------------------------------------------- /scripts/kushman/runSubhroAgg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cp config/config4.json config/config.json 4 | sh run.sh Main 0 5 | sh run.sh Main 1 6 | sh run.sh Main 2 7 | sh run.sh Main 3 8 | sh run.sh Main 4 9 | 10 | 11 | cp config/config5.json config/config.json 12 | sh run.sh Main 0 13 | sh run.sh Main 1 14 | sh run.sh Main 2 15 | sh run.sh Main 3 16 | sh run.sh Main 4 17 | 18 | 19 | 20 | cp config/config6.json config/config.json 21 | sh run.sh Main 0 22 | sh run.sh Main 1 23 | sh run.sh Main 2 24 | sh run.sh Main 3 25 | sh run.sh Main 4 26 | 27 | 28 | -------------------------------------------------------------------------------- /scripts/kushman/runSubhroAllArith.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cp config/config1.json config/config.json 4 | sh run.sh Main 0 5 | sh run.sh Main 1 6 | sh run.sh Main 2 7 | sh run.sh Main 3 8 | sh run.sh Main 4 9 | 10 | 11 | cp config/config2.json config/config.json 12 | sh run.sh Main 0 13 | sh run.sh Main 1 14 | sh run.sh Main 2 15 | sh run.sh Main 3 16 | sh run.sh Main 4 17 | 18 | 19 | 20 | cp config/config3.json config/config.json 21 | sh run.sh Main 0 22 | sh run.sh Main 1 23 | sh run.sh Main 2 24 | sh run.sh Main 3 25 | sh run.sh Main 4 26 | 27 | 28 | cp config/config9.json config/config.json 29 | sh run.sh Main 0 30 | -------------------------------------------------------------------------------- /scripts/kushman/runSubhroHandcraft.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cp config/config7.json config/config.json 4 | sh run.sh Main 0 5 | 6 | 7 | cp config/config8.json config/config.json 8 | sh run.sh Main 0 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/main/java/constraints/ConsDriver.java: -------------------------------------------------------------------------------- 1 | package constraints; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import pair.PairDriver; 7 | import rate.RateDriver; 8 | import relevance.RelDriver; 9 | import run.Annotations; 10 | import run.RunDriver; 11 | import structure.Problem; 12 | import utils.Folds; 13 | import utils.Params; 14 | import edu.illinois.cs.cogcomp.sl.core.SLModel; 15 | 16 | public class ConsDriver { 17 | 18 | public static double doTest(List testProblems, int id) throws Exception { 19 | SLModel relModel = SLModel.loadModel(Params.modelDir+Params.relPrefix+id+Params.modelSuffix); 20 | SLModel pairModel = SLModel.loadModel(Params.modelDir+Params.pairPrefix+id+Params.modelSuffix); 21 | SLModel runModel = null, rateModel = null; 22 | if(!Params.noUDG) { 23 | runModel = SLModel.loadModel(Params.modelDir + Params.runPrefix + id + Params.modelSuffix); 24 | rateModel = SLModel.loadModel(Params.modelDir + Params.ratePrefix + id + Params.modelSuffix); 25 | } 26 | return ConsInfSolver.constrainedInf(testProblems, relModel, pairModel, runModel, rateModel, false); 27 | } 28 | 29 | public static void tune(List validation, SLModel relModel, SLModel pairModel, 30 | SLModel runModel, SLModel rateModel) throws Exception { 31 | double vals[] = {0.01, 0.1, 1.0, 10.0, 100.0}; 32 | double bestRate = 0.0; 33 | double bestRun = 0.0; 34 | double bestRel = 0.0; 35 | double bestAcc = 0.0; 36 | if(Params.noUDG) { 37 | for(int c=0; c bestAcc) { 42 | bestAcc = acc; 43 | bestRel = ConsInfSolver.wRel; 44 | } 45 | } 46 | } else { 47 | for(int a=0; a bestAcc) { 56 | bestAcc = acc; 57 | bestRun = ConsInfSolver.wRun; 58 | bestRate = ConsInfSolver.wRate; 59 | bestRel = ConsInfSolver.wRel; 60 | } 61 | } 62 | } 63 | } 64 | } 65 | ConsInfSolver.wRate = bestRate; 66 | ConsInfSolver.wRun = bestRun; 67 | ConsInfSolver.wRel = bestRel; 68 | } 69 | 70 | public static void tune(List devProbs, int id) throws Exception { 71 | SLModel relModel = SLModel.loadModel(Params.modelDir+Params.relPrefix+id+Params.modelSuffix); 72 | SLModel pairModel = SLModel.loadModel(Params.modelDir+Params.pairPrefix+id+Params.modelSuffix); 73 | SLModel runModel = null, rateModel = null; 74 | if(!Params.noUDG) { 75 | runModel = SLModel.loadModel(Params.modelDir + Params.runPrefix + id + Params.modelSuffix); 76 | rateModel = SLModel.loadModel(Params.modelDir + Params.ratePrefix + id + Params.modelSuffix); 77 | } 78 | tune(devProbs, relModel, pairModel, runModel, rateModel); 79 | } 80 | 81 | public static void crossVal( 82 | List probs, List> foldIndices) throws Exception { 83 | double acc = 0.0; 84 | for(int i=0;i train = new ArrayList<>(); 86 | List test = new ArrayList<>(); 87 | for(int j=0; j probs, List trainIndices, 97 | List testIndices, int id) throws Exception { 98 | double validationFrac; 99 | System.out.println("Tuning ..."); 100 | validationFrac = 0.2; 101 | List> split = Folds.getDataSplit(probs, trainIndices, testIndices, validationFrac); 102 | System.out.println("Training Relevance model ... "); 103 | RelDriver.trainModel(Params.modelDir+Params.relPrefix+id+Params.modelSuffix, 104 | RelDriver.getSP(split.get(0))); 105 | System.out.println("Training Pair model ... "); 106 | PairDriver.trainModel(Params.modelDir+Params.pairPrefix+id+Params.modelSuffix, 107 | PairDriver.getSP(split.get(0))); 108 | if(!Params.noUDG) { 109 | System.out.println("Training Run model ... "); 110 | RunDriver.trainModel(Params.modelDir + Params.runPrefix + id + Params.modelSuffix, 111 | Annotations.getSP(split.get(0))); 112 | System.out.println("Training Rate model ... "); 113 | RateDriver.trainModel(Params.modelDir + Params.ratePrefix + id + Params.modelSuffix, 114 | RateDriver.getSP(split.get(0))); 115 | } 116 | tune(split.get(1), id); 117 | System.out.println("Tuned parameters"); 118 | System.out.println("wRate : "+ConsInfSolver.wRate); 119 | System.out.println("wRun : "+ConsInfSolver.wRun); 120 | System.out.println("wRel : "+ConsInfSolver.wRel); 121 | System.out.println("Retraining on all training data"); 122 | validationFrac = 0.0; 123 | split = Folds.getDataSplit(probs, trainIndices, testIndices, validationFrac); 124 | System.out.println("Training Relevance model ... "); 125 | RelDriver.trainModel(Params.modelDir+Params.relPrefix+id+Params.modelSuffix, 126 | RelDriver.getSP(split.get(0))); 127 | System.out.println("Training Pair model ... "); 128 | PairDriver.trainModel(Params.modelDir+Params.pairPrefix+id+Params.modelSuffix, 129 | PairDriver.getSP(split.get(0))); 130 | if(!Params.noUDG) { 131 | System.out.println("Training Run model ... "); 132 | RunDriver.trainModel(Params.modelDir + Params.runPrefix + id + Params.modelSuffix, 133 | Annotations.getSP(split.get(0))); 134 | System.out.println("Training Rate model ... "); 135 | RateDriver.trainModel(Params.modelDir + Params.ratePrefix + id + Params.modelSuffix, 136 | RateDriver.getSP(split.get(0))); 137 | } 138 | return doTest(split.get(2), id); 139 | } 140 | 141 | } 142 | -------------------------------------------------------------------------------- /src/main/java/constraints/Constraints.java: -------------------------------------------------------------------------------- 1 | package constraints; 2 | 3 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; 4 | import utils.Tools; 5 | 6 | public class Constraints { 7 | 8 | public static boolean isPositive(Double val) { 9 | if(val < 0.0001) return false; 10 | return true; 11 | } 12 | 13 | public static boolean isInteger(TextAnnotation ta, Double val) { 14 | if(ta.getText().contains("How many") || ta.getText().contains("how many")) { 15 | if(!Tools.safeEquals(val, val.intValue()*1.0) && 16 | !Tools.safeEquals(val, val.intValue()*1.0+1.0)) { 17 | return false; 18 | } 19 | } 20 | return true; 21 | } 22 | 23 | 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/constraints/GraphDriver.java: -------------------------------------------------------------------------------- 1 | package constraints; 2 | 3 | import edu.illinois.cs.cogcomp.sl.core.SLModel; 4 | import rate.RateDriver; 5 | import run.Annotations; 6 | import run.RunDriver; 7 | import structure.Problem; 8 | import utils.Folds; 9 | import utils.Params; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | public class GraphDriver { 15 | 16 | public static double doTest(List testProblems, int id) throws Exception { 17 | SLModel runModel = SLModel.loadModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix); 18 | SLModel rateModel = SLModel.loadModel(Params.modelDir+Params.ratePrefix+id+Params.modelSuffix); 19 | return GraphInfSolver.constrainedInf(testProblems, runModel, rateModel); 20 | } 21 | 22 | public static void tune(List validation, SLModel runModel, SLModel rateModel) 23 | throws Exception { 24 | double vals[] = {0.01, 0.1, 1.0, 10.0, 100.0}; 25 | double bestRun = 0.0; 26 | double bestAcc = 0.0; 27 | for(int a=0; a bestAcc) { 31 | bestAcc = acc; 32 | bestRun = GraphInfSolver.wRun; 33 | } 34 | } 35 | ConsInfSolver.wRun = bestRun; 36 | } 37 | 38 | public static void tune(List devProbs, int id) throws Exception { 39 | SLModel runModel = SLModel.loadModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix); 40 | SLModel rateModel = SLModel.loadModel(Params.modelDir+Params.ratePrefix+id+Params.modelSuffix); 41 | tune(devProbs, runModel, rateModel); 42 | } 43 | 44 | public static Double doTrainTest(List probs, List trainIndices, 45 | List testIndices, int id) throws Exception { 46 | System.out.println("Tuning ..."); 47 | Params.printMistakes = false; 48 | double validationFrac = 0.2; 49 | List> split = Folds.getDataSplit(probs, trainIndices, testIndices, validationFrac); 50 | System.out.println("Training Run model ... "); 51 | RunDriver.trainModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix, 52 | Annotations.getSP(split.get(0))); 53 | System.out.println("Training Rate model ... "); 54 | RateDriver.trainModel(Params.modelDir+Params.ratePrefix+id+Params.modelSuffix, 55 | RateDriver.getSP(split.get(0))); 56 | tune(split.get(1), id); 57 | System.out.println("Tuned parameters"); 58 | System.out.println("wRun : "+GraphInfSolver.wRun); 59 | System.out.println("Retraining on all training data"); 60 | validationFrac = 0.0; 61 | split = Folds.getDataSplit(probs, trainIndices, testIndices, validationFrac); 62 | System.out.println("Training Run model ... "); 63 | RunDriver.trainModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix, 64 | Annotations.getSP(split.get(0))); 65 | System.out.println("Training Rate model ... "); 66 | RateDriver.trainModel(Params.modelDir+Params.ratePrefix+id+Params.modelSuffix, 67 | RateDriver.getSP(split.get(0))); 68 | return doTest(split.get(2), id); 69 | } 70 | 71 | public static void crossVal( 72 | List probs, List> foldIndices) throws Exception { 73 | double acc = 0.0; 74 | for(int i=0;i train = new ArrayList<>(); 76 | List test = new ArrayList<>(); 77 | for(int j=0; j features = getFeatures(x, y); 28 | return FeatGen.getFeatureVectorFromListString(features, lm); 29 | } 30 | 31 | public static List getFeatures(CorefX x, CorefY y) { 32 | List features = new ArrayList<>(); 33 | try { 34 | features.addAll(logic.LogicFeatGen.getKeyFeatures( 35 | x, 36 | x.schema.get(x.quantIndex1), 37 | x.schema.get(x.quantIndex2), 38 | x.questionSchema, 39 | x.infType, 40 | y.key)); 41 | } catch (Exception e) { 42 | // System.out.println("Exception in "+x.text); 43 | } 44 | features.addAll(FeatGen.getConjunctions(features)); 45 | return features; 46 | } 47 | 48 | 49 | 50 | } 51 | 52 | -------------------------------------------------------------------------------- /src/main/java/coref/CorefInfSolver.java: -------------------------------------------------------------------------------- 1 | package coref; 2 | 3 | import edu.illinois.cs.cogcomp.sl.core.AbstractInferenceSolver; 4 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 5 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 6 | import edu.illinois.cs.cogcomp.sl.util.WeightVector; 7 | import logic.Logic; 8 | import utils.Params; 9 | 10 | import java.io.Serializable; 11 | 12 | public class CorefInfSolver extends AbstractInferenceSolver implements Serializable { 13 | 14 | private static final long serialVersionUID = 5253748728743334706L; 15 | private CorefFeatGen featGen; 16 | 17 | public CorefInfSolver(CorefFeatGen featGen) throws Exception { 18 | this.featGen = featGen; 19 | } 20 | 21 | @Override 22 | public IStructure getBestStructure(WeightVector weight, IInstance ins) 23 | throws Exception { 24 | return getLossAugmentedBestStructure(weight, ins, null); 25 | } 26 | 27 | @Override 28 | public IStructure getLossAugmentedBestStructure(WeightVector weight, 29 | IInstance ins, IStructure goldStructure) throws Exception { 30 | CorefX x = (CorefX) ins; 31 | return getBestStructure(x, null, weight, false); 32 | } 33 | 34 | @Override 35 | public float getLoss(IInstance ins, IStructure gold, IStructure pred) { 36 | return CorefY.getLoss((CorefY)gold, (CorefY)pred); 37 | } 38 | 39 | public CorefY getBestStructure(CorefX x, 40 | CorefY gold, 41 | WeightVector weight, 42 | boolean labelCompletion) { 43 | double bestScore = -Double.MAX_VALUE; 44 | CorefY best = null; 45 | String label; 46 | for(String key : Logic.getRelevantKeys(x.infType)) { 47 | label = null; 48 | if(x.infType.startsWith("Verb")) { 49 | label = Logic.verb(x.tokens, x.schema.get(x.quantIndex1), 50 | x.schema.get(x.quantIndex2), key); 51 | } 52 | if(x.infType.startsWith("Partition")) { 53 | label = Logic.partition(key); 54 | } 55 | if(x.infType.startsWith("Math")) { 56 | label = Logic.math(x.infType, key); 57 | } 58 | if(x.infType.startsWith("Rate")) { 59 | label = Logic.unitDependency(x.infType, key); 60 | } 61 | if(label == null) continue; 62 | if(labelCompletion) { 63 | if(!label.equals(gold.label)) { 64 | continue; 65 | } 66 | } 67 | CorefY y = new CorefY(label, key); 68 | // List feats = CorefFeatGen.getFeatures(x, y); 69 | // if(feats.contains("BestOption")) return y; 70 | double score = weight.dotProduct(featGen.getFeatureVector(x, y)); 71 | if (bestScore < score) { 72 | best = y; 73 | bestScore = score; 74 | } 75 | } 76 | return best; 77 | } 78 | 79 | } -------------------------------------------------------------------------------- /src/main/java/coref/CorefX.java: -------------------------------------------------------------------------------- 1 | package coref; 2 | 3 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 4 | import structure.StanfordProblem; 5 | 6 | public class CorefX extends logic.LogicX implements IInstance { 7 | 8 | public int quantIndex1; 9 | public int quantIndex2; 10 | public String infType; 11 | 12 | public CorefX(StanfordProblem prob, int quantIndex1, int quantIndex2, String infType) { 13 | super(prob); 14 | this.quantIndex1 = quantIndex1; 15 | this.quantIndex2 = quantIndex2; 16 | this.infType = infType; 17 | } 18 | 19 | public CorefX(logic.LogicX prob, int quantIndex1, int quantIndex2, String infType) { 20 | super(prob); 21 | this.quantIndex1 = quantIndex1; 22 | this.quantIndex2 = quantIndex2; 23 | this.infType = infType; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/coref/CorefY.java: -------------------------------------------------------------------------------- 1 | package coref; 2 | 3 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 4 | 5 | public class CorefY implements IStructure { 6 | 7 | public String label; 8 | public String key; 9 | 10 | public CorefY(String label, String key) { 11 | this.label = label; 12 | this.key = key; 13 | } 14 | 15 | @Override 16 | public String toString() { 17 | return label+"_"+key; 18 | } 19 | 20 | public static float getLoss(CorefY gold, CorefY pred) { 21 | if(gold == null || pred == null) { 22 | return 1.0f; 23 | } 24 | if(gold.label.equals(pred.label)) { 25 | return 0.0f; 26 | } 27 | return 1.0f; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/demo/Demo.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | import constraints.ConsInfSolver; 4 | import edu.illinois.cs.cogcomp.sl.core.SLModel; 5 | import logic.LogicY; 6 | import structure.*; 7 | import utils.Params; 8 | import utils.Tools; 9 | 10 | 11 | public class Demo { 12 | 13 | public static SLModel relModel, pairModel, runModel, rateModel, corefModel, logicModel; 14 | 15 | public static void loadModels() throws Exception { 16 | if(Driver.mode.equals("LCA") || Driver.mode.equals("UnitDep")) { 17 | relModel = SLModel.loadModel(Params.modelDir+Params.relPrefix+"100"+Params.modelSuffix); 18 | pairModel = SLModel.loadModel(Params.modelDir+Params.pairPrefix+"100"+Params.modelSuffix); 19 | if(Driver.mode.equals("UnitDep")) { 20 | runModel = SLModel.loadModel(Params.modelDir+Params.runPrefix+"100"+Params.modelSuffix); 21 | rateModel = SLModel.loadModel(Params.modelDir+Params.ratePrefix+"100"+Params.modelSuffix); 22 | } 23 | } else { 24 | corefModel = SLModel.loadModel(Params.modelDir+Params.corefPrefix+"100"+Params.modelSuffix); 25 | logicModel = SLModel.loadModel(Params.modelDir+Params.logicPrefix+"100"+Params.modelSuffix); 26 | } 27 | } 28 | 29 | public static String answerMathProblemWithLCA(String input) throws Exception { 30 | Problem problem = new Problem(0, input.trim(), 0); 31 | problem.quantities = Tools.quantifier.getSpans(input.trim()); 32 | String newQues = ""; 33 | int index = 0; 34 | for(QuantSpan qs : problem.quantities) { 35 | newQues += problem.question.substring(index, qs.start); 36 | newQues += qs.val; 37 | index = qs.end; 38 | } 39 | newQues += problem.question.substring(index); 40 | problem = new Problem(0, newQues, 0); 41 | problem.quantities = Tools.quantifier.getSpans(problem.question); 42 | if(problem.quantities.size() < 2) return ""; 43 | problem.extractAnnotations(); 44 | Node node = ConsInfSolver.constrainedInf(problem, Demo.relModel, 45 | Demo.pairModel, Demo.runModel, Demo.rateModel); 46 | return node.toString()+" = "+node.getValue(); 47 | } 48 | 49 | public static String answerMathProblemWithLogic(String input) throws Exception { 50 | StanfordProblem problem = new StanfordProblem(0, input.trim(), 0); 51 | problem.quantities = Tools.quantifier.getSpans(input.trim()); 52 | String newQues = ""; 53 | int index = 0; 54 | for(QuantSpan qs : problem.quantities) { 55 | newQues += problem.question.substring(index, qs.start); 56 | newQues += qs.val; 57 | index = qs.end; 58 | } 59 | newQues += problem.question.substring(index); 60 | problem = new StanfordProblem(0, newQues, 0); 61 | problem.quantities = Tools.quantifier.getSpans(problem.question); 62 | if(problem.quantities.size() < 2) return ""; 63 | problem.extractAnnotations(); 64 | logic.LogicY y = (LogicY) logicModel.infSolver.getBestStructure( 65 | logicModel.wv, new logic.LogicX(problem)); 66 | return y.expr.toString()+" = "+y.expr.getValue(); 67 | } 68 | 69 | public static String answerQuestion(String input) throws Exception { 70 | if(Driver.mode.equals("LCA") || Driver.mode.equals("UnitDep")) { 71 | return answerMathProblemWithLCA(input); 72 | } 73 | if(Driver.mode.equals("E2ELogic")) { 74 | return answerMathProblemWithLogic(input); 75 | } 76 | return ""; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/demo/Server.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | import java.io.File; 4 | import java.util.Date; 5 | 6 | import org.apache.commons.io.FileUtils; 7 | import org.apache.xmlrpc.server.PropertyHandlerMapping; 8 | import org.apache.xmlrpc.server.XmlRpcServer; 9 | import org.apache.xmlrpc.server.XmlRpcServerConfigImpl; 10 | import org.apache.xmlrpc.webserver.WebServer; 11 | 12 | public class Server { 13 | 14 | public static String logFile = "/shared/trollope/sroy9/arithmetic_demo/log.txt"; 15 | 16 | public String rain(String input) throws Exception { 17 | try { 18 | if(input.trim().equals("")) return ""; 19 | FileUtils.writeStringToFile(new File(logFile), 20 | "\n"+new Date()+" : Query : "+input+"\n", true); 21 | String answer = Demo.answerQuestion(input); 22 | FileUtils.writeStringToFile(new File(logFile), 23 | new Date()+" : Returned : "+answer+"\n\n", true); 24 | return answer; 25 | } catch (Exception e) { 26 | e.printStackTrace(); 27 | return "Sorry, we could not solve it"; 28 | } 29 | } 30 | 31 | public static void startServer(int portNumber) { 32 | try { 33 | System.out.println("Attempting to start XML-RPC Server..."); 34 | WebServer webServer = new WebServer(portNumber); 35 | XmlRpcServer xmlRpcServer = webServer.getXmlRpcServer(); 36 | PropertyHandlerMapping phm = new PropertyHandlerMapping(); 37 | phm.addHandler("sample", Server.class); //new JavaServer().getClass()); 38 | xmlRpcServer.setHandlerMapping(phm); 39 | XmlRpcServerConfigImpl serverConfig = (XmlRpcServerConfigImpl) xmlRpcServer.getConfig(); 40 | serverConfig.setEnabledForExtensions(true); 41 | serverConfig.setContentLengthOptional(false); 42 | webServer.start(); 43 | System.out.println("Started successfully."); 44 | System.out.println("Accepting requests. (Halt program to stop.)"); 45 | } catch (Exception exception) { 46 | System.err.println("JavaServer: " + exception); 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /src/main/java/graph/GraphDriver.java: -------------------------------------------------------------------------------- 1 | package graph; 2 | 3 | import java.util.*; 4 | 5 | import structure.Problem; 6 | import utils.Folds; 7 | import utils.Params; 8 | import edu.illinois.cs.cogcomp.core.datastructures.Pair; 9 | import edu.illinois.cs.cogcomp.sl.core.SLModel; 10 | import edu.illinois.cs.cogcomp.sl.core.SLParameters; 11 | import edu.illinois.cs.cogcomp.sl.core.SLProblem; 12 | import edu.illinois.cs.cogcomp.sl.learner.Learner; 13 | import edu.illinois.cs.cogcomp.sl.learner.LearnerFactory; 14 | import edu.illinois.cs.cogcomp.sl.util.Lexiconer; 15 | 16 | public class GraphDriver { 17 | 18 | public static void crossVal(List probs, List> foldIndices) 19 | throws Exception { 20 | double acc1 = 0.0, acc2 = 0.0; 21 | for(int i=0;i train = new ArrayList<>(); 23 | List test = new ArrayList<>(); 24 | for(int j=0; j pair = doTrainTest(probs, train, test, i); 29 | acc1 += pair.getFirst(); 30 | acc2 += pair.getSecond(); 31 | } 32 | System.out.println("CV : " + (acc1/foldIndices.size()) + " " + (acc2/foldIndices.size())); 33 | } 34 | 35 | public static Pair doTrainTest(List probs, List trainIndices, 36 | List testIndices, int id) throws Exception { 37 | List> split = Folds.getDataSplit(probs, trainIndices, testIndices, 0.0); 38 | List trainProbs = split.get(0); 39 | List testProbs = split.get(2); 40 | SLProblem train = getSP(trainProbs); 41 | SLProblem test = getSP(testProbs); 42 | System.out.println("Train : "+train.instanceList.size()+" Test : "+test.instanceList.size()); 43 | trainModel(Params.modelDir+Params.graphPrefix+id+Params.modelSuffix, train); 44 | return testModel(Params.modelDir+Params.graphPrefix+id+Params.modelSuffix, test); 45 | } 46 | 47 | public static SLProblem getSP(List problemList) throws Exception{ 48 | SLProblem problem = new SLProblem(); 49 | for(Problem prob : problemList){ 50 | List indices = constraints.GraphInfSolver.createRelevantQuantIndexList(prob); 51 | GraphX x = new GraphX(prob, indices); 52 | GraphY y = new GraphY(constraints.GraphInfSolver.getGoldLabels(prob)); 53 | problem.addExample(x, y); 54 | } 55 | return problem; 56 | } 57 | 58 | public static Pair testModel(String modelPath, SLProblem sp) 59 | throws Exception { 60 | SLModel model = SLModel.loadModel(modelPath); 61 | Set incorrect = new HashSet<>(); 62 | Set total = new HashSet<>(); 63 | double acc = 0.0; 64 | for (int i = 0; i < sp.instanceList.size(); i++) { 65 | GraphX prob = (GraphX) sp.instanceList.get(i); 66 | GraphY gold = (GraphY) sp.goldStructureList.get(i); 67 | GraphY pred = (GraphY) model.infSolver.getBestStructure(model.wv, prob); 68 | total.add(prob.problemId); 69 | boolean correct =false; 70 | if(GraphY.getLoss(gold, pred) < 0.0001) { 71 | acc += 1; 72 | correct = true; 73 | } else { 74 | incorrect.add(prob.problemId); 75 | } 76 | if((correct && Params.printCorrect) || 77 | (!correct && Params.printMistakes)){ 78 | System.out.println(prob.problemId+" : "+prob.ta.getText()); 79 | System.out.println(); 80 | System.out.println("Schema : "+prob.schema); 81 | System.out.println(); 82 | System.out.println("Quantities : "+prob.quantities); 83 | System.out.println("Gold : "+gold); 84 | System.out.println("Pred : "+pred); 85 | System.out.println(); 86 | } 87 | } 88 | System.out.println("Accuracy : = " + acc + " / " + sp.instanceList.size() 89 | + " = " + (acc/sp.instanceList.size())); 90 | System.out.println("Strict Accuracy : ="+ (1-1.0*incorrect.size()/total.size())); 91 | return new Pair<>(acc/sp.instanceList.size(), 92 | 1-1.0*incorrect.size()/total.size()); 93 | } 94 | 95 | public static void trainModel(String modelPath, SLProblem train) throws Exception { 96 | SLModel model = new SLModel(); 97 | Lexiconer lm = new Lexiconer(); 98 | lm.setAllowNewFeatures(true); 99 | model.lm = lm; 100 | GraphFeatGen fg = new GraphFeatGen(lm); 101 | model.featureGenerator = fg; 102 | model.infSolver = new GraphInfSolver(fg); 103 | SLParameters para = new SLParameters(); 104 | para.loadConfigFile(Params.spConfigFile); 105 | para.MAX_NUM_ITER = 5; 106 | Learner learner = LearnerFactory.getLearner(model.infSolver, fg, para); 107 | model.wv = learner.train(train); 108 | lm.setAllowNewFeatures(false); 109 | model.saveModel(modelPath); 110 | } 111 | 112 | } -------------------------------------------------------------------------------- /src/main/java/graph/GraphFeatGen.java: -------------------------------------------------------------------------------- 1 | package graph; 2 | 3 | import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator; 4 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 5 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 6 | import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; 7 | import edu.illinois.cs.cogcomp.sl.util.Lexiconer; 8 | import rate.RateFeatGen; 9 | import rate.RateX; 10 | import run.RunFeatGen; 11 | import run.RunX; 12 | import utils.FeatGen; 13 | 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | 17 | public class GraphFeatGen extends AbstractFeatureGenerator { 18 | 19 | public Lexiconer lm = null; 20 | 21 | public GraphFeatGen(Lexiconer lm) { 22 | this.lm = lm; 23 | } 24 | 25 | @Override 26 | public IFeatureVector getFeatureVector(IInstance iInstance, IStructure iStructure) { 27 | GraphX x = (GraphX) iInstance; 28 | GraphY y = (GraphY) iStructure; 29 | return FeatGen.getFeatureVectorFromListString(getFeatures(x, y), lm); 30 | } 31 | 32 | public IFeatureVector getRateFeatureVector( 33 | GraphX x, int relevantQuantIndex, String label) { 34 | return FeatGen.getFeatureVectorFromListString( 35 | getRateFeatures(x, relevantQuantIndex, label), lm); 36 | } 37 | 38 | public IFeatureVector getRunFeatureVector( 39 | GraphX x, int relevantQuantIndex1, int relevantQuantIndex2, String label) { 40 | return FeatGen.getFeatureVectorFromListString( 41 | getRunFeatures(x, relevantQuantIndex1, relevantQuantIndex2, label), lm); 42 | } 43 | 44 | public static List getFeatures(GraphX x, GraphY y) { 45 | List feats = new ArrayList<>(); 46 | int n = x.relevantQuantIndices.size(); 47 | assert y.labels.size() == (n + n*(n-1)/2); 48 | for(int i=0; i getRateFeatures( 62 | GraphX x, int relevantQuantIndex, String label) { 63 | List feats = new ArrayList<>(); 64 | feats.addAll(RateFeatGen.getFeatures(new RateX( 65 | x, x.relevantQuantIndices.get(relevantQuantIndex)), label)); 66 | return feats; 67 | } 68 | 69 | public static List getRunFeatures( 70 | GraphX x, int relevantQuantIndex1, int relevantQuantIndex2, String label) { 71 | List feats = new ArrayList<>(); 72 | feats.addAll(RunFeatGen.getFeatures(new RunX( 73 | x, x.relevantQuantIndices.get(relevantQuantIndex1), 74 | x.relevantQuantIndices.get(relevantQuantIndex2)), label)); 75 | return feats; 76 | } 77 | 78 | 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/graph/GraphInfSolver.java: -------------------------------------------------------------------------------- 1 | package graph; 2 | 3 | import com.google.common.collect.MinMaxPriorityQueue; 4 | import edu.illinois.cs.cogcomp.core.datastructures.Pair; 5 | import edu.illinois.cs.cogcomp.sl.core.AbstractInferenceSolver; 6 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 7 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 8 | import edu.illinois.cs.cogcomp.sl.util.WeightVector; 9 | import structure.PairComparator; 10 | 11 | import java.util.ArrayList; 12 | import java.util.Arrays; 13 | import java.util.List; 14 | 15 | public class GraphInfSolver extends AbstractInferenceSolver { 16 | 17 | private GraphFeatGen featGen; 18 | 19 | public GraphInfSolver(GraphFeatGen featGen) { 20 | this.featGen = featGen; 21 | } 22 | 23 | @Override 24 | public IStructure getBestStructure(WeightVector weightVector, IInstance iInstance) 25 | throws Exception { 26 | return getLossAugmentedBestStructure(weightVector, iInstance, null); 27 | } 28 | 29 | @Override 30 | public IStructure getLossAugmentedBestStructure( 31 | WeightVector weightVector, IInstance iInstance, IStructure iStructure) 32 | throws Exception { 33 | boolean useConstraints = true; 34 | GraphX x = (GraphX) iInstance; 35 | PairComparator> nodePairComparator = 36 | new PairComparator>() {}; 37 | MinMaxPriorityQueue, Double>> beam1 = 38 | MinMaxPriorityQueue.orderedBy(nodePairComparator) 39 | .maximumSize(200).create(); 40 | MinMaxPriorityQueue, Double>> beam2 = 41 | MinMaxPriorityQueue.orderedBy(nodePairComparator) 42 | .maximumSize(200).create(); 43 | int n = x.relevantQuantIndices.size(); 44 | List init = new ArrayList<>(); 45 | for(int i=0; i labels = new ArrayList<>(); 49 | labels.addAll(init); 50 | beam1.add(new Pair<>(labels, 0.0)); 51 | for(int i=0; i(); 53 | labels.addAll(init); 54 | labels.set(i, "RATE"); 55 | beam1.add(new Pair<>(labels, 1.0*weightVector.dotProduct( 56 | featGen.getRateFeatureVector(x, i, "RATE")))); 57 | for(int j=i+1; j(); 59 | labels.addAll(init); 60 | labels.set(i, "RATE"); 61 | labels.set(j, "RATE"); 62 | beam1.add(new Pair<>(labels, 1.0*weightVector.dotProduct( 63 | featGen.getRunFeatureVector(x, i, j, "RATE")))); 64 | } 65 | } 66 | for(int i=0; i, Double> pair : beam1) { 72 | labels = new ArrayList<>(); 73 | labels.addAll(pair.getFirst()); 74 | labels.add(label); 75 | if(useConstraints && !constraints.GraphInfSolver.satisfyConstraints( 76 | labels.get(i), 77 | labels.get(j), 78 | label 79 | )) continue; 80 | beam2.add(new Pair<>(labels, 81 | pair.getSecond() + 1.0*weightVector.dotProduct( 82 | featGen.getRunFeatureVector(x, i, j, label)))); 83 | } 84 | } 85 | beam1.clear(); 86 | beam1.addAll(beam2); 87 | beam2.clear(); 88 | } 89 | } 90 | assert beam1.element().getFirst().size() == n+n*(n-1)/2; 91 | return new GraphY(beam1.element().getFirst()); 92 | } 93 | 94 | @Override 95 | public float getLoss(IInstance iInstance, IStructure iStructure, IStructure iStructure1) { 96 | return GraphY.getLoss((GraphY) iStructure, (GraphY) iStructure1); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/graph/GraphX.java: -------------------------------------------------------------------------------- 1 | package graph; 2 | 3 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 4 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; 5 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 6 | import structure.Problem; 7 | import structure.QuantSpan; 8 | import structure.Schema; 9 | 10 | import java.util.List; 11 | 12 | public class GraphX implements IInstance { 13 | 14 | public int problemId; 15 | public TextAnnotation ta; 16 | public List quantities; 17 | public List posTags; 18 | public List chunks; 19 | public List lemmas; 20 | public Schema schema; 21 | public List relevantQuantIndices; 22 | 23 | public GraphX(Problem prob, List relevantQuantIndices) { 24 | this.problemId = prob.id; 25 | this.ta = prob.ta; 26 | this.quantities = prob.quantities; 27 | this.posTags = prob.posTags; 28 | this.chunks = prob.chunks; 29 | this.schema = prob.schema; 30 | this.lemmas = prob.lemmas; 31 | this.relevantQuantIndices = relevantQuantIndices; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/graph/GraphY.java: -------------------------------------------------------------------------------- 1 | package graph; 2 | 3 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 4 | 5 | import java.util.Arrays; 6 | import java.util.List; 7 | 8 | public class GraphY implements IStructure { 9 | 10 | // Labels only for relevant quants, and for question 11 | // Order : Vertex label for each quantity, followed by vertex label of question, 12 | // next, edge labels for all edges (following order of for loop on the above 13 | // vertex order) 14 | public List labels; 15 | 16 | public GraphY(List labels) { 17 | this.labels = labels; 18 | } 19 | 20 | public static float getLoss(GraphY y1, GraphY y2) { 21 | if((""+ Arrays.asList(y1.labels)).equals((""+ Arrays.asList(y2.labels)))) { 22 | return 0.0f; 23 | } 24 | return 1.0f; 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return ""+Arrays.asList(labels); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/logic/Driver.java: -------------------------------------------------------------------------------- 1 | package logic; 2 | 3 | import coref.CorefDriver; 4 | import edu.illinois.cs.cogcomp.core.datastructures.Pair; 5 | import edu.illinois.cs.cogcomp.sl.core.SLProblem; 6 | import structure.StanfordProblem; 7 | import utils.Folds; 8 | import utils.Params; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | public class Driver { 14 | 15 | public static double doTrainTest(List probs, 16 | List trainIndices, 17 | List testIndices, 18 | int id) throws Exception { 19 | List> split = Folds.getDataSplitForStanford( 20 | probs, trainIndices, testIndices, 0.0); 21 | LogicDriver.useGoldRelevance = true; 22 | System.out.println("Training Coreference model ... "); 23 | CorefDriver.trainModel(Params.modelDir+Params.corefPrefix+id+Params.modelSuffix, 24 | CorefDriver.getSP(split.get(0), true)); 25 | System.out.println("Training Logic model ... "); 26 | LogicDriver.trainModel(Params.modelDir+Params.logicPrefix+id+Params.modelSuffix, 27 | LogicDriver.getSP(split.get(0), true), 28 | Params.modelDir+Params.corefPrefix+id+Params.modelSuffix); 29 | LogicDriver.useGoldRelevance = false; 30 | SLProblem test = new SLProblem(); 31 | for(StanfordProblem prob : split.get(2)) { 32 | LogicX x = new LogicX(prob); 33 | LogicY y = new LogicY(prob.expr); 34 | test.addExample(x, y); 35 | } 36 | Pair scores = LogicDriver.testModel( 37 | Params.modelDir+Params.logicPrefix+id+Params.modelSuffix, 38 | test); 39 | return scores.getSecond(); 40 | } 41 | 42 | public static void crossVal(List probs, 43 | List> foldIndices) throws Exception { 44 | double acc = 0.0; 45 | LogicDriver.useInfModel = true; 46 | for(int i=0; i train = new ArrayList<>(); 48 | List test = new ArrayList<>(); 49 | for(int j=0; j quantities; 23 | public List> tokens; 24 | public List dependencies; 25 | public List schema; 26 | public StanfordSchema questionSchema; 27 | public IntPair questionSpan; 28 | public Map, String> wordnetRelations; 29 | public Set relevantQuantIndices; 30 | 31 | public LogicX(StanfordProblem prob) { 32 | this.problemId = prob.id; 33 | this.text = prob.question; 34 | this.quantities = prob.quantities; 35 | this.tokens = prob.tokens; 36 | this.dependencies = prob.dependencies; 37 | this.schema = prob.schema; 38 | this.questionSchema = prob.questionSchema; 39 | this.questionSpan = StanfordProblem.getQuestionSpan( 40 | prob.tokens.get(questionSchema.sentId)); 41 | this.wordnetRelations = prob.wordnetRelations; 42 | this.relevantQuantIndices = new HashSet<>(); 43 | if(prob.expr != null) { 44 | for (Node leaf : prob.expr.getLeaves()) { 45 | relevantQuantIndices.add(leaf.quantIndex); 46 | } 47 | } 48 | } 49 | 50 | public LogicX(LogicX prob) { 51 | this.problemId = prob.problemId; 52 | this.text = prob.text; 53 | this.quantities = prob.quantities; 54 | this.tokens = prob.tokens; 55 | this.dependencies = prob.dependencies; 56 | this.schema = prob.schema; 57 | this.questionSchema = prob.questionSchema; 58 | this.questionSpan = StanfordProblem.getQuestionSpan( 59 | prob.tokens.get(questionSchema.sentId)); 60 | this.wordnetRelations = prob.wordnetRelations; 61 | this.relevantQuantIndices = prob.relevantQuantIndices; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/logic/Verbs.java: -------------------------------------------------------------------------------- 1 | package logic; 2 | 3 | import utils.Params; 4 | import utils.Tools; 5 | 6 | import java.io.*; 7 | import java.util.*; 8 | 9 | public class Verbs { 10 | 11 | public static Map vectors; 12 | 13 | static { 14 | try { 15 | System.out.println("Reading vectors for verbs ..."); 16 | vectors = readVectors(Params.vectorsFile); 17 | } catch (IOException e) { 18 | e.printStackTrace(); 19 | } 20 | } 21 | 22 | public static String verbCategory(String verb) { 23 | List state = Arrays.asList("be", "have", "own", "start", "worth", "remain"); 24 | List positive = Arrays.asList("get", "gain", "borrow", "receive", 25 | "take", "collect"); 26 | List negative = Arrays.asList("give", "lose", "lend", "spend", "pay", 27 | "share", "leave"); 28 | List construct = Arrays.asList( "add", "increase", "build", 29 | "score", "save", "win", "arrive", "fill"); 30 | List destroy = Arrays.asList("defeat", "destroy", "need", "eat", 31 | "decrease", "reduce", "remove", "use", "throw"); 32 | Map map = new HashMap<>(); 33 | map.put("STATE", -Double.MAX_VALUE); 34 | map.put("POSITIVE", -Double.MAX_VALUE); 35 | map.put("NEGATIVE", -Double.MAX_VALUE); 36 | map.put("CONSTRUCT", -Double.MAX_VALUE); 37 | map.put("DESTROY", -Double.MAX_VALUE); 38 | for(String s : state) { 39 | if(getVectorSim(verb, s) > map.get("STATE")) { 40 | map.put("STATE", getVectorSim(verb, s)); 41 | } 42 | } 43 | for(String s : positive) { 44 | if(getVectorSim(verb, s) > map.get("POSITIVE")) { 45 | map.put("POSITIVE", getVectorSim(verb, s)); 46 | } 47 | } 48 | for(String s : negative) { 49 | if(getVectorSim(verb, s) > map.get("NEGATIVE")) { 50 | map.put("NEGATIVE", getVectorSim(verb, s)); 51 | } 52 | } 53 | for(String s : construct) { 54 | if(getVectorSim(verb, s) > map.get("CONSTRUCT")) { 55 | map.put("CONSTRUCT", getVectorSim(verb, s)); 56 | } 57 | } 58 | for(String s : destroy) { 59 | if(getVectorSim(verb, s) > map.get("DESTROY")) { 60 | map.put("DESTROY", getVectorSim(verb, s)); 61 | } 62 | } 63 | return Tools.getKeyForMaxValue(map); 64 | } 65 | 66 | public static Map verbClassify(String verbLemma, List unit) { 67 | Map map = new HashMap<>(); 68 | map.put("STATE", 0.0); 69 | map.put("POSITIVE", 0.0); 70 | map.put("NEGATIVE", 0.0); 71 | map.put("CONSTRUCT", 0.0); 72 | map.put("DESTROY", 0.0); 73 | // Hard decision for now 74 | if (verbLemma.equals("buy") || verbLemma.equals("purchase")) { 75 | if (unit != null && (unit.contains("$") || 76 | unit.contains("dollar") || 77 | unit.contains("cent") || 78 | unit.contains("dime") || 79 | unit.contains("nickel"))) { 80 | map.put("NEGATIVE", 1.0); 81 | } else { 82 | map.put("POSITIVE", 1.0); 83 | } 84 | return map; 85 | } 86 | if (verbLemma.equals("sell") || verbLemma.equals("return")) { 87 | if (unit != null && (unit.contains("$") || 88 | unit.contains("dollar") || 89 | unit.contains("cent") || 90 | unit.contains("dime") || 91 | unit.contains("nickel"))) { 92 | map.put("POSITIVE", 1.0); 93 | } else { 94 | map.put("NEGATIVE", 1.0); 95 | } 96 | return map; 97 | } 98 | String vcc = verbCategory(verbLemma); 99 | map.put(vcc, 1.0); 100 | return map; 101 | } 102 | 103 | public static double getVectorSim(String word1, String word2) { 104 | // System.out.println("Running vector similarity on "+word1+" and "+word2); 105 | if (word1 == null || word2 == null) { 106 | return 0.0; 107 | } 108 | if(vectors.containsKey(word1) && vectors.containsKey(word2)) { 109 | double[] v1 = vectors.get(word1); 110 | double[] v2 = vectors.get(word2); 111 | double dot = 0.0, norm1 = 0.0, norm2 = 0.0; 112 | for (int i=0; i readVectors(String vectorFile) throws IOException { 125 | Map vectors = new HashMap<>(); 126 | BufferedReader br; 127 | String line; 128 | br = new BufferedReader(new FileReader(new File(vectorFile))); 129 | while((line = br.readLine()) != null) { 130 | String strArr[] = line.split(" "); 131 | String word = strArr[0].trim(); 132 | double d[] = new double[strArr.length-1]; 133 | for(int i=1; i probs, List> foldIndices) 20 | throws Exception { 21 | double acc1 = 0.0, acc2 = 0.0; 22 | for(int i=0;i train = new ArrayList<>(); 24 | List test = new ArrayList<>(); 25 | for(int j=0; j pair = doTrainTest(probs, train, test, i); 30 | acc1 += pair.getFirst(); 31 | acc2 += pair.getSecond(); 32 | } 33 | System.out.println("CV : " + (acc1/foldIndices.size()) + " " + (acc2/foldIndices.size())); 34 | } 35 | 36 | public static Pair doTrainTest(List probs, List trainIndices, 37 | List testIndices, int id) throws Exception { 38 | List> split = Folds.getDataSplit(probs, trainIndices, testIndices, 0.0); 39 | List trainProbs = split.get(0); 40 | List testProbs = split.get(2); 41 | SLProblem train = getSP(trainProbs); 42 | SLProblem test = getSP(testProbs); 43 | System.out.println("Train : "+train.instanceList.size()+" Test : "+test.instanceList.size()); 44 | trainModel(Params.modelDir+Params.pairPrefix+id+Params.modelSuffix, train); 45 | return testModel(Params.modelDir+Params.pairPrefix+id+Params.modelSuffix, test); 46 | } 47 | 48 | public static SLProblem getSP(List problemList) throws Exception{ 49 | SLProblem problem = new SLProblem(); 50 | for(Problem prob : problemList){ 51 | for(int i=0; i testModel(String modelPath, SLProblem sp) 66 | throws Exception { 67 | SLModel model = SLModel.loadModel(modelPath); 68 | Set incorrect = new HashSet<>(); 69 | Set total = new HashSet<>(); 70 | double acc = 0.0; 71 | for (int i = 0; i < sp.instanceList.size(); i++) { 72 | PairX prob = (PairX) sp.instanceList.get(i); 73 | PairY gold = (PairY) sp.goldStructureList.get(i); 74 | PairY pred = (PairY) model.infSolver.getBestStructure(model.wv, prob); 75 | total.add(prob.problemId); 76 | boolean correct = false; 77 | if(PairY.getLoss(gold, pred) < 0.0001) { 78 | acc += 1; 79 | correct = true; 80 | } else { 81 | incorrect.add(prob.problemId); 82 | } 83 | if((correct && Params.printCorrect) || 84 | (!correct && Params.printMistakes)){ 85 | System.out.println(prob.problemId+" : "+prob.ta.getText()); 86 | System.out.println(); 87 | System.out.println("Schema : "+prob.schema); 88 | System.out.println(); 89 | System.out.println("Quantities : "+prob.quantities); 90 | System.out.println("Quant of Interest: "+prob.quantIndex1+" "+prob.quantIndex2); 91 | System.out.println("Gold : "+gold); 92 | System.out.println("Pred : "+pred); 93 | System.out.println("Loss : "+PairY.getLoss(gold, pred)); 94 | System.out.println("Labels : "+Arrays.asList(getLabelsWithScores(prob, model))); 95 | System.out.println(); 96 | } 97 | } 98 | System.out.println("Accuracy : = " + acc + " / " + sp.instanceList.size() 99 | + " = " + (acc/sp.instanceList.size())); 100 | System.out.println("Strict Accuracy : ="+ (1-1.0*incorrect.size()/total.size())); 101 | return new Pair<>(acc/sp.instanceList.size(), 1-1.0*incorrect.size()/total.size()); 102 | } 103 | 104 | public static void trainModel(String modelPath, SLProblem train) 105 | throws Exception { 106 | SLModel model = new SLModel(); 107 | Lexiconer lm = new Lexiconer(); 108 | lm.setAllowNewFeatures(true); 109 | model.lm = lm; 110 | AbstractFeatureGenerator fg = new PairFeatGen(lm); 111 | model.featureGenerator = fg; 112 | model.infSolver = new PairInfSolver(fg); 113 | SLParameters para = new SLParameters(); 114 | para.loadConfigFile(Params.spConfigFile); 115 | para.MAX_NUM_ITER = 5; 116 | Learner learner = LearnerFactory.getLearner(model.infSolver, fg, para); 117 | model.wv = learner.train(train); 118 | lm.setAllowNewFeatures(false); 119 | model.saveModel(modelPath); 120 | } 121 | 122 | public static Map getLabelsWithScores(PairX prob, SLModel model) { 123 | List labels = Arrays.asList("ADD", "SUB", "MUL", "DIV", "SUB_REV", "DIV_REV"); 124 | Map labelsWithScores = new HashMap<>(); 125 | for(String label : labels) { 126 | labelsWithScores.put(prob.quantIndex1+"_"+prob.quantIndex2+"_"+label, 127 | 1.0*model.wv.dotProduct(model.featureGenerator.getFeatureVector( 128 | prob, 129 | new PairY(label)))); 130 | } 131 | return labelsWithScores; 132 | } 133 | } -------------------------------------------------------------------------------- /src/main/java/pair/PairInfSolver.java: -------------------------------------------------------------------------------- 1 | package pair; 2 | 3 | import java.io.Serializable; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator; 8 | import edu.illinois.cs.cogcomp.sl.core.AbstractInferenceSolver; 9 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 10 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 11 | import edu.illinois.cs.cogcomp.sl.util.WeightVector; 12 | 13 | public class PairInfSolver extends AbstractInferenceSolver implements 14 | Serializable { 15 | 16 | private static final long serialVersionUID = 5253748728743334706L; 17 | private AbstractFeatureGenerator featGen; 18 | 19 | public PairInfSolver(AbstractFeatureGenerator featGen) throws Exception { 20 | this.featGen = featGen; 21 | } 22 | 23 | @Override 24 | public IStructure getBestStructure(WeightVector weight, IInstance ins) 25 | throws Exception { 26 | return getLossAugmentedBestStructure(weight, ins, null); 27 | } 28 | 29 | @Override 30 | public IStructure getLossAugmentedBestStructure(WeightVector weight, 31 | IInstance ins, IStructure goldStructure) throws Exception { 32 | List labels = Arrays.asList("ADD", "SUB", "SUB_REV", "MUL", "DIV", "DIV_REV"); 33 | double bestScore = -Double.MAX_VALUE; 34 | PairY best = null; 35 | for(String label : labels) { 36 | double score = weight.dotProduct(featGen.getFeatureVector(ins, new PairY(label))); 37 | if(bestScore < score) { 38 | best = new PairY(label); 39 | bestScore = score; 40 | } 41 | } 42 | return best; 43 | } 44 | 45 | @Override 46 | public float getLoss(IInstance ins, IStructure gold, IStructure pred) { 47 | return PairY.getLoss((PairY)gold, (PairY)pred); 48 | } 49 | } -------------------------------------------------------------------------------- /src/main/java/pair/PairX.java: -------------------------------------------------------------------------------- 1 | package pair; 2 | 3 | import java.util.List; 4 | 5 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 6 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; 7 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 8 | import structure.Problem; 9 | import structure.QuantSpan; 10 | import structure.Schema; 11 | 12 | public class PairX implements IInstance { 13 | 14 | public int problemId; 15 | public int quantIndex1; 16 | public int quantIndex2; 17 | public TextAnnotation ta; 18 | public List quantities; 19 | public List posTags; 20 | public List chunks; 21 | public List parse; 22 | public List dependency; 23 | public List lemmas; 24 | public Schema schema; 25 | 26 | public PairX(Problem prob, int quantIndex1, int quantIndex2) { 27 | this.problemId = prob.id; 28 | this.quantIndex1 = quantIndex1; 29 | this.quantIndex2 = quantIndex2; 30 | this.ta = prob.ta; 31 | this.quantities = prob.quantities; 32 | this.posTags = prob.posTags; 33 | this.chunks = prob.chunks; 34 | this.schema = prob.schema; 35 | this.lemmas = prob.lemmas; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/pair/PairY.java: -------------------------------------------------------------------------------- 1 | package pair; 2 | 3 | 4 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 5 | 6 | public class PairY implements IStructure { 7 | 8 | public String label; 9 | 10 | public PairY(String label) { 11 | this.label = label; 12 | } 13 | 14 | public PairY(PairY other) { 15 | this.label = other.label; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return label; 21 | } 22 | 23 | public static float getLoss(PairY gold, PairY pred) { 24 | if(gold.label.equals(pred.label)) { 25 | return 0.0f; 26 | } 27 | return 1.0f; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/rate/RateFeatGen.java: -------------------------------------------------------------------------------- 1 | package rate; 2 | 3 | import java.io.Serializable; 4 | import java.util.*; 5 | 6 | import utils.FeatGen; 7 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 8 | import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator; 9 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 10 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 11 | import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; 12 | import edu.illinois.cs.cogcomp.sl.util.Lexiconer; 13 | 14 | public class RateFeatGen extends AbstractFeatureGenerator 15 | implements Serializable{ 16 | 17 | private static final long serialVersionUID = -5902462551801564955L; 18 | public Lexiconer lm = null; 19 | 20 | public RateFeatGen(Lexiconer lm) { 21 | this.lm = lm; 22 | } 23 | 24 | @Override 25 | public IFeatureVector getFeatureVector(IInstance prob, IStructure struct) { 26 | RateX x = (RateX) prob; 27 | RateY y = (RateY) struct; 28 | return FeatGen.getFeatureVectorFromListString(getFeatures(x, y.label), lm); 29 | } 30 | 31 | public static List getFeatures(RateX x, String label) { 32 | List featuresWithPrefix = new ArrayList(); 33 | String prefix = label; 34 | featuresWithPrefix.add(prefix); 35 | List features = getFeatures(x); 36 | features.addAll(FeatGen.getConjunctions(features)); 37 | for(String feature : features) { 38 | featuresWithPrefix.add(prefix + "_" + feature); 39 | } 40 | return featuresWithPrefix; 41 | } 42 | 43 | public static List getFeatures(RateX x) { 44 | List features = new ArrayList(); 45 | features.addAll(perQuantityFeatures(x)); 46 | return features; 47 | } 48 | 49 | public static List perQuantityFeatures(RateX x) { 50 | List features = new ArrayList(); 51 | for(String feature : perQuantityFeatures(x, x.quantIndex)) { 52 | features.add("1_"+feature); 53 | } 54 | return features; 55 | } 56 | 57 | public static List perQuantityFeatures(RateX x, int quantIndex) { 58 | List features = new ArrayList(); 59 | if(quantIndex == -1) { 60 | for(int i=0; i labels = Arrays.asList("RATE", "NOT_RATE"); 33 | double bestScore = -Double.MAX_VALUE; 34 | RateY best = null; 35 | for(String label : labels) { 36 | double score = weight.dotProduct(featGen.getFeatureVector(ins, new RateY(label))); 37 | if(bestScore < score) { 38 | best = new RateY(label); 39 | bestScore = score; 40 | } 41 | } 42 | return best; 43 | } 44 | 45 | @Override 46 | public float getLoss(IInstance ins, IStructure gold, IStructure pred) { 47 | return RateY.getLoss((RateY)gold, (RateY)pred); 48 | } 49 | } -------------------------------------------------------------------------------- /src/main/java/rate/RateX.java: -------------------------------------------------------------------------------- 1 | package rate; 2 | 3 | import java.util.List; 4 | 5 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 6 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; 7 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 8 | import graph.GraphX; 9 | import structure.Problem; 10 | import structure.QuantSpan; 11 | import structure.Schema; 12 | 13 | public class RateX implements IInstance { 14 | 15 | public int problemId; 16 | public int quantIndex; 17 | public TextAnnotation ta; 18 | public List quantities; 19 | public List posTags; 20 | public List chunks; 21 | public List parse; 22 | public List dependency; 23 | public List lemmas; 24 | public Schema schema; 25 | 26 | public RateX(Problem prob, int quantIndex) { 27 | this.problemId = prob.id; 28 | this.quantIndex = quantIndex; 29 | this.ta = prob.ta; 30 | this.quantities = prob.quantities; 31 | this.posTags = prob.posTags; 32 | this.chunks = prob.chunks; 33 | this.schema = prob.schema; 34 | this.lemmas = prob.lemmas; 35 | } 36 | 37 | public RateX(GraphX prob, int quantIndex) { 38 | this.problemId = prob.problemId; 39 | this.quantIndex = quantIndex; 40 | this.ta = prob.ta; 41 | this.quantities = prob.quantities; 42 | this.posTags = prob.posTags; 43 | this.chunks = prob.chunks; 44 | this.schema = prob.schema; 45 | this.lemmas = prob.lemmas; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/rate/RateY.java: -------------------------------------------------------------------------------- 1 | package rate; 2 | 3 | 4 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 5 | 6 | public class RateY implements IStructure { 7 | 8 | public String label; 9 | 10 | public RateY(String label) { 11 | this.label = label; 12 | } 13 | 14 | public RateY(RateY other) { 15 | this.label = other.label; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return label; 21 | } 22 | 23 | public static float getLoss(RateY gold, RateY pred) { 24 | if(gold.label.equals(pred.label)) { 25 | return 0.0f; 26 | } 27 | return 1.0f; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/reader/CrowdFlower.java: -------------------------------------------------------------------------------- 1 | package reader; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.reflect.TypeToken; 5 | import org.apache.commons.io.FileUtils; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | class CFJudgmentData { 13 | public String question1; 14 | } 15 | 16 | class CFJudgment { 17 | public int worker_id; 18 | public boolean tainted; 19 | public CFJudgmentData data; 20 | } 21 | 22 | class CFResults { 23 | public List judgments; 24 | } 25 | 26 | class CFData { 27 | public String question; 28 | public String answer; 29 | } 30 | 31 | public class CrowdFlower { 32 | public int id; 33 | public CFData data; 34 | public CFResults results; 35 | 36 | public static List readCrowdFlowerFile(String fileName) 37 | throws IOException { 38 | List problems = new ArrayList<>(); 39 | for(String jsonLine : FileUtils.readLines(new File(fileName))) { 40 | CrowdFlower prob =new Gson().fromJson(jsonLine, 41 | new TypeToken(){}.getType()); 42 | problems.add(prob); 43 | } 44 | return problems; 45 | } 46 | 47 | public static void main(String args[]) throws Exception { 48 | List probs = readCrowdFlowerFile("data/job_1012604.json"); 49 | for(CrowdFlower prob : probs) { 50 | String inp = prob.data.question; 51 | for(CFJudgment judgment : prob.results.judgments) { 52 | if(judgment.tainted) continue; 53 | String mod = judgment.data.question1; 54 | if(inp.trim().equals(mod.trim())) { 55 | System.out.println(prob.id+" : Same Problem"); 56 | } 57 | } 58 | } 59 | } 60 | 61 | } 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/main/java/reader/Reader.java: -------------------------------------------------------------------------------- 1 | package reader; 2 | 3 | 4 | import java.io.File; 5 | import java.io.IOException; 6 | import java.util.ArrayList; 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | import org.apache.commons.io.FileUtils; 11 | 12 | import com.google.gson.Gson; 13 | import com.google.gson.reflect.TypeToken; 14 | 15 | import structure.*; 16 | import utils.Params; 17 | import utils.Tools; 18 | 19 | public class Reader { 20 | 21 | public static List readProblemsFromJson() throws Exception { 22 | String json = FileUtils.readFileToString(new File(Params.questionsFile)); 23 | List kushmanProbs = new Gson().fromJson(json, 24 | new TypeToken>(){}.getType()); 25 | List problemList = new ArrayList<>(); 26 | for(DataFormat kushmanProb : kushmanProbs) { 27 | Problem prob = new Problem(kushmanProb.iIndex, kushmanProb.sQuestion, 28 | kushmanProb.lSolutions.get(0)); 29 | // prob.quants = kushmanProb.quants; 30 | prob.rates = kushmanProb.rates; 31 | prob.extractQuantities(); 32 | prob.expr = Node.parseNode(kushmanProb.lEquations.get(0)); 33 | assert prob.expr.getLeaves().size() == kushmanProb.lAlignments.size(); 34 | for(int j=0; j readStanfordProblemsFromJson() 46 | throws Exception { 47 | String json = FileUtils.readFileToString(new File(Params.questionsFile)); 48 | List kushmanProbs = new Gson().fromJson(json, 49 | new TypeToken>(){}.getType()); 50 | List problemList = new ArrayList<>(); 51 | for(DataFormat kushmanProb : kushmanProbs) { 52 | StanfordProblem prob = new StanfordProblem( 53 | kushmanProb.iIndex, 54 | kushmanProb.sQuestion, 55 | kushmanProb.lSolutions.get(0)); 56 | // prob.quants = kushmanProb.quants; 57 | prob.rates = kushmanProb.rates; 58 | prob.extractQuantities(); 59 | prob.expr = Node.parseNode(kushmanProb.lEquations.get(0)); 60 | assert prob.expr.getLeaves().size() == kushmanProb.lAlignments.size(); 61 | for(int j=0; j problems, String tabFile) throws IOException { 74 | String str = ""; 75 | List labels = Arrays.asList("ADD", "SUB", "MUL", "DIV", "SUB_REV", "DIV_REV"); 76 | for(StanfordProblem prob : problems) { 77 | Node copy = new Node(prob.expr); 78 | for(Node node : copy.getAllSubNodes()) { 79 | if(!labels.contains(node.label)) continue; 80 | String origLabel = node.label; 81 | for(String label : labels) { 82 | if((origLabel.equals("ADD") || origLabel.equals("SUB")) && 83 | (label.startsWith("MUL") || label.startsWith("DIV"))) continue; 84 | if((origLabel.equals("MUL") || origLabel.equals("DIV")) && 85 | (label.startsWith("ADD") || label.startsWith("SUB"))) continue; 86 | if(label.equals(origLabel)) continue; 87 | if(label.endsWith("REV")) { 88 | node.label = label.substring(0, 3); 89 | node.children.add(node.children.get(0)); 90 | node.children.remove(0); 91 | } else { 92 | node.label = label; 93 | } 94 | double val = copy.getValue(); 95 | boolean answerSame = false; 96 | if(Tools.safeEquals(val, prob.answer)) { 97 | System.out.println("Problem here between "+ 98 | prob.expr.toString()+" and "+copy.toString()); 99 | answerSame = true; 100 | } 101 | if(val > 1.0 && !answerSame) { 102 | str += prob.question + "\t" + copy.toString() + "=" + 103 | copy.getValue() + "\n"; 104 | } 105 | if(label.endsWith("REV")) { 106 | node.children.add(node.children.get(0)); 107 | node.children.remove(0); 108 | } 109 | } 110 | node.label = origLabel; 111 | } 112 | } 113 | FileUtils.writeStringToFile(new File(tabFile), str); 114 | } 115 | 116 | public static void main(String args[]) throws Exception { 117 | createTabSeparatedAnswerPerturbFile( 118 | Reader.readStanfordProblemsFromJson(), "perturbFile"); 119 | } 120 | 121 | 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/relevance/RelDriver.java: -------------------------------------------------------------------------------- 1 | package relevance; 2 | 3 | import java.util.*; 4 | 5 | import structure.Problem; 6 | import utils.Folds; 7 | import utils.Params; 8 | import edu.illinois.cs.cogcomp.core.datastructures.Pair; 9 | import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator; 10 | import edu.illinois.cs.cogcomp.sl.core.SLModel; 11 | import edu.illinois.cs.cogcomp.sl.core.SLParameters; 12 | import edu.illinois.cs.cogcomp.sl.core.SLProblem; 13 | import edu.illinois.cs.cogcomp.sl.learner.Learner; 14 | import edu.illinois.cs.cogcomp.sl.learner.LearnerFactory; 15 | import edu.illinois.cs.cogcomp.sl.util.Lexiconer; 16 | 17 | public class RelDriver { 18 | 19 | public static void crossVal(List probs, List> foldIndices) 20 | throws Exception { 21 | double acc1 = 0.0, acc2 = 0.0; 22 | for(int i=0;i train = new ArrayList<>(); 24 | List test = new ArrayList<>(); 25 | for(int j=0; j pair = doTrainTest(probs, train, test, i); 30 | acc1 += pair.getFirst(); 31 | acc2 += pair.getSecond(); 32 | } 33 | System.out.println("CV : " + (acc1/foldIndices.size()) + " " + (acc2/foldIndices.size())); 34 | } 35 | 36 | public static Pair doTrainTest(List probs, List trainIndices, 37 | List testIndices, int id) throws Exception { 38 | List> split = Folds.getDataSplit(probs, trainIndices, testIndices, 0.0); 39 | List trainProbs = split.get(0); 40 | List testProbs = split.get(2); 41 | SLProblem train = getSP(trainProbs); 42 | SLProblem test = getSP(testProbs); 43 | System.out.println("Train : "+train.instanceList.size()+" Test : "+test.instanceList.size()); 44 | trainModel(Params.modelDir+Params.relPrefix+id+Params.modelSuffix, train); 45 | return testModel(Params.modelDir+Params.relPrefix+id+Params.modelSuffix, test); 46 | } 47 | 48 | public static SLProblem getSP(List problemList) throws Exception{ 49 | SLProblem problem = new SLProblem(); 50 | for(Problem prob : problemList){ 51 | for(int i=0; i testModel(String modelPath, SLProblem sp) 61 | throws Exception { 62 | SLModel model = SLModel.loadModel(modelPath); 63 | Set incorrect = new HashSet<>(); 64 | Set total = new HashSet<>(); 65 | double acc = 0.0; 66 | for (int i = 0; i < sp.instanceList.size(); i++) { 67 | RelX prob = (RelX) sp.instanceList.get(i); 68 | RelY gold = (RelY) sp.goldStructureList.get(i); 69 | RelY pred = (RelY) model.infSolver.getBestStructure(model.wv, prob); 70 | total.add(prob.problemId); 71 | boolean correct = false; 72 | if(RelY.getLoss(gold, pred) < 0.0001) { 73 | acc += 1; 74 | correct = true; 75 | } else { 76 | incorrect.add(prob.problemId); 77 | } 78 | if((correct && Params.printCorrect) || 79 | (!correct && Params.printMistakes)){ 80 | System.out.println(prob.problemId+" : "+prob.ta.getText()); 81 | System.out.println(); 82 | System.out.println("Schema : "+prob.schema); 83 | System.out.println(); 84 | System.out.println("Quantities : "+prob.quantities); 85 | System.out.println("Quant of Interest: "+prob.quantIndex); 86 | System.out.println("Gold : "+gold); 87 | System.out.println("Pred : "+pred); 88 | System.out.println(); 89 | } 90 | } 91 | System.out.println("Accuracy : = " + acc + " / " + sp.instanceList.size() 92 | + " = " + (acc/sp.instanceList.size())); 93 | System.out.println("Strict Accuracy : ="+ (1-1.0*incorrect.size()/total.size())); 94 | return new Pair(acc/sp.instanceList.size(), 95 | 1-1.0*incorrect.size()/total.size()); 96 | } 97 | 98 | public static void trainModel(String modelPath, SLProblem train) throws Exception { 99 | SLModel model = new SLModel(); 100 | Lexiconer lm = new Lexiconer(); 101 | lm.setAllowNewFeatures(true); 102 | model.lm = lm; 103 | AbstractFeatureGenerator fg = new RelFeatGen(lm); 104 | model.featureGenerator = fg; 105 | model.infSolver = new RelInfSolver(fg); 106 | SLParameters para = new SLParameters(); 107 | para.loadConfigFile(Params.spConfigFile); 108 | para.MAX_NUM_ITER = 5; 109 | Learner learner = LearnerFactory.getLearner(model.infSolver, fg, para); 110 | model.wv = learner.train(train); 111 | lm.setAllowNewFeatures(false); 112 | model.saveModel(modelPath); 113 | } 114 | 115 | public static Map getLabelsWithScores(RelX prob, SLModel model) { 116 | List labels = Arrays.asList("REL", "IRR"); 117 | Map labelsWithScores = new HashMap(); 118 | for(String label : labels) { 119 | labelsWithScores.put(prob.quantIndex+"_"+label, 120 | 1.0*model.wv.dotProduct(model.featureGenerator.getFeatureVector( 121 | prob, 122 | new RelY(label)))); 123 | } 124 | return labelsWithScores; 125 | } 126 | } -------------------------------------------------------------------------------- /src/main/java/relevance/RelInfSolver.java: -------------------------------------------------------------------------------- 1 | package relevance; 2 | 3 | import java.io.Serializable; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator; 8 | import edu.illinois.cs.cogcomp.sl.core.AbstractInferenceSolver; 9 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 10 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 11 | import edu.illinois.cs.cogcomp.sl.util.WeightVector; 12 | 13 | public class RelInfSolver extends AbstractInferenceSolver implements 14 | Serializable { 15 | 16 | private static final long serialVersionUID = 5253748728743334706L; 17 | private AbstractFeatureGenerator featGen; 18 | 19 | public RelInfSolver(AbstractFeatureGenerator featGen) throws Exception { 20 | this.featGen = featGen; 21 | } 22 | 23 | @Override 24 | public IStructure getBestStructure(WeightVector weight, IInstance ins) 25 | throws Exception { 26 | return getLossAugmentedBestStructure(weight, ins, null); 27 | } 28 | 29 | @Override 30 | public IStructure getLossAugmentedBestStructure(WeightVector weight, 31 | IInstance ins, IStructure goldStructure) throws Exception { 32 | List labels = Arrays.asList("REL", "IRR"); 33 | double bestScore = -Double.MAX_VALUE; 34 | RelY best = null; 35 | for(String label : labels) { 36 | double score = weight.dotProduct(featGen.getFeatureVector( 37 | ins, new RelY(label))); 38 | if(bestScore < score) { 39 | best = new RelY(label); 40 | bestScore = score; 41 | } 42 | } 43 | return best; 44 | } 45 | 46 | @Override 47 | public float getLoss(IInstance ins, IStructure gold, IStructure pred) { 48 | return RelY.getLoss((RelY)gold, (RelY)pred); 49 | } 50 | } -------------------------------------------------------------------------------- /src/main/java/relevance/RelX.java: -------------------------------------------------------------------------------- 1 | package relevance; 2 | 3 | import java.util.List; 4 | 5 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 6 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; 7 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 8 | import structure.Problem; 9 | import structure.QuantSpan; 10 | import structure.Schema; 11 | 12 | public class RelX implements IInstance { 13 | 14 | public int problemId; 15 | public int quantIndex; 16 | public TextAnnotation ta; 17 | public List quantities; 18 | public List posTags; 19 | public List chunks; 20 | public List parse; 21 | public List dependency; 22 | public List lemmas; 23 | public Schema schema; 24 | 25 | public RelX(Problem prob, int quantIndex) { 26 | this.problemId = prob.id; 27 | this.quantIndex = quantIndex; 28 | this.ta = prob.ta; 29 | this.quantities = prob.quantities; 30 | this.posTags = prob.posTags; 31 | this.chunks = prob.chunks; 32 | this.lemmas = prob.lemmas; 33 | this.schema = prob.schema; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/relevance/RelY.java: -------------------------------------------------------------------------------- 1 | package relevance; 2 | 3 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 4 | 5 | public class RelY implements IStructure { 6 | 7 | public String relevance; 8 | 9 | public RelY(String relevance) { 10 | this.relevance = relevance; 11 | } 12 | 13 | public RelY(RelY other) { 14 | this.relevance = other.relevance; 15 | } 16 | 17 | @Override 18 | public String toString() { 19 | return relevance; 20 | } 21 | 22 | public static float getLoss(RelY gold, RelY pred) { 23 | if(gold.relevance.equals(pred.relevance)) { 24 | return 0.0f; 25 | } 26 | return 1.0f; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/run/Annotations.java: -------------------------------------------------------------------------------- 1 | package run; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.InputStreamReader; 6 | import java.util.ArrayList; 7 | import java.util.Arrays; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import org.apache.commons.io.FileUtils; 13 | 14 | import edu.illinois.cs.cogcomp.sl.core.SLProblem; 15 | import structure.Node; 16 | import structure.Problem; 17 | 18 | public class Annotations { 19 | 20 | public static String getLabel(List path, boolean rate1, boolean rate2) { 21 | int count = 0; 22 | for(String label : path) { 23 | if(label.startsWith("MUL") || label.startsWith("DIV")) { 24 | count++; 25 | } 26 | } 27 | if(rate1 == rate2 && count == 0) { 28 | return "SAME_UNIT"; 29 | } 30 | if(rate1 != rate2 && count == 1) { 31 | if(path.contains("MUL")) { 32 | if(rate1) { 33 | return "1_RATE_2"; 34 | } else { 35 | return "2_RATE_2"; 36 | } 37 | } 38 | if(path.contains("DIV")) { 39 | if(rate1) { 40 | return "NO_REL"; 41 | } else { 42 | return "2_RATE_1"; 43 | } 44 | } 45 | if(path.contains("DIV_REV")) { 46 | if(rate1) { 47 | return "1_RATE_1"; 48 | } else { 49 | return "NO_REL"; 50 | } 51 | } 52 | } 53 | if(count == 2) { 54 | // Currently no good heuristic, so just returning no relation 55 | } 56 | return "NO_REL"; 57 | } 58 | 59 | public static SLProblem getSP(List problemList) 60 | throws Exception{ 61 | SLProblem problem = new SLProblem(); 62 | for(Problem prob : problemList) { 63 | for(int i=0; i path = prob.expr.getPath(i, j); 69 | RunX x = new RunX(prob, i, j); 70 | RunY y = new RunY(getLabel(path, prob.rates.contains(i), 71 | prob.rates.contains(j))); 72 | problem.addExample(x, y); 73 | } 74 | // Relation with question 75 | List path = prob.expr.getPathToRoot(i); 76 | RunX x = new RunX(prob, i, -1); 77 | RunY y = new RunY(getLabel(path, prob.rates.contains(i), 78 | prob.rates.contains(-1))); 79 | problem.addExample(x, y); 80 | } 81 | 82 | } 83 | return problem; 84 | } 85 | 86 | public static Map> readRateAnnotations(String fileName) 87 | throws Exception{ 88 | Map> rates = new HashMap>(); 89 | for(String line : FileUtils.readLines(new File(fileName))) { 90 | if(line.trim().equals("")) continue; 91 | String strArr[] = line.split("\t"); 92 | List indices = new ArrayList<>(); 93 | if(strArr.length >= 2) { 94 | for(String str : strArr[1].split(" ")) { 95 | indices.add(Integer.parseInt(str)); 96 | } 97 | } 98 | rates.put(Integer.parseInt(strArr[0]), indices); 99 | } 100 | return rates; 101 | } 102 | 103 | public static void getRateAnnotations(List problemList, String dir) throws Exception{ 104 | BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); 105 | int count = 0; 106 | String str = ""; 107 | for(Problem prob : problemList) { 108 | boolean onlyAddSub = true; 109 | for(Node node : prob.expr.getAllSubNodes()) { 110 | if(node.label.startsWith("MUL") || node.label.startsWith("DIV")) { 111 | onlyAddSub = false; 112 | break; 113 | } 114 | } 115 | if(onlyAddSub) continue; 116 | count++; 117 | if(count%10 == 0) { 118 | System.out.println("Annotated "+count+" problems"); 119 | FileUtils.writeStringToFile(new File(dir+"/rateAnnotations.txt"), str); 120 | } 121 | System.out.println(prob.id+" "+prob.question); 122 | System.out.println("-1 : "+Arrays.asList(prob.schema.questionTokens)); 123 | for(int i=0; i problemList, Map> rateAnns) 135 | throws Exception{ 136 | BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); 137 | int matched = 0, total = 0; 138 | String str; 139 | 140 | for(int j=problemList.size()-1; j>=0; j-=20) { 141 | Problem prob = problemList.get(j); 142 | System.out.println(prob.id+" : "+prob.question); 143 | System.out.println("\nQuantities detected : "+Arrays.asList(prob.quantities)); 144 | System.out.println("\nIs the question asking for a rate ? (y/n) : "); 145 | str = br.readLine().trim(); 146 | total++; 147 | if(str.equalsIgnoreCase("y") && rateAnns.containsKey(prob.id) && 148 | rateAnns.get(prob.id).contains(-1)) { 149 | matched++; 150 | } 151 | if(str.equalsIgnoreCase("n") && (!rateAnns.containsKey(prob.id) || 152 | !rateAnns.get(prob.id).contains(-1))) { 153 | matched++; 154 | } 155 | for(int i=0; i probs, List> foldIndices) 20 | throws Exception { 21 | double acc1 = 0.0, acc2 = 0.0; 22 | for(int i=0;i train = new ArrayList<>(); 24 | List test = new ArrayList<>(); 25 | for(int j=0; j pair = doTrainTest(probs, train, test, i); 30 | acc1 += pair.getFirst(); 31 | acc2 += pair.getSecond(); 32 | } 33 | System.out.println("CV : " + (acc1/foldIndices.size()) + " " + (acc2/foldIndices.size())); 34 | } 35 | 36 | public static Pair doTrainTest(List probs, List trainIndices, 37 | List testIndices, int id) throws Exception { 38 | List> split = Folds.getDataSplit(probs, trainIndices, testIndices, 0.0); 39 | List trainProbs = split.get(0); 40 | List testProbs = split.get(2); 41 | SLProblem train = Annotations.getSP(trainProbs); 42 | SLProblem test = Annotations.getSP(testProbs); 43 | System.out.println("Train : "+train.instanceList.size()+" Test : "+test.instanceList.size()); 44 | trainModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix, train); 45 | return testModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix, test); 46 | } 47 | 48 | public static Pair testModel(String modelPath, SLProblem sp) 49 | throws Exception { 50 | SLModel model = SLModel.loadModel(modelPath); 51 | Set incorrect = new HashSet<>(); 52 | Set total = new HashSet<>(); 53 | double acc = 0.0; 54 | int countNoRel = 0; 55 | Map, Integer> counts = new HashMap<>(); 56 | for (int i = 0; i < sp.instanceList.size(); i++) { 57 | RunX prob = (RunX) sp.instanceList.get(i); 58 | RunY gold = (RunY) sp.goldStructureList.get(i); 59 | RunY pred = (RunY) model.infSolver.getBestStructure(model.wv, prob); 60 | if(pred.label.equalsIgnoreCase("NO_REL")) countNoRel++; 61 | total.add(prob.problemId); 62 | if(!counts.containsKey(new Pair<>(gold.label, pred.label))) { 63 | counts.put(new Pair<>(gold.label, pred.label), 1); 64 | } else { 65 | counts.put(new Pair<>(gold.label, pred.label), 66 | counts.get(new Pair<>(gold.label, pred.label))+1); 67 | } 68 | boolean correct = false; 69 | if(RunY.getLoss(gold, pred) < 0.0001) { 70 | acc += 1; 71 | correct = true; 72 | } else { 73 | incorrect.add(prob.problemId); 74 | } 75 | if((correct && Params.printCorrect) || 76 | (!correct && Params.printMistakes)){ 77 | System.out.println(prob.problemId+" : "+prob.ta.getText()); 78 | System.out.println(); 79 | System.out.println("Schema : "+prob.schema); 80 | System.out.println(); 81 | System.out.println("Quantities : "+prob.quantities); 82 | System.out.println("Quant of Interest: "+prob.quantIndex1+" "+prob.quantIndex2); 83 | System.out.println("Gold : "+gold); 84 | System.out.println("Pred : "+pred); 85 | System.out.println("Loss : "+RunY.getLoss(gold, pred)); 86 | System.out.println("Labels : "+Arrays.asList(getLabelsWithScores(prob, model))); 87 | System.out.println(); 88 | } 89 | } 90 | System.out.println("Accuracy : = " + acc + " / " + sp.instanceList.size() 91 | + " = " + (acc/sp.instanceList.size())); 92 | for(Pair key : counts.keySet()) { 93 | double tot = 0, count = 0; 94 | for(Pair key1 : counts.keySet()) { 95 | if(key1.getFirst().equals(key.getFirst())) { 96 | tot += counts.get(key1); 97 | if(key1.getFirst().equals(key1.getSecond())) { 98 | count += counts.get(key1); 99 | } 100 | } 101 | } 102 | System.out.println(key.getFirst()+" : "+count+" "+tot+" "+(count/tot)); 103 | } 104 | System.out.println("Strict Accuracy : ="+ (1-1.0*incorrect.size()/total.size())); 105 | System.out.println("NoRel : ="+ countNoRel); 106 | return new Pair<>(acc/sp.instanceList.size(), 1-1.0*incorrect.size()/total.size()); 107 | } 108 | 109 | public static void trainModel(String modelPath, SLProblem train) 110 | throws Exception { 111 | SLModel model = new SLModel(); 112 | Lexiconer lm = new Lexiconer(); 113 | lm.setAllowNewFeatures(true); 114 | model.lm = lm; 115 | AbstractFeatureGenerator fg = new RunFeatGen(lm); 116 | model.featureGenerator = fg; 117 | model.infSolver = new RunInfSolver(fg); 118 | SLParameters para = new SLParameters(); 119 | para.loadConfigFile(Params.spConfigFile); 120 | para.MAX_NUM_ITER = 5; 121 | Learner learner = LearnerFactory.getLearner(model.infSolver, fg, para); 122 | model.wv = learner.train(train); 123 | lm.setAllowNewFeatures(false); 124 | model.saveModel(modelPath); 125 | } 126 | 127 | public static Map getLabelsWithScores(RunX prob, SLModel model) { 128 | List labels = Arrays.asList("SAME_UNIT", "1_RATE_1", "1_RATE_2", 129 | "2_RATE_1", "2_RATE_2", "NO_REL"); 130 | Map labelsWithScores = new HashMap(); 131 | for(String label : labels) { 132 | labelsWithScores.put(prob.quantIndex1+"_"+prob.quantIndex2+"_"+label, 133 | 1.0*model.wv.dotProduct(model.featureGenerator.getFeatureVector( 134 | prob, 135 | new RunY(label)))); 136 | } 137 | return labelsWithScores; 138 | } 139 | } -------------------------------------------------------------------------------- /src/main/java/run/RunFeatGen.java: -------------------------------------------------------------------------------- 1 | package run; 2 | 3 | import java.io.Serializable; 4 | import java.util.*; 5 | 6 | import utils.FeatGen; 7 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 8 | import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator; 9 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 10 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 11 | import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; 12 | import edu.illinois.cs.cogcomp.sl.util.Lexiconer; 13 | 14 | public class RunFeatGen extends AbstractFeatureGenerator 15 | implements Serializable{ 16 | 17 | private static final long serialVersionUID = -5902462551801564955L; 18 | public Lexiconer lm = null; 19 | 20 | public RunFeatGen(Lexiconer lm) { 21 | this.lm = lm; 22 | } 23 | 24 | @Override 25 | public IFeatureVector getFeatureVector(IInstance prob, IStructure struct) { 26 | RunX x = (RunX) prob; 27 | RunY y = (RunY) struct; 28 | return FeatGen.getFeatureVectorFromListString(getFeatures(x, y.label), lm); 29 | } 30 | 31 | public static List getFeatures(RunX x, String label) { 32 | List featuresWithPrefix = new ArrayList(); 33 | String prefix = label; 34 | featuresWithPrefix.add(prefix); 35 | List features = getFeatures(x); 36 | features.addAll(FeatGen.getConjunctions(features)); 37 | for(String feature : features) { 38 | featuresWithPrefix.add(prefix + "_" + feature); 39 | } 40 | return featuresWithPrefix; 41 | } 42 | 43 | public static List getFeatures(RunX x) { 44 | List features = new ArrayList(); 45 | features.addAll(perQuantityFeatures(x)); 46 | features.addAll(pairQuantityFeatures(x)); 47 | return features; 48 | } 49 | 50 | public static List perQuantityFeatures(RunX x) { 51 | List features = new ArrayList(); 52 | for(String feature : perQuantityFeatures(x, x.quantIndex1)) { 53 | features.add("1_"+feature); 54 | } 55 | for(String feature : perQuantityFeatures(x, x.quantIndex2)) { 56 | features.add("2_"+feature); 57 | } 58 | return features; 59 | } 60 | 61 | 62 | public static List pairQuantityFeatures(RunX x) { 63 | List features = new ArrayList(); 64 | int tokenId1 = x.ta.getTokenIdFromCharacterOffset(x.quantities.get(x.quantIndex1).start); 65 | String unit1 = x.schema.quantSchemas.get(x.quantIndex1).unit; 66 | Constituent rate1 = x.schema.quantSchemas.get(x.quantIndex1).rateUnit; 67 | if(x.quantIndex2 == -1) { 68 | for(String str1 : unit1.split(" ")) { 69 | for(int i=0; i perQuantityFeatures(RunX x, int quantIndex) { 152 | List features = new ArrayList(); 153 | if(quantIndex == -1) { 154 | for(int i=0; i labels = Arrays.asList("SAME_UNIT", "1_RATE_1", "1_RATE_2", 33 | "2_RATE_1", "2_RATE_2", "NO_REL"); 34 | double bestScore = -Double.MAX_VALUE; 35 | RunY best = null; 36 | for(String label : labels) { 37 | double score = weight.dotProduct(featGen.getFeatureVector(ins, new RunY(label))); 38 | if(bestScore < score) { 39 | best = new RunY(label); 40 | bestScore = score; 41 | } 42 | } 43 | return best; 44 | } 45 | 46 | @Override 47 | public float getLoss(IInstance ins, IStructure gold, IStructure pred) { 48 | return RunY.getLoss((RunY)gold, (RunY)pred); 49 | } 50 | } -------------------------------------------------------------------------------- /src/main/java/run/RunX.java: -------------------------------------------------------------------------------- 1 | package run; 2 | 3 | import java.util.List; 4 | 5 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 6 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; 7 | import edu.illinois.cs.cogcomp.sl.core.IInstance; 8 | import graph.GraphX; 9 | import structure.Problem; 10 | import structure.QuantSpan; 11 | import structure.Schema; 12 | 13 | public class RunX implements IInstance { 14 | 15 | public int problemId; 16 | public int quantIndex1; 17 | public int quantIndex2; 18 | public TextAnnotation ta; 19 | public List quantities; 20 | public List posTags; 21 | public List chunks; 22 | public List parse; 23 | public List dependency; 24 | public List lemmas; 25 | public Schema schema; 26 | 27 | public RunX(Problem prob, int quantIndex1, int quantIndex2) { 28 | this.problemId = prob.id; 29 | this.quantIndex1 = quantIndex1; 30 | this.quantIndex2 = quantIndex2; 31 | this.ta = prob.ta; 32 | this.quantities = prob.quantities; 33 | this.posTags = prob.posTags; 34 | this.chunks = prob.chunks; 35 | this.schema = prob.schema; 36 | this.lemmas = prob.lemmas; 37 | } 38 | 39 | public RunX(GraphX prob, int quantIndex1, int quantIndex2) { 40 | this.problemId = prob.problemId; 41 | this.quantIndex1 = quantIndex1; 42 | this.quantIndex2 = quantIndex2; 43 | this.ta = prob.ta; 44 | this.quantities = prob.quantities; 45 | this.posTags = prob.posTags; 46 | this.chunks = prob.chunks; 47 | this.schema = prob.schema; 48 | this.lemmas = prob.lemmas; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/run/RunY.java: -------------------------------------------------------------------------------- 1 | package run; 2 | 3 | 4 | import edu.illinois.cs.cogcomp.sl.core.IStructure; 5 | 6 | public class RunY implements IStructure { 7 | 8 | public String label; 9 | 10 | public RunY(String label) { 11 | this.label = label; 12 | } 13 | 14 | public RunY(RunY other) { 15 | this.label = other.label; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return label; 21 | } 22 | 23 | public static float getLoss(RunY gold, RunY pred) { 24 | if(gold.label.equals(pred.label)) { 25 | return 0.0f; 26 | } 27 | return 1.0f; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/structure/DataFormat.java: -------------------------------------------------------------------------------- 1 | package structure; 2 | 3 | import java.util.List; 4 | 5 | public class DataFormat { 6 | public int iIndex; 7 | public String sQuestion; 8 | public List quants; 9 | public List lAlignments; 10 | public List lEquations; 11 | public List lSolutions; 12 | public List rates; 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/structure/PairComparator.java: -------------------------------------------------------------------------------- 1 | package structure; 2 | 3 | import java.util.Comparator; 4 | 5 | import edu.illinois.cs.cogcomp.core.datastructures.Pair; 6 | 7 | public abstract class PairComparator implements Comparator> { 8 | 9 | public int compare(Pair pair1, Pair pair2) { 10 | if(pair1.getSecond() == pair2.getSecond()) return 0; 11 | return (pair1.getSecond() < pair2.getSecond()) ? 1 : -1; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/structure/Problem.java: -------------------------------------------------------------------------------- 1 | package structure; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import javatools.parsers.PlingStemmer; 8 | import utils.Tools; 9 | import edu.illinois.cs.cogcomp.annotation.AnnotatorException; 10 | import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; 11 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; 12 | import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; 13 | 14 | public class Problem { 15 | 16 | public int id; 17 | public String question; 18 | public TextAnnotation ta; 19 | public Double answer; 20 | public List quantities; 21 | public Node expr; 22 | public List stems; 23 | public List lemmas; 24 | public List posTags; 25 | public List chunks; 26 | public List dependency; 27 | public List ner; 28 | public Schema schema; 29 | public List rates; 30 | 31 | @Override 32 | public String toString() { 33 | String str = ""; 34 | str += "\nQuestion : "+question+"\nAnswer : "+answer + 35 | "\nQuantities : "+Arrays.asList(quantities) + 36 | "\nExpression : "+expr; 37 | return str; 38 | } 39 | 40 | public Problem(int id, String q, double a) throws AnnotatorException { 41 | this.id = id; 42 | question = q; 43 | ta = Tools.pipeline.createAnnotatedTextAnnotation("", "", q); 44 | answer = a; 45 | quantities = new ArrayList<>(); 46 | } 47 | 48 | public void extractQuantities() throws Exception { 49 | quantities = Tools.quantifier.getSpans(question); 50 | } 51 | 52 | public void extractAnnotations() throws Exception { 53 | posTags = ta.getView(ViewNames.POS).getConstituents(); 54 | chunks = ta.getView(ViewNames.SHALLOW_PARSE).getConstituents(); 55 | dependency = ta.getView(ViewNames.DEPENDENCY_STANFORD).getConstituents(); 56 | ner = ta.getView(ViewNames.NER_CONLL).getConstituents(); 57 | stems = new ArrayList<>(); 58 | for(String token : ta.getTokens()) { 59 | stems.add(PlingStemmer.stem(token)); 60 | } 61 | lemmas = new ArrayList<>(); 62 | for(Constituent cons : ta.getView(ViewNames.LEMMA).getConstituents()) { 63 | lemmas.add(cons.getLabel()); 64 | } 65 | // Fix some chunker issues 66 | chunks = fixChunkerIssues(chunks); 67 | schema = new Schema(this); 68 | } 69 | 70 | public List fixChunkerIssues(List chunks) { 71 | List newChunks = new ArrayList<>(); 72 | for(int i=0; i= chunk.getStartSpan() && 83 | ta.getTokenIdFromCharacterOffset(qs.start) < chunk.getEndSpan()) { 84 | hasQuant = true; 85 | quantToken = ta.getTokenIdFromCharacterOffset(qs.start); 86 | } 87 | } 88 | if(hasQuant) { 89 | if(ta.getToken(chunk.getStartSpan()).equals("him") || ta.getToken(chunk.getStartSpan()).equals("her") 90 | || ta.getToken(chunk.getStartSpan()).equals("his")) { 91 | newChunks.add(new Constituent("NP", null, ta, chunk.getStartSpan(), 92 | quantToken)); 93 | newChunks.add(new Constituent("NP", null, ta, quantToken, 94 | chunk.getEndSpan())); 95 | doneSplit = true; 96 | continue; 97 | } 98 | for(Constituent cons : ner) { 99 | if(cons.getStartSpan() == chunk.getStartSpan() && quantToken > cons.getStartSpan()) { 100 | newChunks.add(new Constituent("NP", null, ta, chunk.getStartSpan(), 101 | quantToken)); 102 | newChunks.add(new Constituent("NP", null, ta, quantToken, 103 | chunk.getEndSpan())); 104 | doneSplit = true; 105 | break; 106 | } 107 | } 108 | } 109 | if(doneSplit) continue; 110 | newChunks.add(new Constituent(chunk.getLabel(), null, ta, chunk.getStartSpan(), chunk.getEndSpan())); 111 | } 112 | return newChunks; 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/structure/QuantSpan.java: -------------------------------------------------------------------------------- 1 | package structure; 2 | import java.io.*; 3 | 4 | import edu.illinois.cs.cogcomp.core.datastructures.IntPair; 5 | 6 | 7 | public class QuantSpan implements Serializable{ 8 | 9 | private static final long serialVersionUID = -5787092712439863265L; 10 | public int start, end; 11 | public double val; 12 | 13 | public QuantSpan(double val, int start, int end) { 14 | this.val = val; 15 | this.start = start; 16 | this.end = end; 17 | } 18 | 19 | public String toString() { 20 | IntPair ip = new IntPair(start, end); 21 | return val+":"+ip; 22 | } 23 | } -------------------------------------------------------------------------------- /src/main/java/utils/Folds.java: -------------------------------------------------------------------------------- 1 | package utils; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.*; 6 | 7 | import org.apache.commons.io.FileUtils; 8 | import structure.Problem; 9 | import structure.StanfordProblem; 10 | 11 | public class Folds { 12 | 13 | public static List readFoldIndices(String foldFile) throws IOException { 14 | String str = FileUtils.readFileToString(new File(foldFile)); 15 | List foldIndices = new ArrayList<>(); 16 | for(String index : str.split("\n")) { 17 | foldIndices.add(Integer.parseInt(index)); 18 | } 19 | return foldIndices; 20 | } 21 | 22 | // Returns List of 3 elements : train, val, test 23 | public static List> getDataSplit( 24 | List probs, List trainIndices, 25 | List testIndices, double validationFrac) throws Exception { 26 | List allTrain = new ArrayList<>(); 27 | List train = new ArrayList<>(); 28 | List val = new ArrayList<>(); 29 | List test = new ArrayList<>(); 30 | for(Problem prob : probs) { 31 | if(testIndices.contains(prob.id)) { 32 | test.add(prob); 33 | } 34 | if(trainIndices.contains(prob.id)) { 35 | allTrain.add(prob); 36 | } 37 | } 38 | Collections.shuffle(allTrain, new Random(0)); 39 | val.addAll(allTrain.subList(0, (int)(validationFrac*allTrain.size()))); 40 | train.addAll(allTrain.subList((int)(validationFrac*allTrain.size()), allTrain.size())); 41 | List> splits = new ArrayList<>(); 42 | splits.add(train); 43 | splits.add(val); 44 | splits.add(test); 45 | return splits; 46 | } 47 | 48 | // Returns List of 3 elements : train, val, test 49 | public static List> getDataSplitForStanford( 50 | List probs, List trainIndices, 51 | List testIndices, double validationFrac) throws Exception { 52 | List allTrain = new ArrayList<>(); 53 | List train = new ArrayList<>(); 54 | List val = new ArrayList<>(); 55 | List test = new ArrayList<>(); 56 | for(StanfordProblem prob : probs) { 57 | if(testIndices.contains(prob.id)) { 58 | test.add(prob); 59 | } 60 | if(trainIndices.contains(prob.id)) { 61 | allTrain.add(prob); 62 | } 63 | } 64 | Collections.shuffle(allTrain, new Random(0)); 65 | val.addAll(allTrain.subList(0, (int)(validationFrac*allTrain.size()))); 66 | train.addAll(allTrain.subList((int)(validationFrac*allTrain.size()), allTrain.size())); 67 | List> splits = new ArrayList<>(); 68 | splits.add(train); 69 | splits.add(val); 70 | splits.add(test); 71 | return splits; 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/utils/Params.java: -------------------------------------------------------------------------------- 1 | package utils; 2 | 3 | public class Params { 4 | 5 | public static String spConfigFile = "config/DCD.config"; 6 | public static boolean printMistakes = false; 7 | public static boolean printCorrect = true; 8 | 9 | public static String pipelineConfig = "config/pipeline.config"; 10 | 11 | public static String patternsFile = "data/patterns.txt"; 12 | public static String questionsFile = "data/questions.json"; 13 | 14 | public static String vectorsFile = "data/glove.6B.300d.verbs.txt"; 15 | 16 | public static boolean useIllinoisTools, useStanfordTools, noUDG, printLog = false, 17 | runDemo = false, startDemoServer = false; 18 | public static String modelDir = "models/"; 19 | public static String relPrefix = "Rel"; 20 | public static String pairPrefix = "Pair"; 21 | public static String ratePrefix = "Rate"; 22 | public static String runPrefix = "Run"; 23 | public static String graphPrefix = "Graph"; 24 | public static String corefPrefix = "Coref"; 25 | public static String logicPrefix = "Logic"; 26 | public static String modelSuffix = ".save"; 27 | 28 | } 29 | --------------------------------------------------------------------------------