├── .dockerignore ├── .gitignore ├── .gitmodules ├── .travis.tar.enc ├── .travis.yml ├── DockerBuild.conf ├── Dockerfile ├── LICENSE-2.0.txt ├── Makefile ├── README.md ├── compiler ├── README.md ├── app-has-been-compiled ├── app-has-nothing-to-compile ├── compile-check │ ├── compile-check-0-input_extractors_well_defined │ ├── compile-check-0-input_schema_wellformed │ ├── compile-check-1-compiled_base_relations_have_input_data │ ├── compile-check-1-compiled_dependencies_correct │ ├── compile-check-1-compiled_input_output_well_defined │ ├── compile-check-1-compiled_output_uniquely_defined │ ├── compile-check-2-codegen_no_path_collision │ └── util.jq ├── compile-code │ ├── compile-code-Makefile │ ├── compile-code-cmd_extractor │ ├── compile-code-dataflow_dot │ ├── compile-code-json_extractor │ ├── compile-code-sql_extractor │ ├── compile-code-tsX_extractor │ ├── constants.jq │ ├── sql.jq │ └── util.jq ├── compile-codegen ├── compile-config │ ├── compile-config-0.00-init_objects │ ├── compile-config-0.01-parse_calibration │ ├── compile-config-0.01-parse_schema │ ├── compile-config-0.51-add_init_app │ ├── compile-config-0.52-input_loader │ ├── compile-config-1.00-qualified_names │ ├── compile-config-1.01-parse_inference_rules │ ├── compile-config-2.01-grounding │ ├── compile-config-2.02-learning_inference │ ├── compile-config-2.03-calibration_plots │ ├── compile-config-9.98-ensure_init_app │ ├── compile-config-9.99-dependencies │ ├── constants.jq │ ├── sql.jq │ └── util.jq ├── ddlog-wrapper.sh ├── ddlog.build.sh ├── ddlog │ ├── .gitignore │ ├── .travis.yml │ ├── Makefile │ ├── README.md │ ├── build.sbt │ ├── examples │ │ ├── chunking.ddlog │ │ ├── ocr.ddlog │ │ ├── smoke.ddlog │ │ └── spouse_example.ddlog │ ├── project │ │ ├── assembly.sbt │ │ ├── build.properties │ │ ├── build.sbt │ │ └── sbt │ │ │ ├── sbt │ │ │ └── sbt-launch.jar │ ├── scala.mk │ ├── src │ │ └── main │ │ │ └── scala │ │ │ └── org │ │ │ └── deepdive │ │ │ └── ddlog │ │ │ ├── DeepDiveLog.scala │ │ │ ├── DeepDiveLogCompiler.scala │ │ │ ├── DeepDiveLogDesugarRewriter.scala │ │ │ ├── DeepDiveLogParser.scala │ │ │ ├── DeepDiveLogPrettyPrinter.scala │ │ │ ├── DeepDiveLogQueryCompiler.scala │ │ │ ├── DeepDiveLogSchemaExporter.scala │ │ │ └── DeepDiveLogSemanticChecker.scala │ └── test │ │ ├── README.md │ │ ├── bats-template.bash │ │ ├── bats.mk │ │ ├── checker-test.bats.template │ │ ├── checker-test │ │ ├── supervision_binary_variable │ │ │ └── input.ddlog │ │ ├── supervision_multinomial │ │ │ └── input.ddlog │ │ └── supervision_with_undeclared_relation │ │ │ └── input.ddlog │ │ ├── compile-error-test.bats.template │ │ ├── compile-error-test │ │ ├── column_dismatch │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── composing_boolean_categorical_vars │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── invalid_variable_schema │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── optional_modifier │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── repated_rule_name │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── unbound_variable_1 │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── unbound_variable_2 │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── unbound_variable_3 │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── unbound_variable_4 │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── unbound_variable_5 │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── undefined_function │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── undefined_relation │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── undefined_relation_2 │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ └── weight │ │ │ ├── compile-error.expected │ │ │ └── input.ddlog │ │ ├── ddlog │ │ ├── enumerate-tests.sh │ │ ├── expected-output-test.bats.template │ │ ├── expected-output-test │ │ ├── README.md │ │ ├── annotations │ │ │ ├── export-schema.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── arrays │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── chunking_example │ │ │ ├── compile.expected │ │ │ └── input.ddlog │ │ ├── comments │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── expressions │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── factor_functions │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── if-then-else │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── logical_rules │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── many_joins │ │ │ └── input.ddlog │ │ ├── materialize_annotation │ │ │ ├── compile.expected │ │ │ └── input.ddlog │ │ ├── multibody │ │ │ └── input.ddlog │ │ ├── multiple_rules_for_same_head │ │ │ ├── compile.expected │ │ │ ├── desugar.expected │ │ │ └── input.ddlog │ │ ├── ocr_example │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── semantics │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── smoke_example │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── spouse_example │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── views │ │ │ ├── compile.expected │ │ │ ├── export-schema.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ └── weights │ │ │ ├── compile.expected │ │ │ ├── input.ddlog │ │ │ └── print.expected │ │ ├── parse-error-test.bats.template │ │ ├── parse-error-test │ │ ├── annotation_semantics │ │ │ ├── input.ddlog │ │ │ └── parse-error.expected │ │ └── nested_multiline_comments │ │ │ ├── input.ddlog │ │ │ └── parse-error.expected │ │ ├── query-test.bats.template │ │ └── query-test │ │ └── spouse_example │ │ ├── app.ddlog │ │ ├── article_ids-filtered.ddlogq │ │ ├── article_ids-filtered.ddlogq.expected │ │ ├── article_ids-ordered-revsered.ddlogq │ │ ├── article_ids-ordered-revsered.ddlogq.expected │ │ ├── article_ids-ordered-revsered2.ddlogq │ │ ├── article_ids-ordered-revsered2.ddlogq.expected │ │ ├── article_ids-ordered.ddlogq │ │ ├── article_ids-ordered.ddlogq.expected │ │ ├── article_ids-parens.ddlogq │ │ ├── article_ids-parens.ddlogq.expected │ │ ├── article_ids-without_parens.ddlogq │ │ ├── article_ids-without_parens.ddlogq.expected │ │ ├── article_ids.ddlogq │ │ ├── article_ids.ddlogq.expected │ │ ├── count-has_spouse.ddlogq │ │ ├── count-has_spouse.ddlogq.expected │ │ ├── histogram-num_candidates-per-article-ordered.ddlogq │ │ ├── histogram-num_candidates-per-article-ordered.ddlogq.expected │ │ ├── histogram-num_candidates-per-article-ordered_multikey.ddlogq │ │ ├── histogram-num_candidates-per-article-ordered_multikey.ddlogq.expected │ │ ├── histogram-num_candidates-per-article.ddlogq │ │ ├── histogram-num_candidates-per-article.ddlogq.expected │ │ ├── histogram-supervision.ddlogq │ │ ├── histogram-supervision.ddlogq.expected │ │ ├── incomplete.ddlogq │ │ ├── incomplete.ddlogq-error.expected │ │ ├── redefinition.ddlogq │ │ ├── redefinition.ddlogq-error.expected │ │ ├── repeated-vars.ddlogq │ │ ├── repeated-vars.ddlogq.expected │ │ ├── schema-check.ddlogq │ │ ├── schema-check.ddlogq-error.expected │ │ ├── undefined.ddlogq │ │ ├── undefined.ddlogq-error.expected │ │ ├── union-no-common-vars.ddlogq │ │ ├── union-no-common-vars.ddlogq.expected │ │ ├── union.ddlogq │ │ └── union.ddlogq.expected ├── deepdive-check ├── deepdive-compile ├── deepdive-default.conf ├── hocon2json ├── jq2sh └── test │ ├── .gitignore │ ├── .should_accept.bats.template │ ├── .should_reject.bats.template │ ├── README.md │ ├── hocon2json.bats │ ├── jq2sh.bats │ ├── should_accept-calibration_holdout_fraction.bats │ ├── should_accept-calibration_holdout_fraction │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-calibration_holdout_query.bats │ ├── should_accept-calibration_holdout_query │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-calibration_holdout_query_over_fraction.bats │ ├── should_accept-calibration_holdout_query_over_fraction │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-calibration_observation_query.bats │ ├── should_accept-calibration_observation_query │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-ddlog_undeclared_relations.bats │ ├── should_accept-ddlog_undeclared_relations │ ├── app.ddlog │ ├── db.url │ └── input │ │ └── foo.tsv │ ├── should_accept-empty_app.bats │ ├── should_accept-empty_app │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_accept-extraction_only_app.bats │ ├── should_accept-extraction_only_app │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-extractor_style_cmd.bats │ ├── should_accept-extractor_style_cmd │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-extractor_style_json.skip │ ├── should_accept-extractor_style_json │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-extractor_style_sql.bats │ ├── should_accept-extractor_style_sql │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-extractor_style_tsv.bats │ ├── should_accept-extractor_style_tsv │ ├── db.url │ ├── deepdive.conf │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_and.bats │ ├── should_accept-factor_function_and │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_array_imply.bats │ ├── should_accept-factor_function_array_imply │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_equal.bats │ ├── should_accept-factor_function_equal │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_imply3.bats │ ├── should_accept-factor_function_imply3 │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_imply_n_ary.bats │ ├── should_accept-factor_function_imply_n_ary │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_imply_unary.bats │ ├── should_accept-factor_function_imply_unary │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_istrue.bats │ ├── should_accept-factor_function_istrue │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_linear.bats │ ├── should_accept-factor_function_linear │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_logical.bats │ ├── should_accept-factor_function_logical │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_negated_variable.bats │ ├── should_accept-factor_function_negated_variable │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_or.bats │ ├── should_accept-factor_function_or │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_function_ratio.bats │ ├── should_accept-factor_function_ratio │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_weight_float.bats │ ├── should_accept-factor_weight_float │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_weight_int.bats │ ├── should_accept-factor_weight_int │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_weight_unknown.bats │ ├── should_accept-factor_weight_unknown │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_weight_unknown_parameterized.bats │ ├── should_accept-factor_weight_unknown_parameterized │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-factor_weight_unknown_parameterized_with_dots.bats │ ├── should_accept-factor_weight_unknown_parameterized_with_dots │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-variable_boolean.bats │ ├── should_accept-variable_boolean │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_accept-variable_categorical.bats │ ├── should_accept-variable_categorical │ ├── db.url │ ├── deepdive.conf │ ├── input │ ├── schema.sql │ └── validate-compiled-config │ ├── should_reject-empty_dir.bats │ ├── should_reject-empty_dir │ └── .gitignore │ ├── should_reject-extractor_input_missing.bats │ ├── should_reject-extractor_input_missing │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-extractor_output_missing.bats │ ├── should_reject-extractor_output_missing │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-extractor_style_invalid.bats │ ├── should_reject-extractor_style_invalid │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-extractor_style_missing.bats │ ├── should_reject-extractor_style_missing │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-extractor_udf_missing.bats │ ├── should_reject-extractor_udf_missing │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-factor_function_equal_n_ary.bats │ ├── should_reject-factor_function_equal_n_ary │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-factor_function_equal_unary.bats │ ├── should_reject-factor_function_equal_unary │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-factor_function_istrue_n_ary.bats │ ├── should_reject-factor_function_istrue_n_ary │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-factor_function_malformed.bats │ ├── should_reject-factor_function_malformed │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-multiple_processes_same_output.bats │ ├── should_reject-multiple_processes_same_output │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-schema_variables_malformed.bats │ ├── should_reject-schema_variables_malformed │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-schema_variables_missing_column.bats │ ├── should_reject-schema_variables_missing_column │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── should_reject-variable_invalid_type.bats │ ├── should_reject-variable_invalid_type │ ├── db.url │ ├── deepdive.conf │ └── schema.sql │ ├── stub-app │ ├── db.url │ ├── input │ │ └── init.sh │ ├── schema.json │ └── schema.sql │ ├── test_environ.bash │ ├── testutil.jq │ └── update-bats-symlinks ├── database ├── db-driver │ ├── greenplum │ │ ├── db-analyze │ │ ├── db-assign_sequential_id │ │ ├── db-create-table │ │ ├── db-create-table-as │ │ ├── db-create-table-def │ │ ├── db-create-table-like │ │ ├── db-create-table-options │ │ ├── db-create-view-as │ │ ├── db-create_calibration_view │ │ ├── db-execute │ │ ├── db-generate_series │ │ ├── db-init │ │ ├── db-is_ready │ │ ├── db-load │ │ ├── db-parse │ │ ├── db-prompt │ │ ├── db-psql │ │ ├── db-query │ │ ├── db-query-tsj │ │ ├── db-supports_pg_lang │ │ ├── db-unload │ │ └── sql_for_column_types │ ├── mysql │ │ ├── db-analyze │ │ ├── db-assign_sequential_id │ │ ├── db-create-table │ │ ├── db-create-table-as │ │ ├── db-create-table-def │ │ ├── db-create-table-like │ │ ├── db-create-table-options │ │ ├── db-create-view-as │ │ ├── db-create_calibration_view │ │ ├── db-execute │ │ ├── db-generate_series │ │ ├── db-init │ │ ├── db-is_ready │ │ ├── db-load │ │ ├── db-parse │ │ ├── db-prompt │ │ ├── db-query │ │ └── db-unload │ ├── postgresql-xl │ │ ├── db-analyze │ │ ├── db-assign_sequential_id │ │ ├── db-create-table │ │ ├── db-create-table-as │ │ ├── db-create-table-def │ │ ├── db-create-table-like │ │ ├── db-create-table-options │ │ ├── db-create-view-as │ │ ├── db-create_calibration_view │ │ ├── db-execute │ │ ├── db-generate_series │ │ ├── db-init │ │ ├── db-is_ready │ │ ├── db-load │ │ ├── db-parse │ │ ├── db-prompt │ │ ├── db-psql │ │ ├── db-query │ │ ├── db-supports_pg_lang │ │ └── db-unload │ └── postgresql │ │ ├── README.md │ │ ├── db-analyze │ │ ├── db-assign_sequential_id │ │ ├── db-create-table │ │ ├── db-create-table-as │ │ ├── db-create-table-def │ │ ├── db-create-table-like │ │ ├── db-create-table-options │ │ ├── db-create-view-as │ │ ├── db-create_calibration_view │ │ ├── db-execute │ │ ├── db-generate_series │ │ ├── db-init │ │ ├── db-is_ready │ │ ├── db-load │ │ ├── db-parse │ │ ├── db-prompt │ │ ├── db-psql │ │ ├── db-query │ │ ├── db-query-tsj │ │ ├── db-supports_pg_lang │ │ ├── db-unload │ │ └── sql_for_column_types ├── deepdive-create ├── deepdive-db ├── deepdive-initdb ├── deepdive-load ├── deepdive-query ├── deepdive-relation ├── deepdive-reload ├── deepdive-sql ├── deepdive-unload ├── load-db-driver.sh ├── partition_id_range ├── pgtsv_to_json ├── test │ ├── corner_cases.bash │ ├── deepdive-relation.bats │ ├── pgtsv_to_json.bats │ ├── test_environ.bash │ └── tsv2tsj.bats ├── tsj2fmt ├── tsv2json ├── tsv2tsj └── tsv2tsj.pl ├── ddlib ├── ddlib │ ├── .gitignore │ ├── __init__.py │ ├── dd.py │ ├── gen_feats.py │ └── util.py ├── deepdive.py └── test │ ├── ddlib.bats │ ├── test.json.bz2 │ ├── test.py │ ├── test2.json.bz2 │ ├── test_dep.py │ ├── test_dep.test2.expected │ ├── test_environ.bash │ ├── tsj_extractor_decorator.bats │ ├── tsj_extractor_identity_udf.py │ ├── tsv_extractor_decorator.bats │ ├── tsv_extractor_identity_udf.py │ ├── with_ddlib.py │ └── without_ddlib.py ├── doc ├── .gitignore ├── .htaccess ├── .linkchecker.ignore-urls ├── Makefile ├── README ├── _config.yml ├── _config_2.x.yml ├── _includes │ ├── css │ │ ├── bootstrap.min.css │ │ ├── bs-docs-sidebar.css │ │ ├── font-awesome.min.css │ │ ├── landing.css │ │ └── pygment_trac.css │ ├── examples │ ├── googleanalytics.html │ └── js │ │ ├── bootstrap.min.js │ │ ├── jquery-1.10.1.min.js │ │ └── toc.js ├── _layouts │ └── default.html ├── _plugins │ ├── rewrite_github_links.rb │ └── rewrite_md_links.rb ├── assets │ ├── deepdive.css │ ├── deepdive.js │ ├── factor_graph.pdf │ ├── factor_graph.png │ ├── factor_graph.tex │ └── fonts │ │ ├── FontAwesome.otf │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.svg │ │ ├── fontawesome-webfont.ttf │ │ ├── fontawesome-webfont.woff │ │ ├── fontawesome-webfont.woff2 │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ └── glyphicons-halflings-regular.woff ├── browsing.md ├── calibration.md ├── changelog │ ├── 0.02-alpha.md │ ├── 0.03-alpha.md │ ├── 0.03.1-alpha.md │ ├── 0.03.2-alpha.md │ ├── 0.04-alpha.md │ ├── 0.04.1-alpha.md │ ├── 0.04.2-alpha.md │ ├── 0.05.01-alpha.md │ ├── 0.6.0-alpha.md │ ├── 0.7.0-alpha.md │ ├── 0.7.1-alpha.md │ └── 0.8.0-alpha.md ├── configuration.md ├── dashboard.md ├── debugging-udf.md ├── deepdiveapp.md ├── developer.md ├── development-cycle.md ├── distant_supervision.md ├── example-chunking.md ├── example-smoke.md ├── example-spouse.md ├── factor_graph_schema.md ├── faq.md ├── gen_feats.md ├── generating_negative_examples.md ├── google498f7e56a944f834.html ├── images │ ├── browsing_corpus.png │ ├── browsing_results.png │ ├── browsing_screenshot.png │ ├── browsing_with_presentation_template.png │ ├── browsing_without_presentation_template.png │ ├── carousel │ │ ├── dark-data.png │ │ ├── dark-data.psd │ │ ├── deepdive.png │ │ ├── deepdive.psd │ │ ├── dimmwitted.png │ │ ├── dimmwitted.psd │ │ ├── dimmwitted.xcf │ │ ├── memex.png │ │ ├── memex.psd │ │ ├── paleo.png │ │ ├── paleo.psd │ │ ├── paleo.xcf │ │ ├── support.png │ │ ├── support.psd │ │ └── support.xcf │ ├── coming_soon.png │ ├── dashboard │ │ ├── calibration_plot_report.png │ │ ├── homepage.png │ │ ├── run_snapshot.png │ │ ├── supervision_report.png │ │ ├── task_control.png │ │ ├── task_input.png │ │ ├── top_positive_features_report.png │ │ ├── trend.png │ │ └── trends.png │ ├── header_logo.png │ ├── logos │ │ ├── amfam.jpeg │ │ ├── chtc.png │ │ ├── darpa.jpg │ │ ├── google.png │ │ ├── lsvp.gif │ │ ├── moore.jpg │ │ ├── nsf.jpg │ │ ├── onr.png │ │ ├── sloan.jpg │ │ └── toshiba.jpg │ ├── mindtagger_screenshot.png │ ├── mindtagger_screenshot_export.png │ ├── mindtagger_screenshot_tags.png │ ├── mindtagger_screenshot_with_features.png │ ├── paleo │ │ ├── Pipeline3.pdf │ │ └── Pipeline3.png │ ├── showcase │ │ ├── PA145011114.png │ │ ├── genetics.jpg │ │ ├── memex-human-trafficking.jpg │ │ ├── paleontology.jpg │ │ └── wisci.png │ ├── spouse │ │ ├── dataflow.svg │ │ ├── has_spouse.png │ │ └── has_spouse.tsv │ └── walkthrough │ │ ├── dataflow.eps │ │ ├── dataflow.graffle │ │ ├── data.plist │ │ └── image1.pdf │ │ ├── dataflow.png │ │ ├── datamodel.graffle │ │ ├── datamodel.png │ │ ├── example_hasspouse.graffle │ │ ├── data.plist │ │ └── image1.pdf │ │ └── example_hasspouse.png ├── index.md ├── inference.md ├── installation.md ├── kbc.md ├── labeling.md ├── opendata │ ├── index.md │ └── schema.md ├── ops-compiling.md ├── ops-data.md ├── ops-execution.md ├── ops-model.md ├── overview.md ├── paleo.md ├── papers.md ├── params.json ├── quickstart.md ├── relation_extraction.md ├── reserved_tables.md ├── sampler.md ├── showcase │ ├── apps.md │ └── memex.md ├── using-docker.md ├── using-ec2.md ├── using-greenplum.md ├── using-pgxl.md ├── using-torque.md ├── writing-dataflow-ddlog.md ├── writing-model-ddlog.md └── writing-udf-python.md ├── examples ├── balance │ ├── .gitignore │ ├── README.md │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ └── input │ │ ├── no.tsv.sh │ │ ├── whatever.tsv │ │ └── yes.tsv.sh ├── census │ ├── .gitignore │ ├── README.md │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ ├── deepdive.conf.numbskull │ └── input │ │ └── adult.csv.sh ├── chain │ ├── .gitignore │ ├── README.md │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ └── input │ │ ├── heads.tsv.sh │ │ ├── tails.tsv.sh │ │ └── unk.tsv.sh ├── chunking │ ├── .gitignore │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ ├── input │ │ ├── init_words.sh │ │ ├── senna.filtered.tsv.bz2 │ │ ├── tags.tsv │ │ ├── test_null_terminated.txt │ │ ├── train_null_terminated.txt │ │ ├── vec_dims.tsv.sh │ │ ├── word_embedding.tsv.sh │ │ └── words_all_lower.txt │ ├── readme.md │ └── result │ │ ├── calibration.png │ │ ├── conlleval.pl │ │ ├── eval.sh │ │ └── evaluation_results.txt ├── multinomial │ ├── .gitignore │ └── run.sh ├── ocr │ ├── .gitignore │ ├── README.md │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ ├── feature-analysis.sh │ └── input │ │ ├── feature_names.tsv.sh │ │ ├── features.csv │ │ ├── init.sh │ │ ├── label1.csv │ │ ├── label2.csv │ │ └── raw │ │ ├── boolean-f52-c3-m620 │ │ ├── 10618.corrected_words.txt │ │ ├── 10618.feature_names.txt │ │ ├── 10618.features.txt │ │ ├── 10618.labels.txt │ │ ├── 10618.options.txt │ │ ├── 164.corrected_words.txt │ │ ├── 164.feature_names.txt │ │ ├── 164.features.txt │ │ ├── 164.labels.txt │ │ └── 164.options.txt │ │ ├── gen_feature_table.py │ │ ├── raw-words │ │ ├── 10618.output.txt │ │ ├── 164.output.txt │ │ └── feature_names.txt │ │ └── real2bool-feature.py ├── run.sh ├── smoke │ ├── .gitignore │ ├── README.md │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ └── input │ │ ├── friends.tsv │ │ ├── person.tsv │ │ ├── person_has_cancer.tsv │ │ └── person_smokes.tsv ├── spouse │ ├── .gitignore │ ├── DeepDive Tutorial - Extracting mentions of spouses from the news.ipynb │ ├── README.md │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ ├── input │ │ ├── articles-100.tsj.bz2 │ │ ├── articles-100.tsv.bz2 │ │ ├── articles-1000.tsj.bz2 │ │ ├── articles-1000.tsv.bz2 │ │ ├── articles.tsj.sh │ │ ├── sentences-100.tsj.bz2 │ │ ├── sentences-100.tsv.bz2 │ │ ├── sentences-1000.tsj.bz2 │ │ ├── sentences-1000.tsv.bz2 │ │ └── spouses_dbpedia.csv.bz2 │ ├── labeling │ │ ├── .gitignore │ │ ├── has_spouse-precision-with_features │ │ │ ├── has_spouse-with_features.csv │ │ │ ├── mindtagger.conf │ │ │ ├── sample-has_spouse-with_features.sql │ │ │ └── template-with_features.html │ │ └── has_spouse-precision │ │ │ ├── has_spouse.csv │ │ │ ├── mindtagger.conf │ │ │ ├── sample-has_spouse.sql │ │ │ ├── tags.json │ │ │ └── template.html │ ├── mindbender │ │ ├── extensions.coffee │ │ └── search-template │ │ │ ├── articles.html │ │ │ ├── has_spouse.html │ │ │ ├── person_mention.html │ │ │ ├── sentences.html │ │ │ ├── spouse_feature.html │ │ │ ├── spouse_label.html │ │ │ └── spouses_dbpedia.html │ └── udf │ │ ├── extract_spouse_features.py │ │ ├── map_person_mention.py │ │ ├── nlp_markup.sh │ │ └── supervise_spouse.py └── template │ ├── .gitignore │ ├── README.md │ ├── app.ddlog │ ├── db.url │ ├── deepdive.conf │ ├── input │ └── init.sh │ └── udf │ └── extractor_template.py ├── extern ├── .gitignore ├── bundle-all.conf ├── bundle-none.conf ├── bundle-runtime-dependencies.sh └── bundled │ ├── bash │ ├── bc │ ├── coreutils │ ├── graphviz │ ├── jq │ ├── moreutils │ ├── pbzip2 │ ├── perl-local-lib │ └── install.sh │ ├── postgresql │ └── install.sh │ ├── pv │ └── python-lib │ ├── install.sh │ └── requirements.txt ├── inference ├── deepdive-model ├── dimmwitted.build.sh ├── dimmwitted │ ├── .clang-format │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── doc │ │ ├── binary_format.md │ │ └── text_format.md │ ├── lib │ │ ├── gtest-1.7.0.zip │ │ ├── has-all-dep.sh │ │ ├── numactl-2.0.11.tar.gz │ │ └── tclap-1.2.1.tar.gz │ ├── src │ │ ├── bin2text.cc │ │ ├── bin2text.h │ │ ├── binary_format.cc │ │ ├── binary_format.h │ │ ├── cmd_parser.cc │ │ ├── cmd_parser.h │ │ ├── common.h │ │ ├── dimmwitted.cc │ │ ├── dimmwitted.h │ │ ├── factor.cc │ │ ├── factor.h │ │ ├── factor_graph.cc │ │ ├── factor_graph.h │ │ ├── gibbs_sampler.cc │ │ ├── gibbs_sampler.h │ │ ├── inference_result.cc │ │ ├── inference_result.h │ │ ├── main.cc │ │ ├── numa_nodes.cc │ │ ├── numa_nodes.h │ │ ├── text2bin.cc │ │ ├── text2bin.h │ │ ├── timer.cc │ │ ├── timer.h │ │ ├── variable.cc │ │ ├── variable.h │ │ ├── variational.cc │ │ ├── weight.cc │ │ └── weight.h │ └── test │ │ ├── .end_to_end_test.bats.template │ │ ├── .gtest.bats.template │ │ ├── README.md │ │ ├── bats.mk │ │ ├── biased_coin-performance.sh │ │ ├── biased_coin.bats │ │ ├── biased_coin.setup.sh │ │ ├── biased_coin │ │ ├── check_result │ │ ├── dw-args │ │ ├── factors.text2bin-args │ │ ├── factors.tsv │ │ ├── graph.meta │ │ ├── variables.tsv │ │ └── weights.tsv │ │ ├── biased_coin_continuous.bats │ │ ├── biased_coin_continuous │ │ ├── check_result │ │ ├── dw-args │ │ ├── factors.text2bin-args │ │ ├── factors.tsv │ │ ├── graph.meta │ │ ├── variables.tsv │ │ └── weights.tsv │ │ ├── biased_coin_truthiness.bats │ │ ├── biased_coin_truthiness │ │ ├── check_result │ │ ├── domains.tsv │ │ ├── dw-args │ │ ├── factors.text2bin-args │ │ ├── factors.tsv │ │ ├── graph.meta │ │ ├── variables.tsv │ │ └── weights.tsv │ │ ├── biased_coin_with_multinomial.bats │ │ ├── biased_coin_with_multinomial │ │ ├── check_result │ │ ├── dw-args │ │ ├── factors.text2bin-args │ │ ├── factors.tsv │ │ ├── graph.meta │ │ ├── variables.tsv │ │ └── weights.tsv │ │ ├── binary_format_test.bats │ │ ├── binary_format_test.cc │ │ ├── binary_format_test.setup.sh │ │ ├── domains │ │ └── domains.tsv │ │ ├── factor_graph_test.bats │ │ ├── factor_graph_test.cc │ │ ├── factor_graph_test.setup.sh │ │ ├── factor_test.bats │ │ ├── factor_test.cc │ │ ├── helpers.bash │ │ ├── loading_test.bats │ │ ├── loading_test.cc │ │ ├── loading_test.setup.sh │ │ ├── partial_observation.bats │ │ ├── partial_observation │ │ ├── check_result │ │ ├── dw-args │ │ ├── factors.text2bin-args │ │ ├── factors.tsv │ │ ├── graph.meta │ │ ├── variables.tsv │ │ └── weights.tsv │ │ ├── run_end_to_end.sh │ │ ├── sampler_test.bats │ │ ├── sampler_test.cc │ │ ├── sparse_domains.bats │ │ ├── sparse_domains │ │ ├── check_result │ │ ├── domains.tsv │ │ ├── dw-args │ │ ├── factors.text2bin-args │ │ ├── factors.tsv │ │ ├── factors2.text2bin-args │ │ ├── factors2.tsv │ │ ├── graph.meta │ │ ├── variables.tsv │ │ └── weights.tsv │ │ ├── sparse_multinomial2.bats │ │ ├── sparse_multinomial2 │ │ ├── check_result │ │ ├── domains.tsv │ │ ├── dw-args │ │ ├── factors1.text2bin-args │ │ ├── factors1.tsv │ │ ├── factors2.text2bin-args │ │ ├── factors2.tsv │ │ ├── factors2a.text2bin-args │ │ ├── factors2a.tsv │ │ ├── graph.meta │ │ ├── variables.tsv │ │ ├── weights1.tsv │ │ ├── weights2.tsv │ │ └── weights2a.tsv │ │ ├── test_main.cc │ │ ├── text2bin.bats │ │ └── text2bin │ │ ├── dd_factors.bin.txt │ │ ├── dd_factors.txt │ │ ├── dd_variables.bin.txt │ │ ├── dd_variables.txt │ │ ├── dd_weights.bin.txt │ │ └── dd_weights.txt ├── restore_partitioned_vids ├── run-sampler └── test │ └── env.bash ├── runner ├── compute-driver │ ├── README.md │ ├── local │ │ └── compute-execute │ └── torque │ │ ├── commons │ │ ├── compute-execute │ │ ├── compute-remote-status │ │ ├── compute-remote-submit │ │ ├── compute-remote-taskinit │ │ ├── compute-setup │ │ ├── compute-teardown │ │ ├── sched_kill │ │ ├── sched_status │ │ ├── sched_submit │ │ └── ssh_with_env ├── computers-default.conf ├── deepdive-compute ├── deepdive-do ├── deepdive-do-getopts.sh ├── deepdive-done ├── deepdive-mark ├── deepdive-plan ├── deepdive-redo ├── deepdive-run ├── format_timestamp ├── load-compute-driver.sh ├── mark_done ├── mkmimo.build.sh ├── ps_descendants ├── reset_timestamp ├── resolve-args-to-do.sh ├── restore_timestamp ├── show_progress └── test │ ├── ps_descendants.bats │ └── test_environ.bash ├── sandbox ├── .dockerignore ├── .gitignore ├── Dockerfile └── docker-compose.yml ├── shell ├── README.md ├── deepdive ├── deepdive-env ├── deepdive-help ├── deepdive-version ├── deepdive-whereis ├── deepdive_bash_completion.sh ├── error ├── error-from-file ├── escape4sh ├── find-deepdive-app ├── jq ├── logging-with-ts ├── make ├── parse-url.sh ├── test │ ├── deepdive-whereis.bats │ ├── escape4sh.bats │ └── test_environ.bash ├── usage └── warning ├── stage.sh ├── test ├── .gitignore ├── bats.mk ├── checkstyle.sh ├── enumerate-tests.sh ├── env.sh ├── greenplum │ ├── biased_coin_example │ ├── biased_coin_example.bats │ ├── broken_example │ ├── broken_example.bats │ ├── chunking_example │ ├── chunking_example.bats │ ├── db-assign_sequential_id.bats │ ├── deepdive_create.bats │ ├── deepdive_create_greenplum.bats │ ├── deepdive_load │ ├── deepdive_sql.bats │ ├── deepdive_unload.bats │ ├── should-work.sh │ ├── spouse_example │ ├── spouse_example.bats │ ├── spouse_example.helpers.sh │ ├── spouse_example_incremental.bats │ └── test_environ.bash ├── postgresql-xl │ ├── biased_coin_example │ ├── biased_coin_example.bats │ ├── chunking_example │ ├── chunking_example.bats │ ├── db-assign_sequential_id.bats │ ├── deepdive_create.bats │ ├── deepdive_load │ ├── deepdive_load.bats │ ├── deepdive_sql.bats │ ├── deepdive_unload.bats │ ├── partition_id_range.bats │ ├── should-work.sh │ ├── spouse_example │ ├── spouse_example.bats │ ├── spouse_example.helpers.sh │ └── test_environ.bash ├── postgresql │ ├── Dockerfile.postgres │ ├── biased_coin_example.bats │ ├── biased_coin_example │ │ ├── app.ddlog │ │ ├── db.url │ │ ├── deepdive.conf │ │ └── input │ │ │ └── flips.tsv │ ├── broken_example.bats │ ├── broken_example │ │ ├── db.url │ │ ├── deepdive.conf │ │ └── schema.sql │ ├── chunking_example │ ├── chunking_example.bats │ ├── db-assign_sequential_id.bats │ ├── deepdive_create.bats │ ├── deepdive_load.bats │ ├── deepdive_load │ │ ├── test.app │ │ │ ├── .gitignore │ │ │ ├── app.ddlog │ │ │ ├── db.url │ │ │ └── input │ │ │ │ └── variable_table_declared_in_app.tsv │ │ ├── test.csv │ │ └── test.tsv │ ├── deepdive_sql.bats │ ├── deepdive_unload.bats │ ├── failing_udf.bats │ ├── failing_udf │ │ ├── app.ddlog │ │ ├── db.url │ │ └── deepdive.conf │ ├── load-db-driver.bats │ ├── multinomial_example │ ├── multinomial_example.bats │ ├── partition_id_range.bats │ ├── should-work.sh │ ├── smoke_example │ ├── smoke_example.bats │ ├── spouse_example │ ├── spouse_example.bats │ ├── spouse_example.helpers.sh │ ├── spouse_example_incremental.bats │ └── test_environ.bash ├── test-installed.sh └── test_environ.bash └── util ├── build ├── build-submodule-if-needed ├── build-submodules.mk ├── decrypt-travis-then-update-github-releases ├── generate-build-info.sh ├── test │ ├── compare_json │ ├── cosine_similarity_of │ ├── inner_product_of │ ├── l2_distance_of │ ├── normalize_by_sum │ ├── probability_distribution_of │ ├── sqrt │ ├── squares │ ├── sum │ └── unit_vector ├── update-github-release-asset └── upload-github-release-asset ├── calibration.plg ├── calibration.py ├── draw_calibration_plot ├── install.sh ├── install ├── install.Mac.sh ├── install.RedHat.sh ├── install.Ubuntu.pgxl.sh └── install.Ubuntu.sh ├── mindbender-wrapper.sh ├── mindbender.build.sh ├── nlp ├── deepdive-corenlp ├── deepdive-corenlp-curl ├── deepdive-corenlp-install ├── deepdive-corenlp-installed ├── deepdive-corenlp-is-running ├── deepdive-corenlp-parse-tsj ├── deepdive-corenlp-sentences-tsj ├── deepdive-corenlp-server-url ├── deepdive-corenlp-start ├── deepdive-corenlp-stop ├── deepdive-corenlp-unique-port ├── deepdive-corenlp-watch-log └── tsj2corenlp-http-reqs ├── partition_integers └── test ├── deepdive_corenlp.bats ├── partition_integers.bats └── test_environ.bash /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !.git 3 | -------------------------------------------------------------------------------- /.travis.tar.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/.travis.tar.enc -------------------------------------------------------------------------------- /compiler/compile-check/compile-check-2-codegen_no_path_collision: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # compile-check-2-codegen_no_path_collision -- Checks if all code are designated to distinct file paths 3 | ## 4 | set -euo pipefail 5 | 6 | # find duplicate paths in the compiled codegen blocks 7 | cat "$@" | jq -c -r '.path' | sort | uniq -d | ( 8 | set +e 9 | while read -r path; do 10 | error "$path: code generated more than once" 11 | done 12 | # exits non-zero by hitting an error on non-empty input 13 | ) 14 | -------------------------------------------------------------------------------- /compiler/compile-check/util.jq: -------------------------------------------------------------------------------- 1 | ../compile-config/util.jq -------------------------------------------------------------------------------- /compiler/compile-code/compile-code-json_extractor: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | # compile-code-json_extractor -- TODO Compiles json_extractors in the normalized JSON into code executable by the runner 3 | ## 4 | .deepdive_ as $deepdive 5 | 6 | | $deepdive.execution.processes | to_entries[] 7 | | .value.name = .key | .value 8 | | select(.style == "json_extractor") 9 | | error("\(.name): Compilation of \(.style) unsupported") 10 | -------------------------------------------------------------------------------- /compiler/compile-code/constants.jq: -------------------------------------------------------------------------------- 1 | ../compile-config/constants.jq -------------------------------------------------------------------------------- /compiler/compile-code/sql.jq: -------------------------------------------------------------------------------- 1 | ../compile-config/sql.jq -------------------------------------------------------------------------------- /compiler/compile-code/util.jq: -------------------------------------------------------------------------------- 1 | ../compile-config/util.jq -------------------------------------------------------------------------------- /compiler/compile-config/compile-config-0.01-parse_calibration: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | # compile-config-0.02-parse_calibration -- Parses deepdive.calibration settings 3 | ## 4 | 5 | .deepdive_ as $deepdive 6 | 7 | | .deepdive_.calibration |= 8 | # holdout_query should have priority when both holdout_query and holdout_fraction are specified 9 | ( if .holdout_query == null or .holdout_fraction == null then . 10 | else .holdout_fraction = null 11 | end) 12 | -------------------------------------------------------------------------------- /compiler/compile-config/util.jq: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ## jq helper functions for DeepDive Code Compiler 3 | ############################################################################### 4 | 5 | def merge(objects): reduce objects as $es ({}; . + $es); 6 | 7 | def trimWhitespace: gsub("^\\s+|\\s+$"; ""; "m"); 8 | 9 | def nullOr(expr): if type == "null" then null else expr end; 10 | -------------------------------------------------------------------------------- /compiler/ddlog.build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # How to build the sampler submodule 3 | set -euo pipefail 4 | cd "${0%.build.sh}" 5 | 6 | make ddlog.jar 7 | -------------------------------------------------------------------------------- /compiler/ddlog/.gitignore: -------------------------------------------------------------------------------- 1 | /ddlog.jar 2 | target 3 | 4 | /test/*-test/**/*.bats 5 | /test/*-test/**/*.actual 6 | 7 | /.idea 8 | -------------------------------------------------------------------------------- /compiler/ddlog/.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - "2.11.7" 4 | jdk: 5 | - oraclejdk8 6 | 7 | sudo: false # to use container-based infrastructure 8 | addons: 9 | apt: 10 | packages: 11 | - wdiff 12 | cache: 13 | directories: 14 | - $HOME/.ivy2/cache 15 | 16 | install: 17 | - make test-build 18 | script: 19 | - make test 20 | after_success: 21 | - sbt coverageReport coveralls 22 | -------------------------------------------------------------------------------- /compiler/ddlog/build.sbt: -------------------------------------------------------------------------------- 1 | scalaVersion := "2.11.7" 2 | 3 | scalacOptions ++= Seq("-feature") 4 | 5 | libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4" 6 | 7 | libraryDependencies += "org.apache.commons" % "commons-lang3" % "3.4" 8 | 9 | libraryDependencies += "com.github.scopt" %% "scopt" % "3.3.0" 10 | 11 | resolvers += Resolver.sonatypeRepo("public") 12 | -------------------------------------------------------------------------------- /compiler/ddlog/project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0") 2 | -------------------------------------------------------------------------------- /compiler/ddlog/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.7 2 | -------------------------------------------------------------------------------- /compiler/ddlog/project/build.sbt: -------------------------------------------------------------------------------- 1 | resolvers += Classpaths.sbtPluginReleases 2 | 3 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") // for sbt-0.13.5 or higher 4 | 5 | addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0") // for sbt-0.13.5 or higher 6 | -------------------------------------------------------------------------------- /compiler/ddlog/project/sbt/sbt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SBT_OPTS="-Xms512M -Xmx1536M -Xss1M -XX:+CMSClassUnloadingEnabled" 3 | java $SBT_OPTS -jar `dirname $0`/sbt-launch.jar "$@" 4 | -------------------------------------------------------------------------------- /compiler/ddlog/project/sbt/sbt-launch.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/compiler/ddlog/project/sbt/sbt-launch.jar -------------------------------------------------------------------------------- /compiler/ddlog/test/checker-test/supervision_binary_variable/input.ddlog: -------------------------------------------------------------------------------- 1 | Q?(x int). 2 | R(x int, label boolean). 3 | 4 | Q(x) = label :- R(x, label). 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/checker-test/supervision_multinomial/input.ddlog: -------------------------------------------------------------------------------- 1 | Q?(@key x int, category int). 2 | R(x int, label int). 3 | 4 | Q(x, label) = TRUE :- R(x, label). 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/checker-test/supervision_with_undeclared_relation/input.ddlog: -------------------------------------------------------------------------------- 1 | Q?(x int). 2 | S(y text, l boolean). 3 | T(x int, y text). 4 | 5 | # XXX R is not declared but implicitly defined 6 | # R(x int, label boolean). 7 | R(x, l) :- T(x,y), S(y,l). 8 | 9 | # ideally, checker should infer the type of undeclared relation to check S.l's type is boolean 10 | Q(x) = label :- R(x, label). 11 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/column_dismatch/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] "c": number of columns in the query does not match number of columns in the schema 2 | Q(y, z) :- 3 | a(x), 4 | c(x, y, z, w). 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/column_dismatch/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | b(k int, p text, q text, r int). 3 | c(s text, n int, t text). 4 | 5 | Q(y, z) :- a(x), c(x,y,z,w). -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/composing_boolean_categorical_vars/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] None or all variables must be categorical 2 | @weight(1.0) 3 | P(x) => Q(y, c) :- 4 | R(x), 5 | R(y), 6 | C(c). 7 | 8 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/composing_boolean_categorical_vars/input.ddlog: -------------------------------------------------------------------------------- 1 | P?(x int). 2 | Q?(@key y int, category text). 3 | 4 | @weight(1.0) 5 | P(x) => Q(y, c) :- R(x), R(y), C(c). 6 | 7 | R(x int). 8 | C(c text). 9 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/invalid_variable_schema/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] variable relation contains reserved column "dd_label" 2 | c?(dd_label int, 3 | s text, 4 | n int, 5 | t text). 6 | 7 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/invalid_variable_schema/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | b(k int, p text, q text, r int). 3 | c?(dd_label int, s text, n int, t text). 4 | 5 | Q(y, z) :- a(x), d(x,y,z,w). 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/optional_modifier/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] One and only one atom should be supplied in OPTIONAL modifier 2 | Q(y, z) :- 3 | a(x), 4 | OPTIONAL[x > 1]. 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/optional_modifier/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | b(k int, p text, q text, r int). 3 | c(s text, n int, t text). 4 | 5 | Q(y, z) :- a(x), OPTIONAL[x > 1]. -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/repated_rule_name/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] @name("foo") repeated 2 times 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/repated_rule_name/input.ddlog: -------------------------------------------------------------------------------- 1 | S(a int). 2 | Q?(x int). 3 | 4 | # disallow repeating the same @name for more than one inference rule 5 | @name("foo") 6 | @weight(a) 7 | Q(a) :- S(a), a > 1000. 8 | 9 | @name("foo") 10 | @weight(1.0) 11 | Q(a) :- S(a), a = 0. 12 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_1/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] Variable y must have bindings 2 | Q(y) :- 3 | a(x). 4 | 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_1/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | 3 | Q(y) :- a(x). -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_2/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] Variable x must have bindings 2 | Q(y) :- 3 | a(y), 4 | x > 100. 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_2/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | 3 | Q(y) :- a(y), x > 100. -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_3/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] Variable y must have bindings 2 | @weight(y) 3 | b(x) :- 4 | a(x). 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_3/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | b?(x int). 3 | 4 | @weight(y) 5 | b(x) :- a(x). -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_4/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] Variable y, z must have bindings 2 | Q(y, z) :- 3 | a(x). 4 | 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_4/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | 3 | Q(y,z) :- a(x). 4 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_5/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] Variable z must have bindings 2 | @weight(z) 3 | b(x) => b(y) :- 4 | x < y. 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/unbound_variable_5/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | b?(x int). 3 | 4 | # these are fine 5 | b(x) :- a(x). 6 | 7 | @weight(y) 8 | b(x) => b(y) :- x < y. 9 | 10 | # but not this 11 | @weight(z) 12 | b(x) => b(y) :- x < y. 13 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/undefined_function/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] function "ext" is not defined 2 | Q += ext(x) :- 3 | a(x). 4 | 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/undefined_function/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | b(k int, p text, q text, r int). 3 | c(s text, n int, t text). 4 | 5 | Q += ext(x) :- a(x). -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/undefined_relation/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] relation "d" is not defined 2 | Q(y, z) :- 3 | a(x), 4 | d(x, y, z, w). 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/undefined_relation/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | b(k int, p text, q text, r int). 3 | c(s text, n int, t text). 4 | 5 | Q(y, z) :- a(x), d(x,y,z,w). -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/undefined_relation_2/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] relation "Q" is not defined 2 | @weight(1) 3 | Q(x) :- 4 | a(x). 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/undefined_relation_2/input.ddlog: -------------------------------------------------------------------------------- 1 | a(k int). 2 | 3 | @weight(1) 4 | Q(x) :- a(x). -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/weight/compile-error.expected: -------------------------------------------------------------------------------- 1 | [error] Weight variables can contain at most one constant 2 | @weight(1, 2) 3 | a(x) :- 4 | b(x, y, _, _). 5 | 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/compile-error-test/weight/input.ddlog: -------------------------------------------------------------------------------- 1 | a?(k int). 2 | b(k int, p text, q text, r int). 3 | c(s text, n int, t text). 4 | 5 | @weight(1,2) 6 | a(x) :- b(x, y, _, _). -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/arrays/input.ddlog: -------------------------------------------------------------------------------- 1 | R(x int[]). 2 | 3 | S(x int, 4 | y int[]). 5 | 6 | T(x[3], z) :- R(x), S(z, y), x[1] = y[1], x[2] = y[2]. 7 | 8 | U(z) :- R(x), S(z, y), z + f(z+1)[z + x[1]] = y[z]. 9 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/arrays/print.expected: -------------------------------------------------------------------------------- 1 | R(x int[]). 2 | 3 | S(x int, 4 | y int[]). 5 | 6 | T(x[3], z) :- 7 | R(x), 8 | S(z, y), 9 | x[1] = y[1], 10 | x[2] = y[2]. 11 | 12 | U(z) :- 13 | R(x), 14 | S(z, y), 15 | (z + f(z + 1)[(z + x[1])]) = y[z]. 16 | 17 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/chunking_example/input.ddlog: -------------------------------------------------------------------------------- 1 | ../../../examples/chunking.ddlog -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/comments/print.expected: -------------------------------------------------------------------------------- 1 | R(a int). 2 | 3 | S(a int, 4 | b text, 5 | c float). 6 | 7 | R(x) :- 8 | S(x, y, z). 9 | 10 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/factor_functions/input.ddlog: -------------------------------------------------------------------------------- 1 | R(a int, b int, c int). 2 | S?(x int). 3 | Q?(x int). 4 | 5 | @weight(y) 6 | Q(x) => Q(y) :- R(x, y, z). 7 | 8 | @weight(y) 9 | Q(x) = !Q(y) :- R(x, y, z). 10 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/factor_functions/print.expected: -------------------------------------------------------------------------------- 1 | R(a int, 2 | b int, 3 | c int). 4 | 5 | S?(x int). 6 | 7 | Q?(x int). 8 | 9 | @weight(y) 10 | Q(x) => Q(y) :- 11 | R(x, y, z). 12 | 13 | @weight(y) 14 | Q(x) = !Q(y) :- 15 | R(x, y, z). 16 | 17 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/if-then-else/input.ddlog: -------------------------------------------------------------------------------- 1 | labels_resolved(x INT, l INT). 2 | q?(x INT). 3 | 4 | q(x) = if l > 0 then TRUE 5 | else if l < 0 then FALSE 6 | else NULL 7 | end :- labels_resolved(x, l). 8 | 9 | labels(x INT, l BOOLEAN). 10 | p(x, if abs(l) > 10 then TRUE else FALSE end) :- labels(x, l). 11 | p(x, if x % 10 = 0 then FALSE end) :- labels(x, l). 12 | 13 | labels_resolved(x, SUM(if label IS NULL then 0 else if label = TRUE then 1 else -1 end)) :- 14 | labels(x, label). 15 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/logical_rules/input.ddlog: -------------------------------------------------------------------------------- 1 | data(x int, y int). 2 | P?(x int). 3 | Q?(x int). 4 | 5 | P(x) :- data(x, 1). 6 | Q(x) :- data(x, 2). 7 | 8 | @weight(1) 9 | P(x) => Q(x) :- TRUE. 10 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/logical_rules/print.expected: -------------------------------------------------------------------------------- 1 | data(x int, 2 | y int). 3 | 4 | P?(x int). 5 | 6 | Q?(x int). 7 | 8 | P(x) :- 9 | data(x, 1). 10 | 11 | Q(x) :- 12 | data(x, 2). 13 | 14 | @weight(1) 15 | P(x) => Q(x) :- 16 | true. 17 | 18 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/many_joins/input.ddlog: -------------------------------------------------------------------------------- 1 | R?(x text). 2 | 3 | S(x text). 4 | 5 | T(x text, 6 | f text). 7 | 8 | U(x text, 9 | l text). 10 | 11 | R(x) :- 12 | S(x), 13 | T(x, f), 14 | U(x, l). 15 | 16 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/multibody/input.ddlog: -------------------------------------------------------------------------------- 1 | R(a int, b int). 2 | S(a int, b int, c int). 3 | T(a int, b int, c int). 4 | Q(x int, z int). 5 | 6 | Q(x, y) :- R(x, y); R(x, y), S(y, z, w); S(y, x, w), T(x, z, w). -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/ocr_example/input.ddlog: -------------------------------------------------------------------------------- 1 | ../../../examples/ocr.ddlog -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/semantics/input.ddlog: -------------------------------------------------------------------------------- 1 | R?(x text). 2 | S?(x text). 3 | T(x text). 4 | 5 | @weight(3) 6 | @semantics("linear") 7 | R(x) => S(x) :- T(x). -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/semantics/print.expected: -------------------------------------------------------------------------------- 1 | R?(x text). 2 | 3 | S?(x text). 4 | 5 | T(x text). 6 | 7 | @weight(3) 8 | @semantics("linear") 9 | R(x) => S(x) :- 10 | T(x). 11 | 12 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/smoke_example/input.ddlog: -------------------------------------------------------------------------------- 1 | ../../../examples/smoke.ddlog -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/spouse_example/input.ddlog: -------------------------------------------------------------------------------- 1 | ../../../examples/spouse_example.ddlog -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/views/export-schema.expected: -------------------------------------------------------------------------------- 1 | {"relations" : {"R" : {"columns" : {"a" : {"type" : "int", "index" : 0}, "b" : {"type" : "int", "index" : 1}}}, "A" : {"type" : "view", "columns" : {"column_0" : {"type" : "UNKNOWN", "index" : 0}, "column_1" : {"type" : "UNKNOWN", "index" : 1}}}, "B" : {"type" : "view", "columns" : {"column_0" : {"type" : "UNKNOWN", "index" : 0}, "column_1" : {"type" : "UNKNOWN", "index" : 1}}}}} 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/views/input.ddlog: -------------------------------------------------------------------------------- 1 | R(a int, b int). 2 | 3 | A(x, x + y) :- R(x, y). 4 | B(x, z) :- A(x, y), R(y, z). -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/views/print.expected: -------------------------------------------------------------------------------- 1 | R(a int, 2 | b int). 3 | 4 | A(x, x + y) :- 5 | R(x, y). 6 | 7 | B(x, z) :- 8 | A(x, y), 9 | R(y, z). 10 | 11 | -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/weights/input.ddlog: -------------------------------------------------------------------------------- 1 | a?(k int). 2 | c?(k int). 3 | d?(k int). 4 | e?(k int). 5 | f?(k int). 6 | b(k int, p int, q text). 7 | 8 | @weight(x + y) 9 | a(x) :- b(x,y,_). 10 | 11 | @weight("string") 12 | c(x) :- b(x,y,_). 13 | 14 | @weight(x, y) 15 | d(x) :- b(x,y,_). 16 | 17 | @weight(-10) 18 | e(x) :- b(x,y,_). 19 | 20 | @weight(-0.3) 21 | f(x) :- b(x,y,_). -------------------------------------------------------------------------------- /compiler/ddlog/test/expected-output-test/weights/print.expected: -------------------------------------------------------------------------------- 1 | a?(k int). 2 | 3 | c?(k int). 4 | 5 | d?(k int). 6 | 7 | e?(k int). 8 | 9 | f?(k int). 10 | 11 | b(k int, 12 | p int, 13 | q text). 14 | 15 | @weight(x + y) 16 | a(x) :- 17 | b(x, y, _). 18 | 19 | @weight("string") 20 | c(x) :- 21 | b(x, y, _). 22 | 23 | @weight(x, y) 24 | d(x) :- 25 | b(x, y, _). 26 | 27 | @weight(-10) 28 | e(x) :- 29 | b(x, y, _). 30 | 31 | @weight(-0.3) 32 | f(x) :- 33 | b(x, y, _). 34 | 35 | -------------------------------------------------------------------------------- /compiler/ddlog/test/parse-error-test/annotation_semantics/input.ddlog: -------------------------------------------------------------------------------- 1 | @weight(3) 2 | @semantics("unknown") 3 | Q(x) => P(x) :- R(x). 4 | -------------------------------------------------------------------------------- /compiler/ddlog/test/parse-error-test/annotation_semantics/parse-error.expected: -------------------------------------------------------------------------------- 1 | [error] unknown: unrecognized @semantics value 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/parse-error-test/nested_multiline_comments/input.ddlog: -------------------------------------------------------------------------------- 1 | /* Multi-line comments are not supported */ 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/parse-error-test/nested_multiline_comments/parse-error.expected: -------------------------------------------------------------------------------- 1 | [error] test/parse-error-test/nested_multiline_comments/input.ddlog[1.1] failure: string matching regex `\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*' expected but `/' found 2 | 3 | /* Multi-line comments are not supported */ 4 | ^ 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/app.ddlog: -------------------------------------------------------------------------------- 1 | ../../../examples/spouse_example.ddlog -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-filtered.ddlogq: -------------------------------------------------------------------------------- 1 | ?- articles(id, content), content LIKE "% wife %". 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-filtered.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | , R0.text AS "content" 3 | FROM articles R0 4 | WHERE R0.text LIKE '% wife %' 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-ordered-revsered.ddlogq: -------------------------------------------------------------------------------- 1 | @order_by(dir="DESC") id ?- articles(id, _). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-ordered-revsered.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | FROM articles R0 3 | ORDER BY R0.article_id DESC 4 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-ordered-revsered2.ddlogq: -------------------------------------------------------------------------------- 1 | @order_by("DESC") id ?- articles(id, _). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-ordered-revsered2.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | FROM articles R0 3 | ORDER BY R0.article_id DESC 4 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-ordered.ddlogq: -------------------------------------------------------------------------------- 1 | @order_by id ?- articles(id, _). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-ordered.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | FROM articles R0 3 | ORDER BY R0.article_id ASC 4 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-parens.ddlogq: -------------------------------------------------------------------------------- 1 | (id) ?- articles(id, _). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-parens.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | FROM articles R0 3 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-without_parens.ddlogq: -------------------------------------------------------------------------------- 1 | id ?- articles(id, _). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids-without_parens.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | FROM articles R0 3 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids.ddlogq: -------------------------------------------------------------------------------- 1 | ?- articles(id, _). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/article_ids.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | FROM articles R0 3 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/count-has_spouse.ddlogq: -------------------------------------------------------------------------------- 1 | COUNT(r) ?- has_spouse(r). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/count-has_spouse.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT COUNT(R0.relation_id) AS "COUNT(r)" 2 | FROM has_spouse R0 3 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/histogram-num_candidates-per-article-ordered.ddlogq: -------------------------------------------------------------------------------- 1 | num_candidates_by_doc(doc, COUNT(candidate)) :- 2 | has_spouse_candidates(_,_,sentence,_,candidate,_), 3 | sentences(doc, _,_,_,_,_,_,_, sentence). 4 | 5 | @order_by num_candidates, COUNT(doc) ?- num_candidates_by_doc(doc, num_candidates). 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/histogram-num_candidates-per-article-ordered.ddlogq.expected: -------------------------------------------------------------------------------- 1 | CREATE TEMPORARY TABLE num_candidates_by_doc AS 2 | SELECT R1.document_id AS column_0 3 | , COUNT(R0.relation_id) AS column_1 4 | FROM has_spouse_candidates R0 5 | , sentences R1 6 | WHERE R1.sentence_id = R0.sentence_id 7 | GROUP BY R1.document_id; 8 | 9 | SELECT R0.column_1 AS "num_candidates" 10 | , COUNT(R0.column_0) AS "COUNT(doc)" 11 | FROM num_candidates_by_doc R0 12 | GROUP BY R0.column_1 13 | ORDER BY R0.column_1 ASC 14 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/histogram-num_candidates-per-article-ordered_multikey.ddlogq: -------------------------------------------------------------------------------- 1 | num_candidates_by_doc(doc, COUNT(candidate)) :- 2 | has_spouse_candidates(_,_,sentence,_,candidate,_), 3 | sentences(doc, _,_,_,_,_,_,_, sentence). 4 | 5 | @order_by num_candidates, @order_by(priority=-1, dir="DESC") COUNT(doc) ?- num_candidates_by_doc(doc, num_candidates). 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/histogram-num_candidates-per-article.ddlogq: -------------------------------------------------------------------------------- 1 | num_candidates_by_doc(doc, COUNT(candidate)) :- 2 | has_spouse_candidates(_,_,sentence,_,candidate,_), 3 | sentences(doc, _,_,_,_,_,_,_, sentence). 4 | 5 | num_candidates, COUNT(doc) ?- num_candidates_by_doc(doc, num_candidates). 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/histogram-num_candidates-per-article.ddlogq.expected: -------------------------------------------------------------------------------- 1 | CREATE TEMPORARY TABLE num_candidates_by_doc AS 2 | SELECT R1.document_id AS column_0 3 | , COUNT(R0.relation_id) AS column_1 4 | FROM has_spouse_candidates R0 5 | , sentences R1 6 | WHERE R1.sentence_id = R0.sentence_id 7 | GROUP BY R1.document_id; 8 | 9 | SELECT R0.column_1 AS "num_candidates" 10 | , COUNT(R0.column_0) AS "COUNT(doc)" 11 | FROM num_candidates_by_doc R0 12 | GROUP BY R0.column_1 13 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/histogram-supervision.ddlogq: -------------------------------------------------------------------------------- 1 | label, COUNT(candidate) ?- has_spouse_candidates(candidate,_,_,_,_,label). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/histogram-supervision.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.is_true AS "label" 2 | , COUNT(R0.person1_id) AS "COUNT(candidate)" 3 | FROM has_spouse_candidates R0 4 | GROUP BY R0.is_true 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/incomplete.ddlogq: -------------------------------------------------------------------------------- 1 | ?- foo 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/incomplete.ddlogq-error.expected: -------------------------------------------------------------------------------- 1 | [error] [1.7] failure: `.' expected but end of source found 2 | 3 | ?- foo 4 | ^ 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/redefinition.ddlogq: -------------------------------------------------------------------------------- 1 | has_spouse(doc, COUNT(candidate)) :- 2 | has_spouse_candidates(_,_,sentence,_,candidate,_), 3 | sentences(doc, _,_,_,_,_,_,_, sentence). 4 | 5 | num_candidates, COUNT(doc) ?- has_spouse(doc, num_candidates). 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/redefinition.ddlogq-error.expected: -------------------------------------------------------------------------------- 1 | [error] Following relations must not be redefined: 'has_spouse' 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/repeated-vars.ddlogq: -------------------------------------------------------------------------------- 1 | ?- articles(id1, content), articles(id2, content). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/repeated-vars.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id1" 2 | , R0.text AS "content" 3 | , R1.article_id AS "id2" 4 | FROM articles R0 5 | , articles R1 6 | WHERE R1.text = R0.text 7 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/schema-check.ddlogq: -------------------------------------------------------------------------------- 1 | ?- articles(id). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/schema-check.ddlogq-error.expected: -------------------------------------------------------------------------------- 1 | [error] "articles": number of columns in the query does not match number of columns in the schema 2 | (id) ?- 3 | articles(id). 4 | 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/undefined.ddlogq: -------------------------------------------------------------------------------- 1 | ?- nonexistent(foo,_,bar,_,_). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/undefined.ddlogq-error.expected: -------------------------------------------------------------------------------- 1 | [error] relation "nonexistent" is not defined 2 | (foo, bar) ?- 3 | nonexistent(foo, _, bar, _, _). 4 | 5 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/union-no-common-vars.ddlogq: -------------------------------------------------------------------------------- 1 | ?- articles(id, _); articles(_, content). 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/union-no-common-vars.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT * 2 | FROM articles R0 3 | UNION ALL 4 | SELECT * 5 | FROM articles R0 6 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/union.ddlogq: -------------------------------------------------------------------------------- 1 | ?- articles(id, _), id < 100; articles(id, _), id > 100000. 2 | -------------------------------------------------------------------------------- /compiler/ddlog/test/query-test/spouse_example/union.ddlogq.expected: -------------------------------------------------------------------------------- 1 | SELECT R0.article_id AS "id" 2 | FROM articles R0 3 | WHERE R0.article_id < 100 4 | UNION ALL 5 | SELECT R0.article_id AS "id" 6 | FROM articles R0 7 | WHERE R0.article_id > 100000 8 | -------------------------------------------------------------------------------- /compiler/deepdive-default.conf: -------------------------------------------------------------------------------- 1 | deepdive { 2 | 3 | sampler.partitions: 1 4 | sampler.sampler_cmd: "sampler-dw" 5 | sampler.sampler_args: "-l 100 -i 300 --alpha 0.01 --sample_evidence" 6 | 7 | } 8 | -------------------------------------------------------------------------------- /compiler/test/.gitignore: -------------------------------------------------------------------------------- 1 | /*/run 2 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_fraction.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_fraction/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_fraction/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.calibration.holdout_fraction: 0.25 2 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_fraction/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_fraction/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("calibration settings"; .deepdive_.calibration; 5 | EXPECT_EQ( "should have correct holdout_fraction" ; .holdout_fraction ; 0.25 ), 6 | EXPECT_EQ( "should have no holdout_query" ; .holdout_query ; null ), 7 | empty) 8 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.calibration.holdout_query: "SELECT 0" 2 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("calibration settings"; .deepdive_.calibration; 5 | EXPECT_EQ( "should have no holdout_fraction" ; .holdout_fraction ; null ), 6 | EXPECT_EQ( "should have correct holdout_query" ; .holdout_query ; "SELECT 0" ), 7 | empty) 8 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query_over_fraction.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query_over_fraction/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query_over_fraction/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.calibration.holdout_query: "SELECT 0" 2 | deepdive.calibration.holdout_fraction: 0.25 3 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query_over_fraction/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_holdout_query_over_fraction/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("calibration settings"; .deepdive_.calibration; 5 | EXPECT_EQ( "should have no holdout_fraction" ; .holdout_fraction ; null ), 6 | EXPECT_EQ( "should have correct holdout_query" ; .holdout_query ; "SELECT 0" ), 7 | empty) 8 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_observation_query.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_observation_query/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_observation_query/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.calibration.holdout_query: "SELECT 0" 2 | deepdive.calibration.observation_query: "SELECT 1" 3 | -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_observation_query/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-calibration_observation_query/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("calibration settings"; .deepdive_.calibration; 5 | EXPECT_EQ( "should have no holdout_fraction" ; .holdout_fraction ; null ), 6 | EXPECT_EQ( "should have correct holdout_query" ; .holdout_query ; "SELECT 0" ), 7 | EXPECT_EQ( "should have correct observation_query" ; .observation_query ; "SELECT 1" ), 8 | empty) 9 | -------------------------------------------------------------------------------- /compiler/test/should_accept-ddlog_undeclared_relations.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-ddlog_undeclared_relations/app.ddlog: -------------------------------------------------------------------------------- 1 | foo(x INT). 2 | 3 | # bar does not have an explicit schema declaration 4 | bar(x, x*2) :- foo(x). 5 | -------------------------------------------------------------------------------- /compiler/test/should_accept-ddlog_undeclared_relations/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-ddlog_undeclared_relations/input/foo.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/compiler/test/should_accept-ddlog_undeclared_relations/input/foo.tsv -------------------------------------------------------------------------------- /compiler/test/should_accept-empty_app.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-empty_app/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-empty_app/deepdive.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/compiler/test/should_accept-empty_app/deepdive.conf -------------------------------------------------------------------------------- /compiler/test/should_accept-empty_app/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-extraction_only_app.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-extraction_only_app/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-extraction_only_app/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: tsv_extractor 3 | input: """ 4 | SELECT * FROM foo 5 | """ 6 | udf: "cat" # identity extractor 7 | output_relation: "bar" 8 | } 9 | 10 | deepdive.schema.variables { 11 | # no variables 12 | } 13 | 14 | deepdive.inference.factors { 15 | # no factors 16 | } 17 | -------------------------------------------------------------------------------- /compiler/test/should_accept-extraction_only_app/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_cmd.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_cmd/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_cmd/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: cmd_extractor 3 | cmd: "udf/test_extractor.py" 4 | } 5 | -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_cmd/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_json.skip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/compiler/test/should_accept-extractor_style_json.skip -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_json/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_json/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: json_extractor 3 | input: """ 4 | SELECT * FROM foo 5 | """ 6 | udf: "udf/test_extractor.py" 7 | output_relation: "bar" 8 | } 9 | -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_json/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_sql.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_sql/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_sql/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: sql_extractor 3 | sql: """ 4 | INSERT INTO bar SELECT * FROM foo 5 | """ 6 | } 7 | -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_sql/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_tsv.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_tsv/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_tsv/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: tsv_extractor 3 | input: """ 4 | SELECT * FROM foo 5 | """ 6 | udf: "udf/test_extractor.py" 7 | output_relation: "bar" 8 | } 9 | -------------------------------------------------------------------------------- /compiler/test/should_accept-extractor_style_tsv/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_and.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_and/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_and/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_and/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_array_imply.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_array_imply/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_array_imply/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_array_imply/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_equal.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_equal/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_equal/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_equal/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply3.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply3/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply3/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply3/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_n_ary.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_n_ary/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_n_ary/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_n_ary/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_unary.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_unary/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_unary/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.inference.factors.test_factor { 6 | input_query: """ 7 | SELECT 1 AS "x.id", NULL AS "x.is_true" 8 | """ 9 | weight: 1 10 | function: "Imply(x.is_true)" 11 | } 12 | 13 | deepdive.schema.relations { 14 | x.columns { 15 | foo { type: TEXT } 16 | is_true { type: BOOLEAN } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_unary/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_imply_unary/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_istrue.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_istrue/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_istrue/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.inference.factors.test_factor { 6 | input_query: """ 7 | SELECT 1 AS "x.id", NULL AS "x.is_true" 8 | """ 9 | weight: 1 10 | function: "IsTrue(x.is_true)" 11 | } 12 | 13 | deepdive.schema.relations { 14 | x.columns { 15 | foo { type: TEXT } 16 | is_true { type: BOOLEAN } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_istrue/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_istrue/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_linear.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_linear/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_linear/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_linear/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_logical.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_logical/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_logical/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_logical/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_negated_variable.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_negated_variable/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_negated_variable/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_negated_variable/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_or.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_or/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_or/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_or/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_ratio.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_ratio/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_ratio/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_function_ratio/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_float.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_float/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_float/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.inference.factors.test_factor { 6 | input_query: """ 7 | SELECT 1 AS "x.id", NULL AS "x.is_true" 8 | """ 9 | weight: 0.5 10 | function: "IsTrue(x.is_true)" 11 | } 12 | 13 | deepdive.schema.relations { 14 | x.columns { 15 | foo { type: TEXT } 16 | is_true { type: BOOLEAN } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_float/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_float/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_float/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("parsed weight"; .deepdive_.inference.factors_byName.test_factor.weight_; 5 | EXPECT_EQ( "should have correct is_fixed" ; .is_fixed ; true ), 6 | EXPECT_EQ( "should have correct params" ; .params ; [] ), 7 | EXPECT_EQ( "should have correct init_value" ; .init_value ; 0.5 ), 8 | empty) 9 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_int.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_int/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_int/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.inference.factors.test_factor { 6 | input_query: """ 7 | SELECT 1 AS "x.id", NULL AS "x.is_true" 8 | """ 9 | weight: 5 10 | function: "IsTrue(x.is_true)" 11 | } 12 | 13 | deepdive.schema.relations { 14 | x.columns { 15 | foo { type: TEXT } 16 | is_true { type: BOOLEAN } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_int/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_int/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_int/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("parsed weight"; .deepdive_.inference.factors_byName.test_factor.weight_; 5 | EXPECT_EQ( "should have correct is_fixed" ; .is_fixed ; true ), 6 | EXPECT_EQ( "should have correct params" ; .params ; [] ), 7 | EXPECT_EQ( "should have correct init_value" ; .init_value ; 5 ), 8 | empty) 9 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.inference.factors.test_factor { 6 | input_query: """ 7 | SELECT 1 AS "x.id" 8 | """ 9 | weight: "?" 10 | function: "IsTrue(x.is_true)" 11 | } 12 | 13 | deepdive.schema.relations { 14 | x.columns { 15 | foo { type: TEXT } 16 | is_true { type: BOOLEAN } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("parsed weight"; .deepdive_.inference.factors_byName.test_factor.weight_; 5 | EXPECT_EQ( "should have correct is_fixed" ; .is_fixed ; false ), 6 | EXPECT_EQ( "should have correct params" ; .params ; [] ), 7 | empty) 8 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("parsed weight"; .deepdive_.inference.factors_byName.test_factor.weight_; 5 | EXPECT_EQ( "should have correct is_fixed" ; .is_fixed ; false ), 6 | EXPECT_EQ( "should have correct params" ; .params ; ["x.feature", "y.feature"] ), 7 | empty) 8 | -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized_with_dots.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized_with_dots/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized_with_dots/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized_with_dots/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-factor_weight_unknown_parameterized_with_dots/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("parsed weight"; .deepdive_.inference.factors_byName.test_factor.weight_; 5 | EXPECT_EQ( "should have correct is_fixed" ; .is_fixed ; false ), 6 | EXPECT_EQ( "should have correct params" ; .params ; ["x.R0.feature", "x.R1.feature"] ), 7 | empty) 8 | -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_boolean.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_boolean/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_boolean/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.schema.relations { 6 | x.columns { 7 | foo { type: TEXT } 8 | is_true { type: BOOLEAN } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_boolean/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_boolean/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_boolean/validate-compiled-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env jq 2 | include "../testutil"; 3 | 4 | TEST("parsed variable datatype"; .deepdive_.schema.variables_byName.x; 5 | EXPECT_EQ( "should have correct name" ; .variableName ; "x" ), 6 | EXPECT_EQ( "should have correct type" ; .variableType ; "boolean" ), 7 | EXPECT_EQ( "should have correct table" ; .variablesTable ; "x" ), 8 | EXPECT_EQ( "should have correct label column" ; .variablesLabelColumn ; "is_true" ), 9 | empty) 10 | -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_categorical.bats: -------------------------------------------------------------------------------- 1 | .should_accept.bats.template -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_categorical/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_categorical/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | tag.category: Categorical 3 | } 4 | 5 | deepdive.schema.relations { 6 | tag.columns { 7 | foo { type: TEXT, annotations: [ { name: "key" } ] } 8 | category { type: TEXT } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_categorical/input: -------------------------------------------------------------------------------- 1 | ../stub-app/input -------------------------------------------------------------------------------- /compiler/test/should_accept-variable_categorical/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-empty_dir.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-empty_dir/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/compiler/test/should_reject-empty_dir/.gitignore -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_input_missing.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_input_missing/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_input_missing/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: tsv_extractor 3 | # input missing 4 | #input: """ 5 | # SELECT * FROM foo 6 | #""" 7 | udf: "udf/test_extractor.py" 8 | output_relation: "bar" 9 | } 10 | -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_input_missing/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_output_missing.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_output_missing/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_output_missing/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: tsv_extractor 3 | input: """ 4 | SELECT * FROM foo 5 | """ 6 | udf: "udf/test_extractor.py" 7 | # output_relation missing 8 | #output_relation: "bar" 9 | } 10 | -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_output_missing/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_invalid.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_invalid/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_invalid/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: my_awesome_extractor 3 | input: """ 4 | SELECT * FROM foo 5 | """ 6 | udf: "udf/test_extractor.py" 7 | output_relation: "bar" 8 | } 9 | -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_invalid/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_missing.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_missing/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_missing/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | # style missing 3 | #style: tsv_extractor 4 | input: """ 5 | SELECT * FROM foo 6 | """ 7 | udf: "udf/test_extractor.py" 8 | output_relation: "bar" 9 | } 10 | -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_style_missing/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_udf_missing.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_udf_missing/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_udf_missing/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors.test_extractor { 2 | style: tsv_extractor 3 | input: """ 4 | SELECT * FROM foo 5 | """ 6 | # udf missing 7 | #udf: "udf/test_extractor.py" 8 | output_relation: "bar" 9 | } 10 | -------------------------------------------------------------------------------- /compiler/test/should_reject-extractor_udf_missing/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_equal_n_ary.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_equal_n_ary/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_equal_n_ary/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_equal_unary.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_equal_unary/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_equal_unary/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.inference.factors.test_factor { 6 | input_query: """ 7 | SELECT 0 AS "x.id", NULL AS "x.is_true" 8 | """ 9 | weight: 1 10 | function: "Equal(x.is_true)" 11 | } 12 | 13 | deepdive.schema.relations { 14 | x.columns { 15 | foo { type: TEXT } 16 | is_true { type: BOOLEAN } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_equal_unary/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_istrue_n_ary.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_istrue_n_ary/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_istrue_n_ary/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_malformed.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_malformed/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_malformed/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Boolean 3 | } 4 | 5 | deepdive.inference.factors.test_factor { 6 | input_query: """ 7 | SELECT 1 AS "x.id", NULL AS "x.is_true" 8 | """ 9 | weight: 1 10 | function: "Imply(a" 11 | } 12 | 13 | deepdive.schema.relations { 14 | x.columns { 15 | foo { type: TEXT } 16 | is_true { type: BOOLEAN } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /compiler/test/should_reject-factor_function_malformed/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-multiple_processes_same_output.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-multiple_processes_same_output/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-multiple_processes_same_output/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.extraction.extractors { 2 | a { 3 | output_relation: foo 4 | } 5 | 6 | b { 7 | output_relation: foo 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /compiler/test/should_reject-multiple_processes_same_output/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_malformed.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_malformed/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_malformed/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables.relation1_4145/24.column : Boolean 2 | -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_malformed/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_missing_column.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_missing_column/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_missing_column/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables.relation1: Boolean 2 | -------------------------------------------------------------------------------- /compiler/test/should_reject-schema_variables_missing_column/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/should_reject-variable_invalid_type.bats: -------------------------------------------------------------------------------- 1 | .should_reject.bats.template -------------------------------------------------------------------------------- /compiler/test/should_reject-variable_invalid_type/db.url: -------------------------------------------------------------------------------- 1 | ../stub-app/db.url -------------------------------------------------------------------------------- /compiler/test/should_reject-variable_invalid_type/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.schema.variables { 2 | x.is_true: Binary 3 | y.is_true: Categorical 4 | z.is_true: String 5 | } 6 | -------------------------------------------------------------------------------- /compiler/test/should_reject-variable_invalid_type/schema.sql: -------------------------------------------------------------------------------- 1 | ../stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/stub-app/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_compiler_tests 2 | -------------------------------------------------------------------------------- /compiler/test/stub-app/input/init.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/compiler/test/stub-app/input/init.sh -------------------------------------------------------------------------------- /compiler/test/stub-app/schema.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /compiler/test/stub-app/schema.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/compiler/test/stub-app/schema.sql -------------------------------------------------------------------------------- /compiler/test/update-bats-symlinks: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Updates symlinks to the .bats.template 3 | set -euo pipefail 4 | cd "$(dirname "$0")" 5 | 6 | for tmpl in {.,}*.bats.template; do 7 | [[ -e "$tmpl" ]] || continue 8 | testCaseGroup=${tmpl%.bats.template} 9 | testCaseGroup=${testCaseGroup#.} 10 | rm -f "$testCaseGroup"-*.bats 11 | for testCase in "$testCaseGroup"-*; do 12 | [[ -d "$testCase" ]] || continue 13 | [[ ! -e "$testCase".skip ]] || continue 14 | ln -sfnv "$tmpl" "$testCase".bats 15 | done 16 | done 17 | -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-analyze: -------------------------------------------------------------------------------- 1 | ../postgresql/db-analyze -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-create-table-as: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-as -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-create-table-def: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-def -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-create-view-as: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-view-as -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-create_calibration_view: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create_calibration_view -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-execute: -------------------------------------------------------------------------------- 1 | ../postgresql/db-execute -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-generate_series: -------------------------------------------------------------------------------- 1 | ../postgresql/db-generate_series -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-init: -------------------------------------------------------------------------------- 1 | ../postgresql/db-init -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-is_ready: -------------------------------------------------------------------------------- 1 | ../postgresql/db-is_ready -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-load: -------------------------------------------------------------------------------- 1 | ../postgresql/db-load -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-parse: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-parse -- Parses URL for Greenplum database and does sanity check 3 | # > db-parse "$url" 4 | set -eu 5 | 6 | # ensure the psql client is actually Greenplum's 7 | psql --help | head -1 | grep -q Greenplum || 8 | # or refuse to continue since the rest of the db-driver depends on it 9 | error "$(type -p psql) is not a Greenplum client" "Set up your PATH correctly for Greenplum" 10 | 11 | # delegate to postgresql driver for the real thing 12 | . "$(dirname "$BASH_SOURCE")"/../postgresql/db-parse 13 | -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-prompt: -------------------------------------------------------------------------------- 1 | ../postgresql/db-prompt -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-psql: -------------------------------------------------------------------------------- 1 | ../postgresql/db-psql -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-query: -------------------------------------------------------------------------------- 1 | ../postgresql/db-query -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-query-tsj: -------------------------------------------------------------------------------- 1 | ../postgresql/db-query-tsj -------------------------------------------------------------------------------- /database/db-driver/greenplum/db-supports_pg_lang: -------------------------------------------------------------------------------- 1 | ../postgresql/db-supports_pg_lang -------------------------------------------------------------------------------- /database/db-driver/greenplum/sql_for_column_types: -------------------------------------------------------------------------------- 1 | ../postgresql/sql_for_column_types -------------------------------------------------------------------------------- /database/db-driver/mysql/db-analyze: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-analyze -- Analyzes statistics of given table to speed up future queries for MySQL 3 | # > db-analyze TABLE 4 | ## 5 | set -eu 6 | 7 | table=${1:?No TABLE given} 8 | 9 | # See: https://dev.mysql.com/doc/refman/ 10 | exec db-execute "ANALYZE TABLE $table;" 11 | -------------------------------------------------------------------------------- /database/db-driver/mysql/db-create-table-as: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-as -------------------------------------------------------------------------------- /database/db-driver/mysql/db-create-table-def: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-def -------------------------------------------------------------------------------- /database/db-driver/mysql/db-create-table-options: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-options -------------------------------------------------------------------------------- /database/db-driver/mysql/db-execute: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-execute -- Executes a given SQL query against the PostgreSQL database configured for a DeepDive application 3 | # > eval "$(db-parse "$url")" 4 | # > db-execute SQL 5 | ## 6 | set -eu 7 | 8 | sql=${1:?No SQL given}; shift 9 | 10 | # Using ANSI_QUOTES sql_mode to run ANSI SQL queries generated by DeepDive compiler 11 | # See: http://dev.mysql.com/doc/refman/5.7/en/sql-mode.html#sqlmode_ansi_quotes 12 | exec db-prompt --local-infile -e "SET sql_mode='ANSI_QUOTES,PIPES_AS_CONCAT'; $sql" "$@" 13 | -------------------------------------------------------------------------------- /database/db-driver/mysql/db-is_ready: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-is_ready -- Exits with zero status if the database is ready 3 | # > eval "$(db-parse "$url")" 4 | # > db-is_ready 5 | ## 6 | # XXX may not be accurate 7 | exec mysql -h "$DBHOST" -P "$DBPORT" -u "$DBUSER" --password="$DBPASSWORD" -e "SHOW DATABASES" >/dev/null 8 | -------------------------------------------------------------------------------- /database/db-driver/mysql/db-prompt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-prompt -- Starts a prompt for the MySQL database configured for a DeepDive application 3 | # > eval "$(db-parse "$url")" 4 | # > db-prompt 5 | ## 6 | set -eu 7 | 8 | exec mysql -h "$DBHOST" -P "$DBPORT" -u "$DBUSER" --password="$DBPASSWORD" -D "$DBNAME" --local-infile "$@" 9 | -------------------------------------------------------------------------------- /database/db-driver/mysql/db-unload: -------------------------------------------------------------------------------- 1 | ../postgresql/db-unload -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-analyze: -------------------------------------------------------------------------------- 1 | ../postgresql/db-analyze -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-create-table: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-create-table-as: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-as -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-create-table-def: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-def -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-create-table-like: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-like -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-create-table-options: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-table-options -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-create-view-as: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create-view-as -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-create_calibration_view: -------------------------------------------------------------------------------- 1 | ../postgresql/db-create_calibration_view -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-execute: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-execute -- Executes a given SQL query against the PostgreSQL database configured for a DeepDive application 3 | # > eval "$(db-parse "$url")" 4 | # > db-execute SQL 5 | ## 6 | set -eu 7 | 8 | sql=${1:?No SQL given}; shift 9 | 10 | exec db-prompt -v ON_ERROR_STOP=1 -f <(echo "$sql") "$@" 11 | -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-generate_series: -------------------------------------------------------------------------------- 1 | ../postgresql/db-generate_series -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-init: -------------------------------------------------------------------------------- 1 | ../postgresql/db-init -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-is_ready: -------------------------------------------------------------------------------- 1 | ../postgresql/db-is_ready -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-load: -------------------------------------------------------------------------------- 1 | ../postgresql/db-load -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-parse: -------------------------------------------------------------------------------- 1 | ../postgresql/db-parse -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-prompt: -------------------------------------------------------------------------------- 1 | ../postgresql/db-prompt -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-psql: -------------------------------------------------------------------------------- 1 | ../postgresql/db-psql -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-query: -------------------------------------------------------------------------------- 1 | ../postgresql/db-query -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-supports_pg_lang: -------------------------------------------------------------------------------- 1 | ../postgresql/db-supports_pg_lang -------------------------------------------------------------------------------- /database/db-driver/postgresql-xl/db-unload: -------------------------------------------------------------------------------- 1 | ../postgresql/db-unload -------------------------------------------------------------------------------- /database/db-driver/postgresql/db-analyze: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-analyze -- Analyzes statistics of given table to speed up future queries for PostgreSQL 3 | # > db-analyze TABLE 4 | ## 5 | set -eu 6 | 7 | table=${1:?No TABLE given} 8 | 9 | # See: http://www.postgresql.org/docs/8.2/static/sql-analyze.html 10 | exec db-execute "ANALYZE $table;" 11 | -------------------------------------------------------------------------------- /database/db-driver/postgresql/db-execute: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-execute -- Executes a given SQL query against the PostgreSQL database configured for a DeepDive application 3 | # > eval "$(db-parse "$url")" 4 | # > db-execute SQL 5 | ## 6 | set -eu 7 | 8 | sql=${1:?No SQL given}; shift 9 | 10 | exec db-prompt -v ON_ERROR_STOP=1 -c "$sql" "$@" 11 | -------------------------------------------------------------------------------- /database/db-driver/postgresql/db-init: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-init -- Initializes PostgreSQL database configured for a DeepDive application 3 | # > eval "$(db-parse "$url")" 4 | # > db-init 5 | ## 6 | set -eu 7 | 8 | createdb "$DBNAME" || true >/dev/null 9 | -------------------------------------------------------------------------------- /database/db-driver/postgresql/db-is_ready: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-is_ready -- Exits with zero status if the database is ready 3 | # > eval "$(db-parse "$url")" 4 | # > db-is_ready 5 | ## 6 | exec db-psql -l >/dev/null 7 | -------------------------------------------------------------------------------- /database/db-driver/postgresql/db-prompt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-prompt -- Starts a prompt for the PostgreSQL database configured for a DeepDive application 3 | # > eval "$(db-parse "$url")" 4 | # > db-prompt 5 | ## 6 | exec db-psql -d "$DBNAME" "$@" 7 | -------------------------------------------------------------------------------- /database/db-driver/postgresql/db-psql: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-psql -- Wraps PostgreSQL's psql client supplying minimal arguments from DeepDive app's db.url 3 | # > eval "$(db-parse "$url")" 4 | # > db-psql [PSQL_OPTS...] 5 | ## 6 | set -eu 7 | 8 | PGPASSWORD="$DBPASSWORD" \ 9 | exec psql -h "$DBHOST" -p "$DBPORT" -U "$DBUSER" "$@" 10 | -------------------------------------------------------------------------------- /database/db-driver/postgresql/db-supports_pg_lang: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # db-supports_pg_lang -- Creates a PostgreSQL language if it doesn't exist yet 3 | # > eval "$(db-parse "$url")" 4 | # > db-supports_pg_lang LANG 5 | ## 6 | set -euo pipefail 7 | 8 | Lang=$1; shift 9 | 10 | [[ $(db-query "SELECT COUNT(*) FROM pg_catalog.pg_language WHERE lanname = '$Lang'" tsv 0) -gt 0 ]] || 11 | db-execute "CREATE LANGUAGE $Lang" 12 | -------------------------------------------------------------------------------- /database/test/test_environ.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Unit tests for DeepDive database components 3 | # Usage: load test_environ # from .bats files typically on the same directory 4 | 5 | load ../../test/test_environ 6 | -------------------------------------------------------------------------------- /ddlib/ddlib/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /ddlib/ddlib/__init__.py: -------------------------------------------------------------------------------- 1 | from .dd import * 2 | from .gen_feats import * 3 | from .util import * 4 | -------------------------------------------------------------------------------- /ddlib/deepdive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # a facade package to expose some useful things 3 | 4 | from __future__ import absolute_import, print_function, unicode_literals 5 | from ddlib.util import tsj_extractor,tsv_extractor,over,returns 6 | from collections import OrderedDict 7 | -------------------------------------------------------------------------------- /ddlib/test/test.json.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/ddlib/test/test.json.bz2 -------------------------------------------------------------------------------- /ddlib/test/test2.json.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/ddlib/test/test2.json.bz2 -------------------------------------------------------------------------------- /ddlib/test/test_dep.test2.expected: -------------------------------------------------------------------------------- 1 | Sen. Obama 2 | Obama invest 3 | invest . 4 | . . 5 | -------------------------------------------------------------------------------- /ddlib/test/test_environ.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Unit tests for ddlib 3 | # Usage: load test_environ # from .bats files typically on the same directory 4 | 5 | load ../../test/test_environ 6 | -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | /_site 2 | /.gems 3 | /.jekyll-metadata 4 | .DS_Store 5 | -------------------------------------------------------------------------------- /doc/.linkchecker.ignore-urls: -------------------------------------------------------------------------------- 1 | http://localhost:8000/#/search 2 | http://localhost:8000 3 | http://omim.org/ 4 | -------------------------------------------------------------------------------- /doc/_config.yml: -------------------------------------------------------------------------------- 1 | source: . 2 | safe: false # need _plugins/rewrite_md_links.rb to generate correct links 3 | baseurl: http://deepdive.stanford.edu 4 | markdown: redcarpet 5 | redcarpet: 6 | extensions: 7 | # - no_intra_emphasis 8 | - fenced_code_blocks 9 | - autolink 10 | - tables 11 | - with_toc_data 12 | exclude: 13 | - params.json 14 | - "*.graffle" 15 | - Makefile 16 | - README 17 | -------------------------------------------------------------------------------- /doc/_config_2.x.yml: -------------------------------------------------------------------------------- 1 | # Extra Jekyll config for 2.x 2 | 3 | # We need to disable syntax highlighting as jekyll-2.x with pygments complains: 4 | # no lexer for alias 'ddlog' found 5 | highlighter: false 6 | -------------------------------------------------------------------------------- /doc/_includes/css/landing.css: -------------------------------------------------------------------------------- 1 | section#landing { 2 | margin-top: 200px; 3 | font-family: Lato; 4 | } 5 | 6 | section#landing h1, section#landing h2, section#landing h3, section#landing p { 7 | font-family: Lato; 8 | letter-spacing: 0px; 9 | font-weight: 300; 10 | } 11 | 12 | section#landing h1 { 13 | font-size: 4em; 14 | } 15 | -------------------------------------------------------------------------------- /doc/_includes/examples: -------------------------------------------------------------------------------- 1 | ../../examples -------------------------------------------------------------------------------- /doc/assets/factor_graph.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/factor_graph.pdf -------------------------------------------------------------------------------- /doc/assets/factor_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/factor_graph.png -------------------------------------------------------------------------------- /doc/assets/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /doc/assets/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /doc/assets/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /doc/assets/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /doc/assets/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /doc/assets/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /doc/assets/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /doc/assets/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/assets/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /doc/changelog/0.05.01-alpha.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Changes for 0.0.5 4 | no_toc: true 5 | --- 6 | 7 | # Changelog for release 0.0.5-alpha (02/08/2015) 8 | 9 | - Added support to build Docker images for DeepDive. See the README.md for more. 10 | - Added SQL "FeatureStatsView" view. Populated with feature 11 | statistics; useful for debugging. 12 | - Added a few fixes to greenplum docs 13 | - Added parallel greenplum loading for extractor data 14 | - A few misc bugfixes 15 | 16 | -------------------------------------------------------------------------------- /doc/google498f7e56a944f834.html: -------------------------------------------------------------------------------- 1 | google-site-verification: google498f7e56a944f834.html 2 | -------------------------------------------------------------------------------- /doc/images/browsing_corpus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/browsing_corpus.png -------------------------------------------------------------------------------- /doc/images/browsing_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/browsing_results.png -------------------------------------------------------------------------------- /doc/images/browsing_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/browsing_screenshot.png -------------------------------------------------------------------------------- /doc/images/browsing_with_presentation_template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/browsing_with_presentation_template.png -------------------------------------------------------------------------------- /doc/images/browsing_without_presentation_template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/browsing_without_presentation_template.png -------------------------------------------------------------------------------- /doc/images/carousel/dark-data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/dark-data.png -------------------------------------------------------------------------------- /doc/images/carousel/dark-data.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/dark-data.psd -------------------------------------------------------------------------------- /doc/images/carousel/deepdive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/deepdive.png -------------------------------------------------------------------------------- /doc/images/carousel/deepdive.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/deepdive.psd -------------------------------------------------------------------------------- /doc/images/carousel/dimmwitted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/dimmwitted.png -------------------------------------------------------------------------------- /doc/images/carousel/dimmwitted.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/dimmwitted.psd -------------------------------------------------------------------------------- /doc/images/carousel/dimmwitted.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/dimmwitted.xcf -------------------------------------------------------------------------------- /doc/images/carousel/memex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/memex.png -------------------------------------------------------------------------------- /doc/images/carousel/memex.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/memex.psd -------------------------------------------------------------------------------- /doc/images/carousel/paleo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/paleo.png -------------------------------------------------------------------------------- /doc/images/carousel/paleo.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/paleo.psd -------------------------------------------------------------------------------- /doc/images/carousel/paleo.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/paleo.xcf -------------------------------------------------------------------------------- /doc/images/carousel/support.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/support.png -------------------------------------------------------------------------------- /doc/images/carousel/support.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/support.psd -------------------------------------------------------------------------------- /doc/images/carousel/support.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/carousel/support.xcf -------------------------------------------------------------------------------- /doc/images/coming_soon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/coming_soon.png -------------------------------------------------------------------------------- /doc/images/dashboard/calibration_plot_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/calibration_plot_report.png -------------------------------------------------------------------------------- /doc/images/dashboard/homepage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/homepage.png -------------------------------------------------------------------------------- /doc/images/dashboard/run_snapshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/run_snapshot.png -------------------------------------------------------------------------------- /doc/images/dashboard/supervision_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/supervision_report.png -------------------------------------------------------------------------------- /doc/images/dashboard/task_control.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/task_control.png -------------------------------------------------------------------------------- /doc/images/dashboard/task_input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/task_input.png -------------------------------------------------------------------------------- /doc/images/dashboard/top_positive_features_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/top_positive_features_report.png -------------------------------------------------------------------------------- /doc/images/dashboard/trend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/trend.png -------------------------------------------------------------------------------- /doc/images/dashboard/trends.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/dashboard/trends.png -------------------------------------------------------------------------------- /doc/images/header_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/header_logo.png -------------------------------------------------------------------------------- /doc/images/logos/amfam.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/amfam.jpeg -------------------------------------------------------------------------------- /doc/images/logos/chtc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/chtc.png -------------------------------------------------------------------------------- /doc/images/logos/darpa.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/darpa.jpg -------------------------------------------------------------------------------- /doc/images/logos/google.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/google.png -------------------------------------------------------------------------------- /doc/images/logos/lsvp.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/lsvp.gif -------------------------------------------------------------------------------- /doc/images/logos/moore.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/moore.jpg -------------------------------------------------------------------------------- /doc/images/logos/nsf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/nsf.jpg -------------------------------------------------------------------------------- /doc/images/logos/onr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/onr.png -------------------------------------------------------------------------------- /doc/images/logos/sloan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/sloan.jpg -------------------------------------------------------------------------------- /doc/images/logos/toshiba.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/logos/toshiba.jpg -------------------------------------------------------------------------------- /doc/images/mindtagger_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/mindtagger_screenshot.png -------------------------------------------------------------------------------- /doc/images/mindtagger_screenshot_export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/mindtagger_screenshot_export.png -------------------------------------------------------------------------------- /doc/images/mindtagger_screenshot_tags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/mindtagger_screenshot_tags.png -------------------------------------------------------------------------------- /doc/images/mindtagger_screenshot_with_features.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/mindtagger_screenshot_with_features.png -------------------------------------------------------------------------------- /doc/images/paleo/Pipeline3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/paleo/Pipeline3.pdf -------------------------------------------------------------------------------- /doc/images/paleo/Pipeline3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/paleo/Pipeline3.png -------------------------------------------------------------------------------- /doc/images/showcase/PA145011114.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/showcase/PA145011114.png -------------------------------------------------------------------------------- /doc/images/showcase/genetics.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/showcase/genetics.jpg -------------------------------------------------------------------------------- /doc/images/showcase/memex-human-trafficking.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/showcase/memex-human-trafficking.jpg -------------------------------------------------------------------------------- /doc/images/showcase/paleontology.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/showcase/paleontology.jpg -------------------------------------------------------------------------------- /doc/images/showcase/wisci.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/showcase/wisci.png -------------------------------------------------------------------------------- /doc/images/spouse/has_spouse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/spouse/has_spouse.png -------------------------------------------------------------------------------- /doc/images/spouse/has_spouse.tsv: -------------------------------------------------------------------------------- 1 | 0 0.1 288791 49 88362 2 | 0.1 0.2 30964 19 106 3 | 0.2 0.3 17219 21 48 4 | 0.3 0.4 12310 20 27 5 | 0.4 0.5 11066 22 11 6 | 0.5 0.6 8445 30 17 7 | 0.6 0.7 6616 41 12 8 | 0.7 0.8 6665 52 15 9 | 0.8 0.9 6612 135 11 10 | 0.9 1 60252 4910 22 11 | -------------------------------------------------------------------------------- /doc/images/walkthrough/dataflow.graffle/image1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/walkthrough/dataflow.graffle/image1.pdf -------------------------------------------------------------------------------- /doc/images/walkthrough/dataflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/walkthrough/dataflow.png -------------------------------------------------------------------------------- /doc/images/walkthrough/datamodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/walkthrough/datamodel.png -------------------------------------------------------------------------------- /doc/images/walkthrough/example_hasspouse.graffle/image1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/walkthrough/example_hasspouse.graffle/image1.pdf -------------------------------------------------------------------------------- /doc/images/walkthrough/example_hasspouse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/doc/images/walkthrough/example_hasspouse.png -------------------------------------------------------------------------------- /doc/using-docker.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: Using DeepDive in Docker containers 4 | --- 5 | 6 |
7 | 8 | - This is very popular question, so we should at least leave some hints after trying something out. 9 | - Usually people have trouble connecting PostgreSQL instance with the DeepDive instance itself. 10 | 11 | 12 | -------------------------------------------------------------------------------- /examples/balance/.gitignore: -------------------------------------------------------------------------------- 1 | /run 2 | /snapshot 3 | -------------------------------------------------------------------------------- /examples/balance/README.md: -------------------------------------------------------------------------------- 1 | # Balance example to illustrate different semantics 2 | 3 | See Example 2.5 of [the VLDB 2015 paper](http://i.stanford.edu/hazy/papers/inc.pdf) for full detail. 4 | 5 | Linear semantics is not good. 6 | -------------------------------------------------------------------------------- /examples/balance/app.ddlog: -------------------------------------------------------------------------------- 1 | yes (vote_id bigint). 2 | no (vote_id bigint). 3 | 4 | whatever? ( 5 | candidate bigint 6 | ). 7 | 8 | @weight(1) 9 | whatever(p) :- yes(_). 10 | 11 | @weight(-1) 12 | whatever(p) :- no(_). 13 | -------------------------------------------------------------------------------- /examples/balance/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_balance_$USER 2 | -------------------------------------------------------------------------------- /examples/balance/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive { 2 | calibration.holdout_fraction: 0.0 3 | sampler.sampler_args: "-l 0 -i 1000 --alpha 0.01" 4 | } 5 | -------------------------------------------------------------------------------- /examples/balance/input/no.tsv.sh: -------------------------------------------------------------------------------- 1 | seq 990 2 | -------------------------------------------------------------------------------- /examples/balance/input/whatever.tsv: -------------------------------------------------------------------------------- 1 | 1 \N 2 | -------------------------------------------------------------------------------- /examples/balance/input/yes.tsv.sh: -------------------------------------------------------------------------------- 1 | seq 1000 2 | -------------------------------------------------------------------------------- /examples/census/.gitignore: -------------------------------------------------------------------------------- 1 | /run 2 | /snapshot 3 | -------------------------------------------------------------------------------- /examples/census/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_census_$USER 2 | -------------------------------------------------------------------------------- /examples/census/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive { 2 | calibration.holdout_fraction: 0.25 3 | sampler.partitions: 3 4 | sampler.sampler_args: "-l 100 -i 100 --sample_evidence" 5 | } 6 | -------------------------------------------------------------------------------- /examples/census/deepdive.conf.numbskull: -------------------------------------------------------------------------------- 1 | deepdive { 2 | calibration.holdout_fraction: 0.25 3 | sampler.sampler_cmd: "numbskull" 4 | sampler.sampler_args: "-l 100 -i 100 --sample_evidence" 5 | } 6 | -------------------------------------------------------------------------------- /examples/census/input/adult.csv.sh: -------------------------------------------------------------------------------- 1 | curl https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data | 2 | sed '/^$/d;s/, /,/g' | # remove last empty line 3 | awk '{printf "%d,%s\n", NR, $0}' # prepend record ID 4 | -------------------------------------------------------------------------------- /examples/chain/.gitignore: -------------------------------------------------------------------------------- 1 | /run 2 | /snapshot 3 | -------------------------------------------------------------------------------- /examples/chain/app.ddlog: -------------------------------------------------------------------------------- 1 | heads (id bigint). 2 | tails (id bigint). 3 | unk (id bigint). 4 | 5 | tags? ( 6 | @key id bigint, 7 | tag text 8 | ). 9 | 10 | tags(id, "H") = TRUE :- heads(id). 11 | tags(id, "T") = TRUE :- tails(id). 12 | 13 | tags(id, "T") = FALSE :- heads(id). 14 | tags(id, "H") = FALSE :- tails(id). 15 | 16 | tags(id, "H") = NULL :- unk(id). 17 | tags(id, "T") = NULL :- unk(id). 18 | 19 | 20 | @weight(t1, t2) 21 | tags(id1, t1) ^ tags(id2, t2) :- [id2 = id1 + 1]. 22 | -------------------------------------------------------------------------------- /examples/chain/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_chain_$USER 2 | -------------------------------------------------------------------------------- /examples/chain/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive { 2 | calibration.holdout_fraction: 0.0 3 | sampler.sampler_args: "-l 1000 -i 1000 --alpha 0.01" 4 | } 5 | -------------------------------------------------------------------------------- /examples/chain/input/heads.tsv.sh: -------------------------------------------------------------------------------- 1 | seq 1 6 1000 2 | -------------------------------------------------------------------------------- /examples/chain/input/tails.tsv.sh: -------------------------------------------------------------------------------- 1 | seq 2 6 1000 2 | -------------------------------------------------------------------------------- /examples/chain/input/unk.tsv.sh: -------------------------------------------------------------------------------- 1 | seq 3 6 1000 2 | seq 4 6 1000 3 | seq 5 6 1000 4 | seq 6 6 1000 5 | -------------------------------------------------------------------------------- /examples/chunking/.gitignore: -------------------------------------------------------------------------------- 1 | /run 2 | /snapshot 3 | /result/output 4 | /result/result 5 | -------------------------------------------------------------------------------- /examples/chunking/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_chunking_$USER 2 | -------------------------------------------------------------------------------- /examples/chunking/deepdive.conf: -------------------------------------------------------------------------------- 1 | # Specify a holdout fraction 2 | deepdive.calibration.holdout_query: """ 3 | INSERT INTO dd_graph_variables_holdout(variable_id) 4 | SELECT dd_id 5 | FROM dd_variables_chunk 6 | WHERE word_id > 220663 7 | """ 8 | #deepdive.sampler.sampler_cmd: "numbskull" 9 | deepdive.sampler.sampler_args: "-l 100 -i 100 --sample_evidence" 10 | -------------------------------------------------------------------------------- /examples/chunking/input/senna.filtered.tsv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/chunking/input/senna.filtered.tsv.bz2 -------------------------------------------------------------------------------- /examples/chunking/input/tags.tsv: -------------------------------------------------------------------------------- 1 | B-ADJP 2 | B-ADVP 3 | B-CONJP 4 | B-INTJ 5 | B-LST 6 | B-NP 7 | B-PP 8 | B-PRT 9 | B-SBAR 10 | B-UCP 11 | B-VP 12 | I-ADJP 13 | I-ADVP 14 | I-CONJP 15 | I-INTJ 16 | I-NP 17 | I-PP 18 | I-PRT 19 | I-SBAR 20 | I-UCP 21 | I-VP 22 | O 23 | -------------------------------------------------------------------------------- /examples/chunking/input/vec_dims.tsv.sh: -------------------------------------------------------------------------------- 1 | seq 50 2 | -------------------------------------------------------------------------------- /examples/chunking/input/word_embedding.tsv.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | cd "$(dirname "$0")" 4 | 5 | bzcat senna.filtered.tsv.bz2 6 | -------------------------------------------------------------------------------- /examples/chunking/readme.md: -------------------------------------------------------------------------------- 1 | # Text chunking example 2 | 3 | STALE: See the [page in the documentation](../../doc/example-chunking.md) for full detail. 4 | 5 | NEW!!! (July 2016): Added word embedding features (Senna) for demonstration. 6 | 7 | With 2K training words, it doesn't improve the quality though... 8 | - Tried Glove and reg_param tuning to no avail. 9 | - Maybe the distributional semantics of words cannot patch the remaining errors. 10 | - Or maybe it'll make a difference on a larger training set. 11 | -------------------------------------------------------------------------------- /examples/chunking/result/calibration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/chunking/result/calibration.png -------------------------------------------------------------------------------- /examples/chunking/result/eval.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | cd "$(dirname "$0")" 4 | 5 | deepdive sql "COPY ( 6 | SELECT c.word, c.pos, c.true_tag, a.tag 7 | FROM chunk_inference a 8 | LEFT JOIN chunk_inference b 9 | ON a.word_id = b.word_id 10 | AND a.expectation < b.expectation 11 | LEFT JOIN words c 12 | ON a.word_id = c.word_id 13 | WHERE b.expectation IS NULL 14 | ORDER BY a.word_id 15 | ) TO STDOUT DELIMITER ' ';" > output 16 | 17 | ./conlleval.pl ' 16 | sys.exit(1) 17 | -------------------------------------------------------------------------------- /examples/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/run.sh -------------------------------------------------------------------------------- /examples/smoke/.gitignore: -------------------------------------------------------------------------------- 1 | /run 2 | /snapshot 3 | -------------------------------------------------------------------------------- /examples/smoke/README.md: -------------------------------------------------------------------------------- 1 | # Simple DeepDive Example: Smoke Example 2 | 3 | See the [documentation about this simple example](../../doc/example-smoke.md) for more detail. 4 | -------------------------------------------------------------------------------- /examples/smoke/app.ddlog: -------------------------------------------------------------------------------- 1 | person ( 2 | person_id bigint, 3 | name text 4 | ). 5 | 6 | person_has_cancer? ( 7 | person_id bigint 8 | ). 9 | 10 | person_smokes? ( 11 | person_id bigint 12 | ). 13 | 14 | friends ( 15 | person_id bigint, 16 | friend_id bigint 17 | ). 18 | 19 | @weight(0.5) 20 | person_smokes(p) => person_has_cancer(p) :- 21 | person(p, _). 22 | 23 | @weight(0.4) 24 | person_smokes(p1) => person_smokes(p2) :- 25 | person(p1, _), person(p2, _), friends(p1, p2). 26 | -------------------------------------------------------------------------------- /examples/smoke/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_smoke_$USER 2 | -------------------------------------------------------------------------------- /examples/smoke/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive { 2 | calibration.holdout_fraction: 0.0 3 | 4 | sampler.sampler_args: "-l 0 -i 1000 --alpha 0.01" 5 | } 6 | -------------------------------------------------------------------------------- /examples/smoke/input/friends.tsv: -------------------------------------------------------------------------------- 1 | 1 2 2 | 1 3 3 | 1 4 4 | 2 1 5 | 3 1 6 | 3 4 7 | 4 1 8 | 4 3 9 | 5 6 10 | 6 5 11 | -------------------------------------------------------------------------------- /examples/smoke/input/person.tsv: -------------------------------------------------------------------------------- 1 | 1 Anna 2 | 2 Bob 3 | 3 Edward 4 | 4 Frank 5 | 5 Gary 6 | 6 Helen 7 | -------------------------------------------------------------------------------- /examples/smoke/input/person_has_cancer.tsv: -------------------------------------------------------------------------------- 1 | 1 \N \N 2 | 2 \N \N 3 | 3 \N \N 4 | 4 \N \N 5 | 5 \N \N 6 | 6 \N \N 7 | -------------------------------------------------------------------------------- /examples/smoke/input/person_smokes.tsv: -------------------------------------------------------------------------------- 1 | 1 TRUE \N 2 | 2 \N \N 3 | 3 TRUE \N 4 | 4 \N \N 5 | 5 \N \N 6 | 6 \N \N 7 | -------------------------------------------------------------------------------- /examples/spouse/.gitignore: -------------------------------------------------------------------------------- 1 | /input/signalmedia 2 | /run 3 | /snapshot 4 | /search 5 | /mindbender/*.js 6 | /mindbender/*.js.map 7 | /mindtagger/.mindtagger.conf.json 8 | /mindtagger/input.csv 9 | /mindtagger/tags.json 10 | -------------------------------------------------------------------------------- /examples/spouse/README.md: -------------------------------------------------------------------------------- 1 | # Spouse Example for DeepDive Tutorial 2 | 3 | See the [DeepDive tutorial in the documentation](../../doc/example-spouse.md) for full detail. 4 | -------------------------------------------------------------------------------- /examples/spouse/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_spouse_$USER 2 | -------------------------------------------------------------------------------- /examples/spouse/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.calibration.holdout_fraction:0.25 2 | deepdive.sampler.sampler_args: "-l 1000 -i 1000 --alpha 0.01 --sample_evidence" 3 | -------------------------------------------------------------------------------- /examples/spouse/input/articles-100.tsj.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/articles-100.tsj.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/articles-100.tsv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/articles-100.tsv.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/articles-1000.tsj.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/articles-1000.tsj.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/articles-1000.tsv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/articles-1000.tsv.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/sentences-100.tsj.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/sentences-100.tsj.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/sentences-100.tsv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/sentences-100.tsv.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/sentences-1000.tsj.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/sentences-1000.tsj.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/sentences-1000.tsv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/sentences-1000.tsv.bz2 -------------------------------------------------------------------------------- /examples/spouse/input/spouses_dbpedia.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/examples/spouse/input/spouses_dbpedia.csv.bz2 -------------------------------------------------------------------------------- /examples/spouse/labeling/.gitignore: -------------------------------------------------------------------------------- 1 | .mindtagger.conf.json 2 | *.csv 3 | tags.json 4 | -------------------------------------------------------------------------------- /examples/spouse/labeling/has_spouse-precision-with_features/mindtagger.conf: -------------------------------------------------------------------------------- 1 | title: Labeling task for estimating has_spouse precision with features 2 | items: { 3 | file: "has_spouse-with_features.csv" 4 | key_columns: [p1_id, p2_id] 5 | } 6 | template: "template-with_features.html" 7 | -------------------------------------------------------------------------------- /examples/spouse/labeling/has_spouse-precision/mindtagger.conf: -------------------------------------------------------------------------------- 1 | title: Labeling task for estimating has_spouse precision 2 | items: { 3 | file: has_spouse.csv 4 | key_columns: [p1_id, p2_id] 5 | } 6 | template: template.html 7 | -------------------------------------------------------------------------------- /examples/spouse/mindbender/search-template/articles.html: -------------------------------------------------------------------------------- 1 |
2 |

{{source.content | limitTo:500}}...

3 |
4 |
5 |

{{source.content}}

6 |
7 | 8 | See anything 9 | related to article({{source.id}}) 10 | 11 | -------------------------------------------------------------------------------- /examples/spouse/mindbender/search-template/sentences.html: -------------------------------------------------------------------------------- 1 | sentence {{source.sentence_index}} in article({{source.doc_id}}): 2 |

{{source.tokens.join(" ")}}

3 | 4 | See anything 5 | related to sentence {{source.sentence_index}} 6 | / 7 | related to article({{source.doc_id}}) 8 | 9 | -------------------------------------------------------------------------------- /examples/spouse/mindbender/search-template/spouse_feature.html: -------------------------------------------------------------------------------- 1 | {{extraction.feature}} 2 | 3 | feature of 4 |
5 | 6 | -------------------------------------------------------------------------------- /examples/spouse/mindbender/search-template/spouses_dbpedia.html: -------------------------------------------------------------------------------- 1 | {{source.person1_name}} 2 | -- 3 | {{source.person2_name}} 4 | -------------------------------------------------------------------------------- /examples/template/.gitignore: -------------------------------------------------------------------------------- 1 | /run 2 | /snapshot 3 | -------------------------------------------------------------------------------- /examples/template/README.md: -------------------------------------------------------------------------------- 1 | # DeepDive Application Template 2 | 3 | This directory contains a skeleton for a new DeepDive application. 4 | See [DeepDive's documentation on application layout](http://deepdive.stanford.edu/deepdiveapp) for full detail. 5 | -------------------------------------------------------------------------------- /examples/template/app.ddlog: -------------------------------------------------------------------------------- 1 | # Almost everything about a DeepDive app goes into this DDlog file 2 | # See: http://deepdive.stanford.edu/writing-dataflow-ddlog 3 | -------------------------------------------------------------------------------- /examples/template/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/my_deepdive_app 2 | -------------------------------------------------------------------------------- /examples/template/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive { 2 | 3 | # Specify a holdout fraction 4 | calibration.holdout_fraction: 0.25 5 | 6 | } 7 | -------------------------------------------------------------------------------- /examples/template/input/init.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # A script for initializing an application 3 | set -eu 4 | cd "$(dirname "$0")" 5 | 6 | # TODO use `deepdive load TABLE FILE` command to load initial data from files 7 | # deepdive load "foo" foo.tsv 8 | # deepdive load "foo(column1,col2,col3)" foo_three_columns.tsv 9 | -------------------------------------------------------------------------------- /examples/template/udf/extractor_template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from deepdive import * 3 | 4 | @tsv_extractor 5 | @returns(lambda 6 | column1 = "text", 7 | column2 = "int", 8 | column3 = "float", 9 | :[]) 10 | def my_udf( 11 | column1 = "text", 12 | column2 = "int", 13 | column3 = "float", 14 | ): 15 | yield [ 16 | column1, 17 | column2, 18 | column3, 19 | ] 20 | -------------------------------------------------------------------------------- /extern/.gitignore: -------------------------------------------------------------------------------- 1 | /bundle.conf 2 | /.build 3 | -------------------------------------------------------------------------------- /extern/bundle-all.conf: -------------------------------------------------------------------------------- 1 | ### List of Bundled Dependencies 2 | 3 | bash 4 | coreutils 5 | jq 6 | pv 7 | moreutils 8 | graphviz 9 | bc 10 | pbzip2 11 | python-lib 12 | perl-local-lib 13 | 14 | postgresql 15 | -------------------------------------------------------------------------------- /extern/bundle-none.conf: -------------------------------------------------------------------------------- 1 | # NOTE this list is kept empty to bundle none of the runtime dependencies 2 | python-lib 3 | perl-local-lib 4 | -------------------------------------------------------------------------------- /extern/bundled/bash: -------------------------------------------------------------------------------- 1 | ../buildkit/depends/common/bash -------------------------------------------------------------------------------- /extern/bundled/bc: -------------------------------------------------------------------------------- 1 | ../buildkit/depends/common/bc -------------------------------------------------------------------------------- /extern/bundled/coreutils: -------------------------------------------------------------------------------- 1 | ../buildkit/depends/common/coreutils -------------------------------------------------------------------------------- /extern/bundled/graphviz: -------------------------------------------------------------------------------- 1 | ../buildkit/depends/common/graphviz-devel -------------------------------------------------------------------------------- /extern/bundled/jq: -------------------------------------------------------------------------------- 1 | ../buildkit/depends/common/jq -------------------------------------------------------------------------------- /extern/bundled/moreutils: -------------------------------------------------------------------------------- 1 | ../buildkit/depends/common/moreutils -------------------------------------------------------------------------------- /extern/bundled/pbzip2: -------------------------------------------------------------------------------- 1 | ../buildkit/depends/common/pbzip2 -------------------------------------------------------------------------------- /extern/bundled/postgresql/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # install PostgreSQL 3 | set -eu 4 | 5 | : ${POSTGRESQL_DOWNLOAD_BASEURL:=https://ftp.postgresql.org/pub/source} 6 | : ${POSTGRESQL_VERSION:=9.5.3} 7 | 8 | name=postgresql 9 | version=$POSTGRESQL_VERSION 10 | ext=.tar.bz2 11 | url=$POSTGRESQL_DOWNLOAD_BASEURL/v$version/$name-$version$ext 12 | 13 | fetch-configure-build-install $name-$version <&2 "Cannot run the template directly. Must be run through a .bats symlink pointing to this file ($BATS_TEST_FILENAME)" 8 | false 9 | esac 10 | 11 | @test "end to end test: $(basename "$BATS_TEST_FILENAME" .bats)" { 12 | cd "${BATS_TEST_FILENAME%.bats}" 13 | 14 | run_end_to_end.sh 15 | 16 | # check result 17 | ! [[ -x ./check_result ]] || 18 | ./check_result 19 | } 20 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/README.md: -------------------------------------------------------------------------------- 1 | # DimmWitted tests 2 | 3 | ## Running all tests 4 | 5 | ```bash 6 | bats *.bats 7 | ``` 8 | 9 | ## End to end test directories 10 | 11 | ### Factor graph TSV format 12 | 13 | See [text_format.md](../doc/text_format.md) 14 | 15 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin-performance.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")" 4 | . "$PWD"/helpers.bash 5 | 6 | cp -af biased_coin/. biased_coin-performance/ 7 | cd biased_coin-performance 8 | run_end_to_end.sh 9 | echo '-l 20000 -i 20000 -s 1 --alpha 0.1 --diminish 0.995 --sample_evidence --reg_param 0' >dw-args 10 | run_end_to_end.sh 11 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin.bats: -------------------------------------------------------------------------------- 1 | .end_to_end_test.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin.setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" 3 | dw text2bin variable biased_coin/variables.tsv biased_coin/graph.variables /dev/stderr 4 | dw text2bin factor biased_coin/factors.tsv biased_coin/graph.factors /dev/stderr 4 1 1 5 | dw text2bin weight biased_coin/weights.tsv biased_coin/graph.weights /dev/stderr 6 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin/dw-args: -------------------------------------------------------------------------------- 1 | -l 2000 -i 2000 --alpha 0.1 --diminish 0.995 --sample_evidence --reg_param 0 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin/factors.text2bin-args: -------------------------------------------------------------------------------- 1 | 4 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin/factors.tsv: -------------------------------------------------------------------------------- 1 | 0 0 1 2 | 1 0 1 3 | 2 0 1 4 | 3 0 1 5 | 4 0 1 6 | 5 0 1 7 | 6 0 1 8 | 7 0 1 9 | 8 0 1 10 | 9 0 1 11 | 10 0 1 12 | 11 0 1 13 | 12 0 1 14 | 13 0 1 15 | 14 0 1 16 | 15 0 1 17 | 16 0 1 18 | 17 0 1 19 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin/graph.meta: -------------------------------------------------------------------------------- 1 | 1,18,18,18,out/test_coin/graph.weights,out/test_coin/graph.variables,out/test_coin/graph.factors,out/test_coin/cfg_rules 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin/variables.tsv: -------------------------------------------------------------------------------- 1 | 0 1 1 0 2 2 | 1 1 1 0 2 3 | 2 1 1 0 2 4 | 3 1 1 0 2 5 | 4 1 1 0 2 6 | 5 1 1 0 2 7 | 6 1 1 0 2 8 | 7 1 1 0 2 9 | 8 1 0 0 2 10 | 9 0 0 0 2 11 | 10 0 0 0 2 12 | 11 0 0 0 2 13 | 12 0 0 0 2 14 | 13 0 0 0 2 15 | 14 0 0 0 2 16 | 15 0 0 0 2 17 | 16 0 0 0 2 18 | 17 0 0 0 2 19 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin/weights.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_continuous.bats: -------------------------------------------------------------------------------- 1 | .end_to_end_test.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_continuous/dw-args: -------------------------------------------------------------------------------- 1 | -l 2000 -i 2000 --alpha 0.1 --diminish 0.995 --sample_evidence --reg_param 0 -c 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_continuous/factors.text2bin-args: -------------------------------------------------------------------------------- 1 | 4 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_continuous/factors.tsv: -------------------------------------------------------------------------------- 1 | 0 0 1 2 | 1 0 1 3 | 2 0 1 4 | 3 0 1 5 | 4 0 -1.5 6 | 5 0 -0.5 7 | 6 0 3.5 8 | 7 0 0.5 9 | 8 0 6 10 | 9 0 1 11 | 10 0 1 12 | 11 0 1 13 | 12 0 1 14 | 13 0 1 15 | 14 0 1 16 | 15 0 1 17 | 16 0 1 18 | 17 0 1 19 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_continuous/graph.meta: -------------------------------------------------------------------------------- 1 | 1,18,18,18,out/test_coin/graph.weights,out/test_coin/graph.variables,out/test_coin/graph.factors,out/test_coin/cfg_rules 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_continuous/variables.tsv: -------------------------------------------------------------------------------- 1 | 0 1 1 0 2 2 | 1 1 1 0 2 3 | 2 1 1 0 2 4 | 3 1 1 0 2 5 | 4 1 1 0 2 6 | 5 1 1 0 2 7 | 6 1 1 0 2 8 | 7 1 1 0 2 9 | 8 1 0 0 2 10 | 9 0 0 0 2 11 | 10 0 0 0 2 12 | 11 0 0 0 2 13 | 12 0 0 0 2 14 | 13 0 0 0 2 15 | 14 0 0 0 2 16 | 15 0 0 0 2 17 | 16 0 0 0 2 18 | 17 0 0 0 2 19 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_continuous/weights.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness.bats: -------------------------------------------------------------------------------- 1 | .end_to_end_test.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness/domains.tsv: -------------------------------------------------------------------------------- 1 | 0 2 {0,1} {0.2,0.8} 2 | 1 2 {0,1} {0.2,0.8} 3 | 2 2 {0,1} {0.2,0.8} 4 | 3 2 {0,1} {0.2,0.8} 5 | 4 2 {0,1} {0.2,0.8} 6 | 5 2 {0,1} {0.2,0.8} 7 | 6 2 {0,1} {0.1,0.9} 8 | 7 2 {0,1} {0.45,0.55} 9 | 8 2 {0,1} {0.05,0.95} 10 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness/dw-args: -------------------------------------------------------------------------------- 1 | -l 2000 -i 2000 --alpha 0.01 --diminish 0.995 --sample_evidence --reg_param 0 --noise_aware 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness/factors.text2bin-args: -------------------------------------------------------------------------------- 1 | 12 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness/factors.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 1 2 | 1 0 0 1 3 | 2 0 0 1 4 | 3 0 0 1 5 | 4 0 0 1 6 | 5 0 0 1 7 | 6 0 0 1 8 | 7 0 0 1 9 | 8 0 0 1 10 | 9 0 0 1 11 | 10 0 0 1 12 | 11 0 0 1 13 | 12 0 0 1 14 | 13 0 0 1 15 | 14 0 0 1 16 | 15 0 0 1 17 | 16 0 0 1 18 | 17 0 0 1 19 | 0 1 1 1 20 | 1 1 1 1 21 | 2 1 1 1 22 | 3 1 1 1 23 | 4 1 1 1 24 | 5 1 1 1 25 | 6 1 1 1 26 | 7 1 1 1 27 | 8 1 1 1 28 | 9 1 1 1 29 | 10 1 1 1 30 | 11 1 1 1 31 | 12 1 1 1 32 | 13 1 1 1 33 | 14 1 1 1 34 | 15 1 1 1 35 | 16 1 1 1 36 | 17 1 1 1 37 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness/graph.meta: -------------------------------------------------------------------------------- 1 | 2,18,36,36,out/test_coin/graph.weights,out/test_coin/graph.variables,out/test_coin/graph.factors,out/test_coin/cfg_rules 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness/variables.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 1 2 2 | 1 0 0 1 2 3 | 2 0 0 1 2 4 | 3 0 0 1 2 5 | 4 0 0 1 2 6 | 5 0 0 1 2 7 | 6 0 0 1 2 8 | 7 0 0 1 2 9 | 8 0 0 1 2 10 | 9 0 0 1 2 11 | 10 0 0 1 2 12 | 11 0 0 1 2 13 | 12 0 0 1 2 14 | 13 0 0 1 2 15 | 14 0 0 1 2 16 | 15 0 0 1 2 17 | 16 0 0 1 2 18 | 17 0 0 1 2 19 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_truthiness/weights.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 2 | 1 0 0 3 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_with_multinomial.bats: -------------------------------------------------------------------------------- 1 | .end_to_end_test.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_with_multinomial/dw-args: -------------------------------------------------------------------------------- 1 | -l 2000 -i 2000 --alpha 0.01 --diminish 0.995 --sample_evidence --reg_param 0 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_with_multinomial/factors.text2bin-args: -------------------------------------------------------------------------------- 1 | 12 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_with_multinomial/factors.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 1 2 | 1 0 0 1 3 | 2 0 0 1 4 | 3 0 0 1 5 | 4 0 0 1 6 | 5 0 0 1 7 | 6 0 0 1 8 | 7 0 0 1 9 | 8 0 0 1 10 | 9 0 0 1 11 | 10 0 0 1 12 | 11 0 0 1 13 | 12 0 0 1 14 | 13 0 0 1 15 | 14 0 0 1 16 | 15 0 0 1 17 | 16 0 0 1 18 | 17 0 0 1 19 | 0 1 1 1 20 | 1 1 1 1 21 | 2 1 1 1 22 | 3 1 1 1 23 | 4 1 1 1 24 | 5 1 1 1 25 | 6 1 1 1 26 | 7 1 1 1 27 | 8 1 1 1 28 | 9 1 1 1 29 | 10 1 1 1 30 | 11 1 1 1 31 | 12 1 1 1 32 | 13 1 1 1 33 | 14 1 1 1 34 | 15 1 1 1 35 | 16 1 1 1 36 | 17 1 1 1 37 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_with_multinomial/graph.meta: -------------------------------------------------------------------------------- 1 | 2,18,36,36,out/test_coin/graph.weights,out/test_coin/graph.variables,out/test_coin/graph.factors,out/test_coin/cfg_rules 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_with_multinomial/variables.tsv: -------------------------------------------------------------------------------- 1 | 0 1 1 1 2 2 | 1 1 1 1 2 3 | 2 1 1 1 2 4 | 3 1 1 1 2 5 | 4 1 1 1 2 6 | 5 1 1 1 2 7 | 6 1 1 1 2 8 | 7 1 1 1 2 9 | 8 1 0 1 2 10 | 9 0 0 1 2 11 | 10 0 0 1 2 12 | 11 0 0 1 2 13 | 12 0 0 1 2 14 | 13 0 0 1 2 15 | 14 0 0 1 2 16 | 15 0 0 1 2 17 | 16 0 0 1 2 18 | 17 0 0 1 2 19 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/biased_coin_with_multinomial/weights.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 2 | 1 0 0 3 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/binary_format_test.bats: -------------------------------------------------------------------------------- 1 | .gtest.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/binary_format_test.setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" 3 | dw text2bin variable biased_coin/variables.tsv biased_coin/graph.variables /dev/stderr 4 | dw text2bin factor biased_coin/factors.tsv biased_coin/graph.factors /dev/stderr 4 1 1 5 | dw text2bin weight biased_coin/weights.tsv biased_coin/graph.weights /dev/stderr 6 | dw text2bin domain domains/domains.tsv domains/graph.domains /dev/stderr 7 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/domains/domains.tsv: -------------------------------------------------------------------------------- 1 | 0 1 {1} {0} 2 | 1 2 {0,3} {0,0} 3 | 2 3 {1,3,5} {0,0,0} 4 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/factor_graph_test.bats: -------------------------------------------------------------------------------- 1 | .gtest.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/factor_graph_test.setup.sh: -------------------------------------------------------------------------------- 1 | biased_coin.setup.sh -------------------------------------------------------------------------------- /inference/dimmwitted/test/factor_test.bats: -------------------------------------------------------------------------------- 1 | .gtest.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/helpers.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # put source tree root on PATH 4 | PATH="${BASH_SOURCE%/test/helpers.bash}:${BASH_SOURCE%/helpers.bash}:$PATH" 5 | 6 | compare_binary_with_xxd_text() { 7 | local expected_xxd_txt=$1 8 | local actual_bin=$2 9 | if ! cmp <(xxd -r "$expected_xxd_txt") "$actual_bin"; then 10 | diff -u "$expected_xxd_txt" <(xxd "$actual_bin") 11 | false 12 | fi 13 | } 14 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/loading_test.bats: -------------------------------------------------------------------------------- 1 | .gtest.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/loading_test.setup.sh: -------------------------------------------------------------------------------- 1 | biased_coin.setup.sh -------------------------------------------------------------------------------- /inference/dimmwitted/test/partial_observation.bats: -------------------------------------------------------------------------------- 1 | .end_to_end_test.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/partial_observation/dw-args: -------------------------------------------------------------------------------- 1 | -l 500 -i 500 --alpha 0.1 --reg_param 0 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/partial_observation/factors.text2bin-args: -------------------------------------------------------------------------------- 1 | 3 2 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/partial_observation/factors.tsv: -------------------------------------------------------------------------------- 1 | 0 4 0 1 2 | 4 8 1 1 3 | 1 5 0 1 4 | 5 9 1 1 5 | 2 6 0 1 6 | 6 10 1 1 7 | 3 7 0 1 8 | 7 11 1 1 9 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/partial_observation/graph.meta: -------------------------------------------------------------------------------- 1 | 2,12,8,16,,, -------------------------------------------------------------------------------- /inference/dimmwitted/test/partial_observation/variables.tsv: -------------------------------------------------------------------------------- 1 | 0 1 1 0 2 2 | 1 1 1 0 2 3 | 2 1 1 0 2 4 | 3 1 1 0 2 5 | 4 1 1 0 2 6 | 5 0 0 0 2 7 | 6 0 0 0 2 8 | 7 0 0 0 2 9 | 8 1 1 0 2 10 | 9 1 1 0 2 11 | 10 1 1 0 2 12 | 11 1 1 0 2 13 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/partial_observation/weights.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 2 | 1 0 0 3 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sampler_test.bats: -------------------------------------------------------------------------------- 1 | .gtest.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains.bats: -------------------------------------------------------------------------------- 1 | .end_to_end_test.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/domains.tsv: -------------------------------------------------------------------------------- 1 | 14 3 {0,1,3} {0,0,0} 2 | 15 1 {1} {0} 3 | 16 2 {1,3} {0.0,0} 4 | 17 1 {1} {0} 5 | 18 3 {0,1,3} {0,0,0} 6 | 19 2 {1,3} {0,0} 7 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/dw-args: -------------------------------------------------------------------------------- 1 | -l 4000 -i 2000 --alpha 0.01 --diminish 0.999 --reg_param 0 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/factors.text2bin-args: -------------------------------------------------------------------------------- 1 | 12 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/factors2.text2bin-args: -------------------------------------------------------------------------------- 1 | 12 2 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/factors2.tsv: -------------------------------------------------------------------------------- 1 | 14 15 0 1 4 1 2 | 14 15 1 1 5 1 3 | 14 15 3 1 7 1 4 | 15 16 1 1 5 1 5 | 15 16 1 3 6 1 6 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/graph.meta: -------------------------------------------------------------------------------- 1 | 8,20,67,72,graph.weights,graph.variables,graph.factors 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/variables.tsv: -------------------------------------------------------------------------------- 1 | 0 1 0 1 4 2 | 1 1 1 1 4 3 | 2 1 1 1 4 4 | 3 1 2 1 4 5 | 4 1 2 1 4 6 | 5 1 2 1 4 7 | 6 1 3 1 4 8 | 7 1 3 1 4 9 | 8 1 3 1 4 10 | 9 1 3 1 4 11 | 10 0 0 1 4 12 | 11 0 0 1 4 13 | 12 0 0 1 4 14 | 13 0 0 1 4 15 | 14 0 0 1 3 16 | 15 0 0 1 1 17 | 16 0 0 1 2 18 | 17 0 0 1 1 19 | 18 0 0 1 3 20 | 19 0 0 1 2 21 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_domains/weights.tsv: -------------------------------------------------------------------------------- 1 | 0 1 0 2 | 1 0 0 3 | 2 0 0 4 | 3 0 0 5 | 4 1 2.0 6 | 5 1 3.0 7 | 6 1 2.0 8 | 7 1 0 9 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2.bats: -------------------------------------------------------------------------------- 1 | .end_to_end_test.bats.template -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/domains.tsv: -------------------------------------------------------------------------------- 1 | 0 3 {0,1,2} {0,0,0} 2 | 1 3 {1,2,3} {0,0,0} 3 | 2 2 {1,3} {0,0} 4 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/dw-args: -------------------------------------------------------------------------------- 1 | -l 0 -i 1000 --alpha 0.1 --sample_evidence 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/factors1.text2bin-args: -------------------------------------------------------------------------------- 1 | 12 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/factors1.tsv: -------------------------------------------------------------------------------- 1 | 0 0 18 1 2 | 0 1 19 1 3 | 0 2 20 1 4 | 1 1 19 1 5 | 1 2 20 1 6 | 1 3 21 1 7 | 2 1 22 1 8 | 2 3 23 1 9 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/factors2.text2bin-args: -------------------------------------------------------------------------------- 1 | 12 2 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/factors2.tsv: -------------------------------------------------------------------------------- 1 | 0 1 0 1 9 1 2 | 0 1 0 2 10 1 3 | 0 1 0 3 11 1 4 | 0 1 1 1 12 1 5 | 0 1 1 2 13 1 6 | 0 1 1 3 14 1 7 | 0 1 2 1 15 1 8 | 0 1 2 2 16 1 9 | 0 1 2 3 17 1 10 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/factors2a.text2bin-args: -------------------------------------------------------------------------------- 1 | 12 2 1 1 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/factors2a.tsv: -------------------------------------------------------------------------------- 1 | 0 1 0 1 0 1 2 | 0 1 0 2 1 1 3 | 0 1 0 3 2 1 4 | 0 1 1 1 3 1 5 | 0 1 1 2 4 1 6 | 0 1 1 3 5 1 7 | 0 1 2 1 6 1 8 | 0 1 2 2 7 1 9 | 0 1 2 3 8 1 10 | 0 1 0 1 0 1 11 | 0 1 1 1 2 1 12 | 0 1 1 2 4 1 13 | 0 1 2 1 6 1 14 | 0 1 2 2 7 1 15 | 0 1 2 3 8 1 16 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/graph.meta: -------------------------------------------------------------------------------- 1 | 24,3,32,56,graph.weights,graph.variables,graph.factors 2 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/variables.tsv: -------------------------------------------------------------------------------- 1 | 0 0 0 1 3 2 | 1 0 0 1 3 3 | 2 0 0 1 2 4 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/weights1.tsv: -------------------------------------------------------------------------------- 1 | 18 1 0 2 | 19 1 0 3 | 20 1 0 4 | 21 1 0 5 | 22 1 1 6 | 23 1 4 7 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/weights2.tsv: -------------------------------------------------------------------------------- 1 | 9 1 0 2 | 10 1 0 3 | 11 1 0 4 | 12 1 10 5 | 13 1 0 6 | 14 1 0 7 | 15 1 0 8 | 16 1 0 9 | 17 1 0 10 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/sparse_multinomial2/weights2a.tsv: -------------------------------------------------------------------------------- 1 | 0 1 0 2 | 1 1 0 3 | 2 1 0 4 | 3 1 0 5 | 4 1 0 6 | 5 1 0 7 | 6 1 0 8 | 7 1 0 9 | 8 1 0 10 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/test_main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main(int argc, char **argv) { 4 | testing::InitGoogleTest(&argc, argv); 5 | return RUN_ALL_TESTS(); 6 | } 7 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/text2bin/dd_factors.txt: -------------------------------------------------------------------------------- 1 | 0 {0} 0 2 | 0 {1,2} 1 3 | 0 {3,4,5} 2 4 | 0 {6,7,8,9} 3 5 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/text2bin/dd_variables.txt: -------------------------------------------------------------------------------- 1 | 0 0 0 0 2 2 | 1 0 0 0 2 3 | 2 0 0 0 2 4 | 3 0 0 0 2 5 | 4 0 0 0 2 6 | 5 0 0 0 2 7 | 6 0 0 0 2 8 | 7 0 0 0 2 9 | 8 0 0 0 2 10 | 9 0 0 0 2 11 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/text2bin/dd_weights.bin.txt: -------------------------------------------------------------------------------- 1 | 00000000: 0000 0000 0000 0000 0140 0800 0000 0000 .........@...... 2 | 00000010: 00 . 3 | -------------------------------------------------------------------------------- /inference/dimmwitted/test/text2bin/dd_weights.txt: -------------------------------------------------------------------------------- 1 | 0 1 3 2 | -------------------------------------------------------------------------------- /inference/restore_partitioned_vids: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | use strict; 3 | my $SHARD_BASE = $ENV{pid} << 48; 4 | while (<>) { 5 | my ($vid, @rest) = split(" "); 6 | $vid >= 0 or die "$vid is negative"; 7 | print join("\t", ($SHARD_BASE | $vid), @rest), "\n"; 8 | } 9 | -------------------------------------------------------------------------------- /inference/test/env.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | load ../../test/test_environ 3 | -------------------------------------------------------------------------------- /runner/compute-driver/torque/commons: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # commons -- Contains several convenience functions for the computers 3 | # 4 | ## 5 | 6 | hash() { 7 | input_str="$1" 8 | if which "shasum" > /dev/null; then 9 | h="shasum" 10 | elif which "sha1sum" > /dev/null; then 11 | h="sha1sum" 12 | else 13 | echo "Could not find SHA1 commands" 14 | exit 1 15 | fi 16 | 17 | echo "$input_str" | eval "$h" | cut -d' ' -f1 18 | } 19 | -------------------------------------------------------------------------------- /runner/compute-driver/torque/sched_kill: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # torque/sched_kill -- Kills a job specified by the user 3 | # This is the specific command used by the Torque scheduler 4 | ## 5 | set -euo pipefail 6 | 7 | JOB_ID=$1 8 | NUM_PROCESSES=$2 9 | 10 | if [[ $NUM_PROCESSES -gt 1 ]]; then 11 | qdel ${JOB_ID}[] 12 | else 13 | qdel $JOB_ID 14 | fi 15 | -------------------------------------------------------------------------------- /runner/compute-driver/torque/sched_status: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # torque/sched_status -- Returns the exit status of the specified job ID. 3 | ## 4 | set -euo pipefail 5 | 6 | JOB_ID=$1 7 | 8 | tracejob -q $JOB_ID | grep -o Exit_status=[0-9]*$ | sed 's/Exit_status=//g' 9 | -------------------------------------------------------------------------------- /runner/computers-default.conf: -------------------------------------------------------------------------------- 1 | deepdive.computers.local { type: local } 2 | deepdive.computer: local 3 | -------------------------------------------------------------------------------- /runner/deepdive-done: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive-done -- Checks whether given targets are all done 3 | # > deepdive done TARGET... 4 | # Exits with status 0 if all are done, otherwise 1. 5 | ## 6 | set -eu 7 | . resolve-args-to-do.sh 8 | exec make -C "$DEEPDIVE_APP"/run --dry-run --silent --question "$@" CMD_data=# CMD_process=# 9 | -------------------------------------------------------------------------------- /runner/deepdive-redo: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive-redo -- Runs processes after marking them as 'todo' 3 | # > deepdive redo TARGET... 4 | ## 5 | set -eu 6 | 7 | . deepdive-do-getopts.sh 8 | [[ $# -gt 0 ]] || usage "$0" "No TARGET to redo given" 9 | 10 | deepdive-mark todo "$@" 11 | onexit() { 12 | # restore timestamps if needed 13 | case $? in 14 | 130) # when deepdive-do was canceled 15 | deepdive-mark restore "$@" 16 | ;; 17 | esac 18 | } 19 | trap 'onexit "$@"' EXIT 20 | deepdive-do "$@" 21 | -------------------------------------------------------------------------------- /runner/mark_done: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # mark_done -- Marks given targets as done 3 | # > mark_done TARGET... 4 | ## 5 | set -eu 6 | for t; do 7 | mkdir -p "$(dirname "$t")" 8 | touch "$t".done 9 | done 10 | -------------------------------------------------------------------------------- /runner/mkmimo.build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # How to build the mkmimo submodule 3 | set -euo pipefail 4 | cd "${0%.build.sh}" 5 | 6 | make 7 | -------------------------------------------------------------------------------- /runner/reset_timestamp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # reset_timestamp -- Resets timestamp files by putting them aside 3 | # > reset_timestamp TIMESTAMP_FILE... 4 | ## 5 | set -eu 6 | 7 | : ${DEEPDIVE_LOG_LEVEL:=1} 8 | 9 | verboseLevel2=; [[ $DEEPDIVE_LOG_LEVEL -lt 2 ]] || verboseLevel2=v 10 | 11 | for ts in "$@"; do 12 | [[ -e "$ts" ]] || continue 13 | mv -f$verboseLevel2 --no-target-directory "$ts" "$ts~" 14 | done 15 | -------------------------------------------------------------------------------- /runner/restore_timestamp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # restore_timestamp -- Restores timestamp files prior to the previous reset 3 | # > restore_timestamp TIMESTAMP_FILE... 4 | ## 5 | set -eu 6 | 7 | : ${DEEPDIVE_LOG_LEVEL:=1} 8 | 9 | verboseLevel2=; [[ $DEEPDIVE_LOG_LEVEL -lt 2 ]] || verboseLevel2=v 10 | 11 | for ts in "$@"; do 12 | [[ -e "$ts~" ]] || continue 13 | mv -f$verboseLevel2 --no-target-directory "$ts~" "$ts" 14 | done 15 | -------------------------------------------------------------------------------- /runner/test/test_environ.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Unit tests for DeepDive runner 3 | # Usage: load test_environ # from .bats files typically on the same directory 4 | 5 | load ../../test/test_environ 6 | -------------------------------------------------------------------------------- /sandbox/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !deepdive-*.tar.gz 3 | !install* 4 | !stanford-corenlp 5 | -------------------------------------------------------------------------------- /sandbox/.gitignore: -------------------------------------------------------------------------------- 1 | /database 2 | /workdir 3 | 4 | /deepdive-*.tar.gz 5 | /install 6 | /install.sh 7 | /stanford-corenlp 8 | -------------------------------------------------------------------------------- /shell/deepdive-env: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive-env -- Runs commands inside DeepDive's environment 3 | # > deepdive env [COMMAND [ARG]...] 4 | ## 5 | # Author: Jaeho Shin 6 | # Created: 2012-05-10 7 | set -eu 8 | 9 | [ $# -gt 0 ] || set -- bash --norc -i 10 | 11 | exec "$@" 12 | -------------------------------------------------------------------------------- /shell/jq: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # jq -- a shebang interpreter for using jq executable files in a portable way 3 | set -e 4 | jq="$DEEPDIVE_HOME"/lib/bundled/.all/bin/jq # XXX hardcoded to minimize overhead 5 | [ -x "$jq" ] || jq=`which -a jq | grep -vxF "$0" | head -1` # XXX this is slow 6 | if test -n "$1" -a -f "$1" -a -x "$1"; then 7 | f=$1; shift 8 | d=`dirname "$f"` 9 | exec "$jq" -f "$f" -L "$d" "$@" 10 | else 11 | exec "$jq" "$@" 12 | fi 13 | -------------------------------------------------------------------------------- /shell/logging-with-ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # logging-with-ts -- Logs stdin to a file with timestamp on each line 3 | set -eu 4 | if [[ $# -gt 0 ]]; then 5 | for f; do mkdir -p "$(dirname "$f")"; done 6 | exec ts "%F %H:%M:%.S" | tee -a "$@" 7 | else 8 | exec ts "%F %H:%M:%.S" 9 | fi 10 | -------------------------------------------------------------------------------- /shell/make: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # make -- a portability wrapper that makes sure `make` within deepdive uses 3 | # uniform configuration, e.g., SHELL=bash, etc. 4 | set -e 5 | make=`which -a make | grep -vxF "$0" | head -1` # XXX this is slow 6 | SHELL="`which bash`" \ 7 | exec "$make" "$@" 8 | -------------------------------------------------------------------------------- /shell/test/test_environ.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Unit tests for DeepDive shell 3 | # Usage: load test_environ # from .bats files on the same directory 4 | 5 | load ../../test/test_environ 6 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | /*/*/run 2 | -------------------------------------------------------------------------------- /test/greenplum/biased_coin_example: -------------------------------------------------------------------------------- 1 | ../postgresql/biased_coin_example -------------------------------------------------------------------------------- /test/greenplum/biased_coin_example.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/biased_coin_example.bats -------------------------------------------------------------------------------- /test/greenplum/broken_example: -------------------------------------------------------------------------------- 1 | ../postgresql/broken_example -------------------------------------------------------------------------------- /test/greenplum/broken_example.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/broken_example.bats -------------------------------------------------------------------------------- /test/greenplum/chunking_example: -------------------------------------------------------------------------------- 1 | ../../examples/chunking -------------------------------------------------------------------------------- /test/greenplum/chunking_example.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/chunking_example.bats -------------------------------------------------------------------------------- /test/greenplum/db-assign_sequential_id.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/db-assign_sequential_id.bats -------------------------------------------------------------------------------- /test/greenplum/deepdive_create.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_create.bats -------------------------------------------------------------------------------- /test/greenplum/deepdive_load: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_load -------------------------------------------------------------------------------- /test/greenplum/deepdive_sql.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_sql.bats -------------------------------------------------------------------------------- /test/greenplum/deepdive_unload.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_unload.bats -------------------------------------------------------------------------------- /test/greenplum/should-work.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # A script to test whether the Greenplum tests should be done or not 3 | set -eu 4 | 5 | cd "$(dirname "$0")" 6 | load() { source "$1".bash; } 7 | load test_environ 8 | { 9 | # gpfdist should be on PATH to say Greenplum is there 10 | type gpfdist 11 | # also check database version 12 | [[ "$(DBNAME=postgres timeout 1s db-execute "COPY (SELECT VERSION() LIKE '%Greenplum%') TO STDOUT")" == t ]] # TODO move this check to db-init? 13 | } &>/dev/null 14 | -------------------------------------------------------------------------------- /test/greenplum/spouse_example: -------------------------------------------------------------------------------- 1 | ../../examples/spouse -------------------------------------------------------------------------------- /test/greenplum/spouse_example.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/spouse_example.bats -------------------------------------------------------------------------------- /test/greenplum/spouse_example.helpers.sh: -------------------------------------------------------------------------------- 1 | ../postgresql/spouse_example.helpers.sh -------------------------------------------------------------------------------- /test/greenplum/spouse_example_incremental.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/spouse_example_incremental.bats -------------------------------------------------------------------------------- /test/postgresql-xl/biased_coin_example: -------------------------------------------------------------------------------- 1 | ../postgresql/biased_coin_example -------------------------------------------------------------------------------- /test/postgresql-xl/biased_coin_example.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/biased_coin_example.bats -------------------------------------------------------------------------------- /test/postgresql-xl/chunking_example: -------------------------------------------------------------------------------- 1 | ../postgresql/chunking_example -------------------------------------------------------------------------------- /test/postgresql-xl/chunking_example.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/chunking_example.bats -------------------------------------------------------------------------------- /test/postgresql-xl/db-assign_sequential_id.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/db-assign_sequential_id.bats -------------------------------------------------------------------------------- /test/postgresql-xl/deepdive_create.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_create.bats -------------------------------------------------------------------------------- /test/postgresql-xl/deepdive_load: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_load -------------------------------------------------------------------------------- /test/postgresql-xl/deepdive_load.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_load.bats -------------------------------------------------------------------------------- /test/postgresql-xl/deepdive_sql.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_sql.bats -------------------------------------------------------------------------------- /test/postgresql-xl/deepdive_unload.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/deepdive_unload.bats -------------------------------------------------------------------------------- /test/postgresql-xl/partition_id_range.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/partition_id_range.bats -------------------------------------------------------------------------------- /test/postgresql-xl/should-work.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # A script to test whether the Greenplum tests should be done or not 3 | set -eu 4 | 5 | cd "$(dirname "$0")" 6 | load() { source "$1".bash; } 7 | load test_environ 8 | { 9 | # check database version 10 | [[ "$(DBNAME=postgres timeout 1s db-execute "COPY (SELECT VERSION() LIKE '%Postgres-XL%') TO STDOUT")" == t ]] # TODO move this check to db-init? 11 | } &>/dev/null 12 | -------------------------------------------------------------------------------- /test/postgresql-xl/spouse_example: -------------------------------------------------------------------------------- 1 | ../postgresql/spouse_example -------------------------------------------------------------------------------- /test/postgresql-xl/spouse_example.bats: -------------------------------------------------------------------------------- 1 | ../postgresql/spouse_example.bats -------------------------------------------------------------------------------- /test/postgresql-xl/spouse_example.helpers.sh: -------------------------------------------------------------------------------- 1 | ../postgresql/spouse_example.helpers.sh -------------------------------------------------------------------------------- /test/postgresql-xl/test_environ.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # A script that sets up environment for testing DeepDive against Greenplum 3 | 4 | # load common test environment settings 5 | load ../test_environ 6 | 7 | export DEEPDIVE_DB_URL="postgresql-xl://${TEST_GREENPLUM_DBHOST:-${TEST_DBHOST:-localhost}}/${TEST_DBNAME:-deepdive_test_$USER}" 8 | . load-db-driver.sh 2>/dev/null 9 | 10 | -------------------------------------------------------------------------------- /test/postgresql/biased_coin_example/app.ddlog: -------------------------------------------------------------------------------- 1 | flips ( 2 | trial int, 3 | outcome bool 4 | ). 5 | 6 | coin? ( 7 | trial int 8 | ). 9 | 10 | coin(t) = out :- flips(t, out). 11 | 12 | @weight("?") coin(p) :- TRUE. 13 | -------------------------------------------------------------------------------- /test/postgresql/biased_coin_example/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_biased_coin_$USER 2 | -------------------------------------------------------------------------------- /test/postgresql/biased_coin_example/deepdive.conf: -------------------------------------------------------------------------------- 1 | deepdive.sampler.sampler_args: "-l 2000 -i 500 --alpha 0.1 --diminish 0.99 --quiet" 2 | -------------------------------------------------------------------------------- /test/postgresql/biased_coin_example/input/flips.tsv: -------------------------------------------------------------------------------- 1 | 1 true 2 | 2 true 3 | 3 true 4 | 4 true 5 | 5 true 6 | 6 true 7 | 7 true 8 | 8 true 9 | 9 false 10 | 10 \N 11 | 11 \N 12 | 12 \N 13 | 13 \N 14 | 14 \N 15 | -------------------------------------------------------------------------------- /test/postgresql/broken_example.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | # Tests for broken example 3 | 4 | load test_environ 5 | 6 | @test "$DBVARIANT DeepDive returns error for a broken application" { 7 | cd "$BATS_TEST_DIRNAME"/broken_example || skip 8 | ! deepdive compile || 9 | ! deepdive redo process/init/app || 10 | ! deepdive redo data/model/{weights,probabilities} || false 11 | } 12 | -------------------------------------------------------------------------------- /test/postgresql/broken_example/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_broken_$USER 2 | -------------------------------------------------------------------------------- /test/postgresql/broken_example/deepdive.conf: -------------------------------------------------------------------------------- 1 | # BrokenTest deepdive.conf 2 | 3 | deepdive.schema.variables { 4 | coin.is_correct: BADCONFIG___ 5 | } 6 | 7 | deepdive.extraction.extractors: { 8 | } 9 | 10 | deepdive.inference.factors { 11 | test { 12 | input_query: """select id as "coin.id", is_correct as "coin.is_correct" from coin""" 13 | function: "IsTrue(coin.is_correct)" 14 | weight: "?" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/postgresql/broken_example/schema.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/test/postgresql/broken_example/schema.sql -------------------------------------------------------------------------------- /test/postgresql/chunking_example: -------------------------------------------------------------------------------- 1 | ../../examples/chunking -------------------------------------------------------------------------------- /test/postgresql/deepdive_load/test.app/.gitignore: -------------------------------------------------------------------------------- 1 | /run 2 | -------------------------------------------------------------------------------- /test/postgresql/deepdive_load/test.app/app.ddlog: -------------------------------------------------------------------------------- 1 | variable_table_declared_in_app?(doc_id text, start int, length int). 2 | -------------------------------------------------------------------------------- /test/postgresql/deepdive_load/test.app/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_test_$USER 2 | -------------------------------------------------------------------------------- /test/postgresql/deepdive_load/test.app/input/variable_table_declared_in_app.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HazyResearch/deepdive/93330afc1dc73d96cd4e61fc4cc13016565f3f54/test/postgresql/deepdive_load/test.app/input/variable_table_declared_in_app.tsv -------------------------------------------------------------------------------- /test/postgresql/deepdive_load/test.csv: -------------------------------------------------------------------------------- 1 | 123,foo bar,"{how,are,you}" 2 | -------------------------------------------------------------------------------- /test/postgresql/deepdive_load/test.tsv: -------------------------------------------------------------------------------- 1 | 123 foo bar {how,are,you} 2 | -------------------------------------------------------------------------------- /test/postgresql/failing_udf.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | # Tests for broken example 3 | 4 | load test_environ 5 | 6 | @test "$DBVARIANT UDFs with error should not be silenced" { 7 | cd "${BATS_TEST_FILENAME%.bats}" || skip "app directory not found" 8 | deepdive compile 9 | deepdive redo process/init/app 10 | deepdive redo process/fine_extractor 11 | ! deepdive redo process/failing_cmd_extractor || false 12 | ! deepdive redo process/failing_tsv_extractor || false 13 | ! deepdive redo process/failing_sql_extractor || false 14 | } 15 | -------------------------------------------------------------------------------- /test/postgresql/failing_udf/app.ddlog: -------------------------------------------------------------------------------- 1 | foo(x INT). 2 | -------------------------------------------------------------------------------- /test/postgresql/failing_udf/db.url: -------------------------------------------------------------------------------- 1 | postgresql://localhost/deepdive_broken_$USER 2 | -------------------------------------------------------------------------------- /test/postgresql/load-db-driver.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | # Tests for load-db-driver.sh 3 | load test_environ 4 | 5 | @test "db.url allows use of environment variables" { 6 | dbname=$( 7 | cd "$BATS_TEST_DIRNAME"/deepdive_load/test.app 8 | unset DEEPDIVE_DB_URL 9 | USER=foobar 10 | . load-db-driver.sh 11 | echo "$DBNAME" 12 | ) 13 | [[ "$dbname" = "deepdive_test_foobar" ]] 14 | } 15 | -------------------------------------------------------------------------------- /test/postgresql/multinomial_example: -------------------------------------------------------------------------------- 1 | ../../examples/multinomial -------------------------------------------------------------------------------- /test/postgresql/multinomial_example.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | # Tests for multinomial example 3 | 4 | load test_environ 5 | 6 | setup() { 7 | cd "$BATS_TEST_DIRNAME"/multinomial_example 8 | } 9 | 10 | @test "$DBVARIANT multinomial example gives correct probabilities" { 11 | ./run.sh 12 | # Tests are inside run.sh 13 | } 14 | 15 | -------------------------------------------------------------------------------- /test/postgresql/should-work.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # A script to test whether the PostgreSQL tests should be done or not 3 | set -eu 4 | 5 | cd "$(dirname "$0")" 6 | load() { source "$1".bash; } 7 | load test_environ 8 | { 9 | # try executing a SQL query against the configured database for tests 10 | [[ "$(DBNAME=postgres timeout 1s db-execute "COPY (SELECT VERSION() LIKE '%PostgreSQL%' 11 | AND NOT ( VERSION() LIKE '%Postgres-XL%' 12 | OR VERSION() LIKE '%Greenplum%' 13 | )) TO STDOUT")" == t ]] 14 | } &>/dev/null 15 | -------------------------------------------------------------------------------- /test/postgresql/smoke_example: -------------------------------------------------------------------------------- 1 | ../../examples/smoke -------------------------------------------------------------------------------- /test/postgresql/smoke_example.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | # Tests for smoke example 3 | 4 | load test_environ 5 | 6 | @test "$DBVARIANT smoke example" { 7 | cd "${BATS_TEST_FILENAME%.bats}"/ || skip 8 | deepdive compile 9 | deepdive redo process/init/app data/model/probabilities 10 | } 11 | -------------------------------------------------------------------------------- /test/postgresql/spouse_example: -------------------------------------------------------------------------------- 1 | ../../examples/spouse -------------------------------------------------------------------------------- /test/postgresql/spouse_example.helpers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | # Helper functions for spouse example for PostgreSQL 3 | 4 | getAccuracyPerCent() { 5 | local variable=${1:-has_spouse_label} 6 | deepdive sql eval " 7 | SELECT 100 * (num_correct::REAL / (num_correct + num_incorrect)) :: INT 8 | FROM ${variable}_calibration 9 | WHERE bucket = 9 10 | " | 11 | tee >(echo >&2 "Accuracy for $variable with p >= 0.9 is: $(cat)") 12 | } 13 | -------------------------------------------------------------------------------- /test/test_environ.bash: -------------------------------------------------------------------------------- 1 | # DeepDive test environment helper for BATS 2 | . "${BASH_SOURCE%/*}"/env.sh 3 | -------------------------------------------------------------------------------- /util/build/decrypt-travis-then-update-github-releases: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # See: .travis.yml's deploy section 3 | set -euo pipefail 4 | 5 | # Decrypt credentials for update-github-release-asset 6 | openssl aes-256-cbc -K "$encrypted_54046ad4fea4_key" -iv "$encrypted_54046ad4fea4_iv" -in .travis.tar.enc -out .travis.tar -d 7 | tar xpvf .travis.tar 8 | ln -sfnv "$PWD"/.travis/.{netrc,gist} ~/ 9 | chmod go= ~/.{netrc,gist} 10 | -------------------------------------------------------------------------------- /util/build/test/cosine_similarity_of: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | inner_product_of <(unit_vector <"$1") <(unit_vector <"$2") 3 | -------------------------------------------------------------------------------- /util/build/test/inner_product_of: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | paste -d '*' "$@" | sum 3 | -------------------------------------------------------------------------------- /util/build/test/l2_distance_of: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | paste -d '-' "$@" | squares | sum | sqrt 3 | -------------------------------------------------------------------------------- /util/build/test/normalize_by_sum: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | vs=$(cat) 4 | { 5 | echo 'scale=6' 6 | sed 's:$: / '"$( sum <<<"$vs" )"':' <<<"$vs" 7 | } | bc 8 | -------------------------------------------------------------------------------- /util/build/test/probability_distribution_of: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | 4 | Variable=$1; shift 5 | Out=$1; shift 6 | calibration_tsv="$Out"/calibration/${Variable}.label.tsv 7 | 8 | cut -f 3 "$calibration_tsv" 9 | -------------------------------------------------------------------------------- /util/build/test/sqrt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec bc <<<"sqrt($(tr '\n' ' '))" 3 | -------------------------------------------------------------------------------- /util/build/test/squares: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | sed 's/.*/(&)^2/' | bc 3 | -------------------------------------------------------------------------------- /util/build/test/sum: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | { 3 | tr '\n' + 4 | echo 0 5 | } | bc 6 | -------------------------------------------------------------------------------- /util/build/test/unit_vector: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eu 3 | vs=$(cat) 4 | vlen=$( (squares | sum | sqrt) <<<"$vs" ) 5 | { 6 | echo 'scale=6' 7 | sed 's:$: / '"$vlen"':' <<<"$vs" 8 | } | bc 9 | -------------------------------------------------------------------------------- /util/mindbender-wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive's wrapper script to mindbender with its bundled dependencies 3 | ## 4 | exec "$(dirname "$0")"/deepdive env \ 5 | sh -c 'exec $(deepdive-whereis installed mindbender/bin/mindbender) "$@"' -- "$@" 6 | -------------------------------------------------------------------------------- /util/mindbender.build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # How to build the Mindbender submodule 3 | set -euo pipefail 4 | cd "${0%.build.sh}" 5 | 6 | make polish 7 | -------------------------------------------------------------------------------- /util/nlp/deepdive-corenlp-curl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive-corenlp-curl -- Hits CoreNLP HTTP server URL with curl 3 | # $ export CORENLP_PORT=PORT 4 | # $ deepdive corenlp curl CURL_OPTS... 5 | ## 6 | set -euo pipefail 7 | 8 | : ${CORENLP_SERVER_ENDPOINT:=$(deepdive-corenlp-server-url)} 9 | 10 | exec curl -fsSL --globoff "$CORENLP_SERVER_ENDPOINT" "$@" 11 | -------------------------------------------------------------------------------- /util/nlp/deepdive-corenlp-installed: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive-corenlp-installed -- Checks whether CoreNLP is already installed or not 3 | # $ deepdive corenlp install 4 | ## 5 | set -euo pipefail 6 | 7 | : ${CORENLP_HOME:?} 8 | 9 | [[ -x "$CORENLP_HOME"/corenlp.sh ]] || 10 | error "No CoreNLP installed at $CORENLP_HOME" "Try: deepdive corenlp install" 11 | -------------------------------------------------------------------------------- /util/nlp/deepdive-corenlp-is-running: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive-corenlp-is-running -- Checks if CoreNLP HTTP server is running 3 | # $ export CORENLP_PORT=PORT 4 | # $ deepdive corenlp running 5 | ## 6 | set -euo pipefail 7 | 8 | : ${CORENLP_PORT:?} 9 | 10 | deepdive-corenlp-curl --data 'ping' &>/dev/null || 11 | error "CoreNLP server at CORENLP_PORT=$CORENLP_PORT not running" 12 | -------------------------------------------------------------------------------- /util/nlp/deepdive-corenlp-unique-port: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # deepdive-corenlp-unique-port -- Prints a unique port number determined by given identifiers 3 | # $ deepdive corenlp unique-port [$USER]... 4 | ## 5 | set -euo pipefail 6 | 7 | : ${CORENLP_MINPORT:=9000} 8 | : ${CORENLP_MAXPORT:=65535} 9 | 10 | [[ $# -gt 0 ]] || set -- "${USER:-${LOGNAME:-}}" 11 | 12 | hash=0x$(echo "$@" | sha1sum | cut -b1-5) 13 | echo $(( $hash % ($CORENLP_MAXPORT - $CORENLP_MINPORT + 1) + $CORENLP_MINPORT )) 14 | -------------------------------------------------------------------------------- /util/test/test_environ.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Unit tests for DeepDive util 3 | # Usage: load test_environ # from .bats files typically on the same directory 4 | load ../../test/test_environ 5 | --------------------------------------------------------------------------------