├── .gitignore ├── LICENSE ├── README.md ├── RedisIntakeRedisClusterSample.py ├── cluster_pipeline.sh ├── cluster_pipeline_docker.sh ├── common └── utils.py ├── conf ├── Dockerfile ├── Dockerfile_redisai ├── cluster-config.conf ├── config.sh ├── create_redis_cluster.sh ├── create_redis_cluster_debug.sh ├── database.ini ├── docker-config.sh ├── docker-config_demo.sh ├── docker-config_prod.sh ├── launch_cluster_docker.sh ├── launch_cluster_docker_debug.sh ├── launch_cluster_docker_demo.sh ├── launch_cluster_local.sh ├── redis.conf └── rediscluster_streaming.json ├── config.py ├── data ├── README.md ├── metadata.zip └── sample_folder │ ├── PMC125340.xml.json │ ├── PMC125375.xml.json │ ├── PMC125543.xml.json │ ├── PMC126080.xml.json │ ├── PMC136939.xml.json │ ├── PMC140314.xml.json │ ├── PMC156578.xml.json │ ├── PMC169038.xml.json │ ├── PMC204495.xml.json │ ├── PMC212558.xml.json │ ├── PMC212776.xml.json │ ├── PMC222908.xml.json │ ├── PMC222911.xml.json │ ├── PMC222961.xml.json │ ├── PMC261870.xml.json │ ├── PMC270692.xml.json │ ├── PMC270701.xml.json │ ├── PMC270703.xml.json │ ├── PMC280685.xml.json │ ├── PMC293432.xml.json │ ├── PMC300679.xml.json │ └── PMC302072.xml.json ├── edges_to_graph_streamed.py ├── gears_pipeline_sentence.py ├── gears_pipeline_sentence_register.py ├── lang_detect_gears_paragraphs_force.py ├── parse_publish_dates.py ├── parse_publish_dates_threaded.py ├── requirements.txt ├── requirements_gears.txt ├── requirements_gears_aho.txt ├── requirements_gears_graph.txt ├── requirements_gears_lang.txt ├── requirements_gears_pipeline.txt ├── sentences_matcher_gears.py ├── sentences_matcher_register.py ├── set_debug_key.py └── streamed_based ├── README.md ├── cluster_pipeline_streams.sh ├── lang_detect_gears_paragraphs.py ├── requirements_gears_spacy.txt ├── requirements_gears_symspell.txt ├── spacy_sentences_streams.py └── symspell_sentences_streamed.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/README.md -------------------------------------------------------------------------------- /RedisIntakeRedisClusterSample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/RedisIntakeRedisClusterSample.py -------------------------------------------------------------------------------- /cluster_pipeline.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/cluster_pipeline.sh -------------------------------------------------------------------------------- /cluster_pipeline_docker.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/cluster_pipeline_docker.sh -------------------------------------------------------------------------------- /common/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/common/utils.py -------------------------------------------------------------------------------- /conf/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/Dockerfile -------------------------------------------------------------------------------- /conf/Dockerfile_redisai: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/Dockerfile_redisai -------------------------------------------------------------------------------- /conf/cluster-config.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/cluster-config.conf -------------------------------------------------------------------------------- /conf/config.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/config.sh -------------------------------------------------------------------------------- /conf/create_redis_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/create_redis_cluster.sh -------------------------------------------------------------------------------- /conf/create_redis_cluster_debug.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/create_redis_cluster_debug.sh -------------------------------------------------------------------------------- /conf/database.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/database.ini -------------------------------------------------------------------------------- /conf/docker-config.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/docker-config.sh -------------------------------------------------------------------------------- /conf/docker-config_demo.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/docker-config_demo.sh -------------------------------------------------------------------------------- /conf/docker-config_prod.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/docker-config_prod.sh -------------------------------------------------------------------------------- /conf/launch_cluster_docker.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/launch_cluster_docker.sh -------------------------------------------------------------------------------- /conf/launch_cluster_docker_debug.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/launch_cluster_docker_debug.sh -------------------------------------------------------------------------------- /conf/launch_cluster_docker_demo.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/launch_cluster_docker_demo.sh -------------------------------------------------------------------------------- /conf/launch_cluster_local.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/launch_cluster_local.sh -------------------------------------------------------------------------------- /conf/redis.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/redis.conf -------------------------------------------------------------------------------- /conf/rediscluster_streaming.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/conf/rediscluster_streaming.json -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/config.py -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/README.md -------------------------------------------------------------------------------- /data/metadata.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/metadata.zip -------------------------------------------------------------------------------- /data/sample_folder/PMC125340.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC125340.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC125375.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC125375.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC125543.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC125543.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC126080.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC126080.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC136939.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC136939.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC140314.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC140314.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC156578.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC156578.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC169038.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC169038.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC204495.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC204495.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC212558.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC212558.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC212776.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC212776.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC222908.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC222908.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC222911.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC222911.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC222961.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC222961.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC261870.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC261870.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC270692.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC270692.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC270701.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC270701.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC270703.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC270703.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC280685.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC280685.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC293432.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC293432.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC300679.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC300679.xml.json -------------------------------------------------------------------------------- /data/sample_folder/PMC302072.xml.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/data/sample_folder/PMC302072.xml.json -------------------------------------------------------------------------------- /edges_to_graph_streamed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/edges_to_graph_streamed.py -------------------------------------------------------------------------------- /gears_pipeline_sentence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/gears_pipeline_sentence.py -------------------------------------------------------------------------------- /gears_pipeline_sentence_register.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/gears_pipeline_sentence_register.py -------------------------------------------------------------------------------- /lang_detect_gears_paragraphs_force.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/lang_detect_gears_paragraphs_force.py -------------------------------------------------------------------------------- /parse_publish_dates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/parse_publish_dates.py -------------------------------------------------------------------------------- /parse_publish_dates_threaded.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/parse_publish_dates_threaded.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements_gears.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/requirements_gears.txt -------------------------------------------------------------------------------- /requirements_gears_aho.txt: -------------------------------------------------------------------------------- 1 | httpimport==0.7.2 2 | joblib==1.1.0 3 | pyahocorasick==1.4.4 4 | -------------------------------------------------------------------------------- /requirements_gears_graph.txt: -------------------------------------------------------------------------------- 1 | redis -------------------------------------------------------------------------------- /requirements_gears_lang.txt: -------------------------------------------------------------------------------- 1 | langdetect==1.0.8 -------------------------------------------------------------------------------- /requirements_gears_pipeline.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/requirements_gears_pipeline.txt -------------------------------------------------------------------------------- /sentences_matcher_gears.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/sentences_matcher_gears.py -------------------------------------------------------------------------------- /sentences_matcher_register.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/sentences_matcher_register.py -------------------------------------------------------------------------------- /set_debug_key.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/set_debug_key.py -------------------------------------------------------------------------------- /streamed_based/README.md: -------------------------------------------------------------------------------- 1 | This is kind of ugly code, but it works 2 | -------------------------------------------------------------------------------- /streamed_based/cluster_pipeline_streams.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/streamed_based/cluster_pipeline_streams.sh -------------------------------------------------------------------------------- /streamed_based/lang_detect_gears_paragraphs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/streamed_based/lang_detect_gears_paragraphs.py -------------------------------------------------------------------------------- /streamed_based/requirements_gears_spacy.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/streamed_based/requirements_gears_spacy.txt -------------------------------------------------------------------------------- /streamed_based/requirements_gears_symspell.txt: -------------------------------------------------------------------------------- 1 | symspellpy -------------------------------------------------------------------------------- /streamed_based/spacy_sentences_streams.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/streamed_based/spacy_sentences_streams.py -------------------------------------------------------------------------------- /streamed_based/symspell_sentences_streamed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-platform/HEAD/streamed_based/symspell_sentences_streamed.py --------------------------------------------------------------------------------