├── .gitignore ├── README.md ├── img ├── demo.gif └── pipeline.jpg ├── src ├── airflow │ ├── airflow.cfg │ ├── airflow.db │ ├── dags │ │ ├── run_tasks.py │ │ └── run_tasks.pyc │ └── unittests.cfg ├── flask │ ├── flaskexample │ │ ├── __init__.py │ │ ├── static │ │ │ ├── css │ │ │ │ ├── bootstrap-grid.css │ │ │ │ ├── bootstrap-grid.css.map │ │ │ │ ├── bootstrap-grid.min.css │ │ │ │ ├── bootstrap-grid.min.css.map │ │ │ │ ├── bootstrap-reboot.css │ │ │ │ ├── bootstrap-reboot.css.map │ │ │ │ ├── bootstrap-reboot.min.css │ │ │ │ ├── bootstrap-reboot.min.css.map │ │ │ │ ├── bootstrap.css │ │ │ │ ├── bootstrap.css.map │ │ │ │ ├── bootstrap.min.css │ │ │ │ ├── bootstrap.min.css.map │ │ │ │ └── signin.css │ │ │ ├── down.png │ │ │ ├── js │ │ │ │ ├── bootstrap.bundle.js │ │ │ │ ├── bootstrap.bundle.js.map │ │ │ │ ├── bootstrap.bundle.min.js │ │ │ │ ├── bootstrap.bundle.min.js.map │ │ │ │ ├── bootstrap.js │ │ │ │ ├── bootstrap.js.map │ │ │ │ ├── bootstrap.min.js │ │ │ │ └── bootstrap.min.js.map │ │ │ └── up.png │ │ ├── templates │ │ │ ├── index.html │ │ │ ├── input.html │ │ │ ├── output.html │ │ │ ├── results.html │ │ │ └── starter-template.css │ │ └── views.py │ ├── run.py │ └── tornadoapp.py ├── gdelt │ ├── data_gatherer.py │ ├── event_adder.py │ └── event_updater.py └── spark │ ├── database_operations.py │ ├── gdelt_schema_v1.py │ ├── gdelt_schema_v2.py │ ├── postgresql-42.2.5.jar │ ├── spark_job.py │ └── spark_run.py └── test ├── README.md ├── gdelt ├── 20190121220000.export2.CSV ├── events.csv ├── eventtypes.txt ├── gdelt_headers.txt ├── test_gdelt_download.py └── test_gdelt_query.py └── spark ├── spark_example.py ├── spark_run.py └── test_spark.py /.gitignore: -------------------------------------------------------------------------------- 1 | */*/config.ini 2 | */*/*.pyc 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/README.md -------------------------------------------------------------------------------- /img/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/img/demo.gif -------------------------------------------------------------------------------- /img/pipeline.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/img/pipeline.jpg -------------------------------------------------------------------------------- /src/airflow/airflow.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/airflow/airflow.cfg -------------------------------------------------------------------------------- /src/airflow/airflow.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/airflow/airflow.db -------------------------------------------------------------------------------- /src/airflow/dags/run_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/airflow/dags/run_tasks.py -------------------------------------------------------------------------------- /src/airflow/dags/run_tasks.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/airflow/dags/run_tasks.pyc -------------------------------------------------------------------------------- /src/airflow/unittests.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/airflow/unittests.cfg -------------------------------------------------------------------------------- /src/flask/flaskexample/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/__init__.py -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-grid.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-grid.css -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-grid.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-grid.css.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-grid.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-grid.min.css -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-grid.min.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-grid.min.css.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-reboot.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-reboot.css -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-reboot.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-reboot.css.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-reboot.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-reboot.min.css -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap-reboot.min.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap-reboot.min.css.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap.css -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap.css.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap.min.css -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/bootstrap.min.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/bootstrap.min.css.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/css/signin.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/css/signin.css -------------------------------------------------------------------------------- /src/flask/flaskexample/static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/down.png -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.bundle.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.bundle.js -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.bundle.js.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.bundle.js.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.bundle.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.bundle.min.js -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.bundle.min.js.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.bundle.min.js.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.js -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.js.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.js.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.min.js -------------------------------------------------------------------------------- /src/flask/flaskexample/static/js/bootstrap.min.js.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/js/bootstrap.min.js.map -------------------------------------------------------------------------------- /src/flask/flaskexample/static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/static/up.png -------------------------------------------------------------------------------- /src/flask/flaskexample/templates/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/templates/index.html -------------------------------------------------------------------------------- /src/flask/flaskexample/templates/input.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/templates/input.html -------------------------------------------------------------------------------- /src/flask/flaskexample/templates/output.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/templates/output.html -------------------------------------------------------------------------------- /src/flask/flaskexample/templates/results.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/templates/results.html -------------------------------------------------------------------------------- /src/flask/flaskexample/templates/starter-template.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/templates/starter-template.css -------------------------------------------------------------------------------- /src/flask/flaskexample/views.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/flaskexample/views.py -------------------------------------------------------------------------------- /src/flask/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/run.py -------------------------------------------------------------------------------- /src/flask/tornadoapp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/flask/tornadoapp.py -------------------------------------------------------------------------------- /src/gdelt/data_gatherer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/gdelt/data_gatherer.py -------------------------------------------------------------------------------- /src/gdelt/event_adder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/gdelt/event_adder.py -------------------------------------------------------------------------------- /src/gdelt/event_updater.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/gdelt/event_updater.py -------------------------------------------------------------------------------- /src/spark/database_operations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/spark/database_operations.py -------------------------------------------------------------------------------- /src/spark/gdelt_schema_v1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/spark/gdelt_schema_v1.py -------------------------------------------------------------------------------- /src/spark/gdelt_schema_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/spark/gdelt_schema_v2.py -------------------------------------------------------------------------------- /src/spark/postgresql-42.2.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/spark/postgresql-42.2.5.jar -------------------------------------------------------------------------------- /src/spark/spark_job.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/spark/spark_job.py -------------------------------------------------------------------------------- /src/spark/spark_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/src/spark/spark_run.py -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/README.md -------------------------------------------------------------------------------- /test/gdelt/20190121220000.export2.CSV: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/gdelt/20190121220000.export2.CSV -------------------------------------------------------------------------------- /test/gdelt/events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/gdelt/events.csv -------------------------------------------------------------------------------- /test/gdelt/eventtypes.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/gdelt/eventtypes.txt -------------------------------------------------------------------------------- /test/gdelt/gdelt_headers.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/gdelt/gdelt_headers.txt -------------------------------------------------------------------------------- /test/gdelt/test_gdelt_download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/gdelt/test_gdelt_download.py -------------------------------------------------------------------------------- /test/gdelt/test_gdelt_query.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/gdelt/test_gdelt_query.py -------------------------------------------------------------------------------- /test/spark/spark_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/spark/spark_example.py -------------------------------------------------------------------------------- /test/spark/spark_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/spark/spark_run.py -------------------------------------------------------------------------------- /test/spark/test_spark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/narayave/Insight-GDELT-Feed/HEAD/test/spark/test_spark.py --------------------------------------------------------------------------------