12 |
20 | );
21 | }
22 | }
23 |
24 | export default App;
25 |
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/App.test.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import ReactDOM from 'react-dom';
3 | import App from './App';
4 |
5 | it('renders without crashing', () => {
6 | const div = document.createElement('div');
7 | ReactDOM.render(
13 | 
14 |
16 | Welcome to React
15 |
17 | To get started, edit src/App.js and save to reload.
18 |
Not Found :(
6 | } 7 | 8 | export default NotFound 9 | -------------------------------------------------------------------------------- /internal-displacement-web/client/src/components/NotFound/index.spec.js: -------------------------------------------------------------------------------- 1 | import chai, { expect } from 'chai' 2 | import chaiEnzyme from 'chai-enzyme' 3 | import { shallow } from 'enzyme' 4 | import React from 'react' 5 | import sinon from 'sinon' 6 | import sinonChai from 'sinon-chai' 7 | 8 | import NotFound from './' 9 | 10 | chai.use(chaiEnzyme()) 11 | chai.use(sinonChai) 12 | 13 | describe('Geojson Custom overlay
80 | Map rendering
109 |
21 |
22 | {HeatMapOverlayRender(data) }
23 |
24 | )
25 | } else {
26 | return( )
27 | }
28 | };
29 | export const RenderMap = (props) => {
30 | console.log(props)
31 |
32 | if ( !props.mapData || props.mapData.length === 0) {
33 | return (
34 |
35 | Map rendering
47 | Maop here!!!
)
83 | };
84 |
85 |
86 | // export default RenderMap
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/containers/MapVizContainer/components/map/mapboxTest.js:
--------------------------------------------------------------------------------
1 |
2 | import React, {Component} from 'react';
3 | import {render} from 'react-dom';
4 | import mapboxgl from 'mapbox-gl';
5 | import * as d3 from 'd3';
6 | import { MAPBOX_ACCESS_TOKEN } from './../../constants/mapConstants';
7 |
8 | export const renderMap = (data, containerID = 'map', centerLat = 0, centerLng = 0, zoom=0, maxCount, minCount, maxRadius, minRadius) => {
9 | mapboxgl.accessToken = MAPBOX_ACCESS_TOKEN;
10 | let map = new mapboxgl.Map({
11 | container: containerID,
12 | style: 'mapbox://styles/mapbox/light-v9',
13 | center: [centerLng, centerLat],
14 | zoom: zoom
15 | });
16 |
17 | map.on('load', () => {
18 | map.addSource('idData', {
19 | 'type': 'geojson',
20 | 'data': data
21 | });
22 |
23 | map.addLayer({
24 | 'id': 'idData-circles',
25 | 'type': 'circle',
26 | 'source': 'idData',
27 | 'paint': {
28 | 'circle-color': {
29 | property: 'mag',
30 | stops: [
31 | [6, '#FCA107'],
32 | [8, '#7F3121']
33 | ]
34 | },
35 | 'circle-opacity': 0.75,
36 | 'circle-radius': {
37 | property: 'radius',
38 | "type": "exponential",
39 | "stops": [
40 | [{ "zoom": 0, "value": 1 }, 10],
41 | [{ "zoom": 0, "value": 10 }, 50],
42 | [{ "zoom": 0, "value": 100 }, 100],
43 | [{ "zoom": 5, "value": 1 }, 20],
44 | [{ "zoom": 5, "value": 10 }, 60],
45 | [{ "zoom": 5, "value": 100 }, 110],
46 | [{ "zoom": 10, "value": 1 }, 30],
47 | [{ "zoom": 10, "value": 10 }, 70],
48 | [{ "zoom": 10, "value": 100 }, 120],
49 | [{ "zoom": 15, "value": 1 }, 40],
50 | [{ "zoom": 15, "value": 10 }, 80],
51 | [{ "zoom": 15, "value": 100 }, 130],
52 | [{ "zoom": 20, "value": 1 }, 50],
53 | [{ "zoom": 20, "value": 10 }, 90],
54 | [{ "zoom": 20, "value": 100 }, 140]
55 | ]
56 | }
57 | }
58 | });
59 | });
60 | };
61 | export const renderVisualizationOverlay = (data) => {
62 | //
63 | // const param = {
64 | // props: this.props,
65 | // state: this.state,
66 | // // onWebGLInitialized: this._onWebGLInitialized,
67 | // // effects: this._effects,
68 | // }
69 |
70 | if (data) {
71 |
72 | return (
73 |
74 |
75 |
76 | )
77 | } else {
78 | return( )
79 | }
80 | };
81 | export const RenderMap = (props) => {
82 | console.log(props)
83 |
84 | if ( !props.mapData || props.mapData.length === 0) {
85 | return (
86 |
87 | Map rendering
99 | Maop here!!!
)
135 | };
136 |
137 |
138 | // export default RenderMap
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/containers/MapVizContainer/components/mapOverlays/customScatterOverlay.js:
--------------------------------------------------------------------------------
1 | import React, {Component} from 'react';
2 | import DeckGL, {ScatterplotLayer} from 'deck.gl';
3 |
4 | // export default ScaledScatterplotLayer extends ScatterplotLayer {
5 | //
6 | // }
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/containers/MapVizContainer/components/mapOverlays/displacementHeatmapOverlay.js:
--------------------------------------------------------------------------------
1 | import React, { Component } from 'react';
2 | import Immutable from 'immutable';
3 | // import DeckGL from 'deck.gl';
4 | // import HeatmapOverlay from 'react-map-gl-heatmap-overlay';
5 | import {ScatterplotOverlay} from 'react-map-gl';
6 |
7 |
8 |
9 | const HeatMapOverlayRender = (param) => {
10 | console.log('heatmapoverlay render', param.mapData.map(d => [d.long, d.lat]))
11 | const idData = Immutable.fromJS(param.mapData);
12 | // const idData = Immutable.fromJS(param.mapData.map(d => [d.long, d.lat]));
13 | const width = param.width;
14 | const height = param.height;
15 | const zoom = param.zoom || 0;
16 | // const { width, height, mapViewState } = param;
17 | return (
18 | //
14 | {this.props.children}
15 |
16 | )
17 | }
18 | }
19 |
20 | App.propTypes = propTypes;
21 |
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/containers/home.js:
--------------------------------------------------------------------------------
1 | import React from 'react'
2 |
3 | const Home = () => {
4 | return Home...nothing here yet
5 | 6 | } 7 | 8 | export default Home 9 | -------------------------------------------------------------------------------- /internal-displacement-web/client/src/index.css: -------------------------------------------------------------------------------- 1 | /*body {*/ 2 | /*margin: 0;*/ 3 | /*padding: 0;*/ 4 | /*font-family: sans-serif;*/ 5 | /*}*/ 6 | -------------------------------------------------------------------------------- /internal-displacement-web/client/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import { Router, browserHistory } from 'react-router'; 4 | 5 | import routes from './routes'; 6 | import Bootstrap from 'bootstrap/dist/css/bootstrap.css'; 7 | import './themeCss/css/main.css'; 8 | import 'bootstrap/dist/css/bootstrap-theme.css'; 9 | import { Provider } from 'react-redux'; 10 | import {createStore} from 'redux'; 11 | import mapReducer from './containers/MapVizContainer/reducers/mapReducers'; 12 | import Layout from './layout'; 13 | 14 | 15 | const store = createStore(mapReducer); 16 | 17 | // We require the routes and render to the DOM using ReactDOM API 18 | ReactDOM.render( 19 |
9 | {this.props.children}
10 |
11 |
12 | )
13 | }
14 | };
15 |
16 | export default Layout;
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/logo.svg:
--------------------------------------------------------------------------------
1 |
8 |
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/reducers/index.js:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Data4Democracy/internal-displacement/fb8fb880c6de4034a4ba1ef6e2fcaee921777c3c/internal-displacement-web/client/src/reducers/index.js
--------------------------------------------------------------------------------
/internal-displacement-web/client/src/routes.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import {Route, IndexRoute }from 'react-router';
3 | import App from './containers/app';
4 | import HomePage from './components/HomePage';
5 | import MapVizPage from './containers/MapVizContainer/MapVizPage';
6 | import MapVizPageTest from './containers/MapVizContainer/MapVizExample';
7 | import MapVizPageScatter from './containers/MapVizContainer/MapVizPage.scatter';
8 |
9 | let routes = (
10 |
83 |
88 |
89 | Article 1
84 |85 | Flash flooding across Afghanistan and Pakistan has left more than 160 dead and dozens stranded in one of South Asia\'s worst natural disasters this year, say officials. The flooding, caused by unusually heavy rain, has left villagers stuck in remote areas without shelter, food or power. Mountainous Afghanistan was the worst hit, with 61 people killed and approximately 500 traditional mud-brick homes washed away in more than a dozen villages in Sarobihomes500 traditional mud-brick homes washed away in more than a dozen villages in Sarobi, a rural district less than an hour from Kabul, officials said. Floods left a village devastated in the remote eastern Afghan province of Nuristan. At least 60 homes were destroyedhomes60 homes were destroyed across three districts, said provincial spokesman Mohammad Yusufi. No one was killed. Authorities have been unable to deliver aid to some badly affected villages by land as roads in the area are controlled by the Taliban, Yusufi added. “We have asked the national government for help as have an overwhelming number of locals asking for assistance, but this is a Taliban-ridden area,” Yusufi said. At least 24 people were also died in two other eastern border provinces, Khost and Nangarharpeople24 people were also died in two other eastern border provinces, Khost and Nangarhar, according to local officials. More than fifty homes and shops were destroyedhomesfifty homes and shops were destroyed and thousands of acres of farmland flooded. In Pakistan monsoon rains claimed more than 80 lives, local media reported. Houses collapsingHouses collapsing, drowning and electrocution all pushed up the death toll, said Sindh Information Minister Sharjeel Inam Memon. In Karachi, the commercial capital and a southern port city that is home to 18 million people, poor neighborhoods were submerged waist-deep in water and many precincts suffered long power outages. Deaths were also reported in the north and west of the country. 86 |
87 |
90 |
95 |
96 |
97 |
--------------------------------------------------------------------------------
/production-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3'
2 | services:
3 | jupyter:
4 | image: aneel/internal-displacement-jupyter:spacy
5 | command: sh -c "jupyter notebook --no-browser --ip=0.0.0.0 --port 3323 /internal-displacement/notebooks"
6 | stdin_open: true
7 | tty: true
8 | ports:
9 | - "3323:3323"
10 | env_file: production.env
11 | nodejs:
12 | build: internal-displacement-web
13 | image: internal-displacement-web
14 | volumes:
15 | - ./internal-displacement-web/src:/internal-displacement-web/src
16 | - ./internal-displacement-web/package.json:/internal-displacement-web/package.json
17 | - ./internal-displacement-web/index.html:/internal-displacement-web/index.html
18 | ports:
19 | - "3322:3322"
20 | env_file: production.env
21 | nginx:
22 | build: docker/nginx
23 | image: nginx
24 | ports:
25 | - "80:80"
26 | depends_on:
27 | - nodejs
28 |
--------------------------------------------------------------------------------
/production.env:
--------------------------------------------------------------------------------
1 | DB_HOST=internal-displacement.cf1y5y4ffeey.us-west-2.rds.amazonaws.com
2 | DB_USER=d4d
3 | DB_PASS=
4 | DB_NAME=id
5 | PYTHONPATH=/internal-displacement
6 | NODE_ENV=production
7 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | appnope==0.1.0
2 | beautifulsoup4==4.5.3
3 | bleach==1.5.0
4 | cssselect==1.0.1
5 | decorator==4.0.11
6 | entrypoints==0.2.2
7 | feedfinder2==0.0.4
8 | feedparser==5.2.1
9 | html5lib==0.9999999
10 | idna==2.2
11 | ipykernel==4.5.2
12 | ipython==5.2.2
13 | ipython-genutils==0.1.0
14 | ipywidgets==5.2.2
15 | jieba3k==0.35.1
16 | Jinja2==2.9.5
17 | jsonschema==2.5.1
18 | jupyter==1.0.0
19 | jupyter-client==4.4.0
20 | jupyter-console==5.1.0
21 | jupyter-core==4.2.1
22 | langdetect==1.0.7
23 | lxml==3.7.2
24 | MarkupSafe==0.23
25 | mistune==0.7.3
26 | nbconvert==5.1.1
27 | nbformat==4.2.0
28 | newspaper3k==0.1.9
29 | nltk==3.2.2
30 | nose==1.3.7
31 | notebook==4.3.2
32 | numpy==1.12.1
33 | olefile==0.44
34 | pandas==0.19.2
35 | pandocfilters==1.4.1
36 | parsedatetime==2.3
37 | pexpect==4.2.1
38 | pickleshare==0.7.4
39 | Pillow==4.0.0
40 | prompt-toolkit==1.0.13
41 | psycopg2==2.6.2
42 | ptyprocess==0.5.1
43 | pycountry==17.1.8
44 | Pygments==2.2.0
45 | python-dateutil==2.6.0
46 | pytz==2016.10
47 | PyYAML==3.12
48 | pyzmq==16.0.2
49 | qtconsole==4.2.1
50 | records==0.5.0
51 | requests==2.13.0
52 | requests-file==1.4.1
53 | simplegeneric==0.8.1
54 | six==1.10.0
55 | spacy>=1.7.5
56 | SQLAlchemy==1.1.5
57 | terminado==0.6
58 | testpath==0.3
59 | textacy==0.3.3
60 | tldextract==2.0.2
61 | tornado==4.4.2
62 | traitlets==4.3.1
63 | wcwidth==0.1.7
64 | widgetsnbextension==1.2.6
65 |
--------------------------------------------------------------------------------
/sql_db.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Data4Democracy/internal-displacement/fb8fb880c6de4034a4ba1ef6e2fcaee921777c3c/sql_db.sqlite
--------------------------------------------------------------------------------
/workplan.md:
--------------------------------------------------------------------------------
1 |
2 | ### Raw Data
3 | We already have certain datasets containing URLs.
4 |
5 | __Activities:__
6 | - *Do we want to obtain more urls for our own dev / testing?*
7 |
8 | ### Scraping
9 | To be moved to `info-sources` repo. Interested parties can contribute there.
10 |
11 | Also covers the following two points from Filtering:
12 | - Identify language of a document (English vs. not English)
13 | - Broken URLs
14 |
15 | __Activities:__
16 | - Maintain some sort of link to `info-sources` to understand how these tools can be used / fit into the `internal-displacement` pipeline,
17 | i.e., function arguments, what they return, API type stuff.
18 |
19 | ### Pipeline
20 | Controls the input and output of data with respect to scraping and interpreting articles.
21 |
22 | __Activities__
23 | - Define how to export / store articles (discussion):
24 | + CSVs for now, but likely some sort of database will be necessary to facilitate the online / interactive tool for modeling and analysis
25 | - Take new data and extract only URLs, converting them into list
26 | - Checks and keeps a log of all previously scraped URLs
27 | - Checks whether a URL is from a domain on an RSS whitelist (future)
28 | - Manages output from scraper and either appends to original database or creates new database
29 |
30 | (please see [#86](https://github.com/Data4Democracy/internal-displacement/issues/86))
31 |
32 | ### Filtering (Interpreter)
33 | The third filtering requirement is more specific to the `internal-displacement` domain:
34 |
35 | - Filtering out documents not reporting on human mobility (binary classification)
36 |
37 | __Activities:__
38 | - Implement filtering out of documents not reporting on human mobility (modeling)
39 |
40 | > @milanoleonardo: *'this can be done by looking at the dependency trees of the sentences in the text to make sure there is a link between a “reporting term” and a “reporting unit” (see challenge for details). This would definitely remove all documents reporting on “hip displacement” or sentences like “displaced the body of people” etc.'*
41 |
42 | *How to test this functionality? Build some hand-crafted examples of things that shouldn't be included?*
43 |
44 | ### Tagging (Interpreter)
45 |
46 | - Classification of documents into 3 classes (Disasters, Conflict & Violence, Other)
47 |
48 | __Activities:__
49 | - Select approach / model that will allow for online learning or re-training in the future with new datasets. (discussion)
50 | - Build and train classifier for classifiying into the 3 required classes. (modeling)
51 |
52 | > @milanoleonardo: *'the best would be to set a fixed threshold on the probability distribution and assign a tag based on the content of the document.'*
53 |
54 | ### NLP
55 |
56 | "Fact extraction" from documents:
57 | - Publication date, locaction (ISO 3166 country codes) , reporting term, reporting units etc.
58 |
59 | __Activities:__
60 | - Select NLP tool or framework (discussion)
61 | - Build and test working tool for extracting facts (modeling)
62 |
63 | ### Article Class
64 |
65 | __Activities:__
66 | - Define the properties each Article needs to have and fill out code for instantiating new Articles (beginner friendly)
67 | - Create / fill-out functions for update articles properties by calling and using return values from Scraper and Interpreter functions (beginner friendly)
68 | - Fill out function for saving articles along with relevant properties (beginner friendly)
69 |
70 | ### Visualization
71 |
72 | Including but not limited to Interactive Map, Histograms
73 |
74 | __Activities:__
75 | - Design of visualizations (data-viz)
76 | - Selection of tool for online visualizations (i.e. D3) (discussion)
77 | - Create visualization functions that take in data in standard format and produce desired and interactive visualizations (data-viz)
78 |
79 | ### Quantitative Analysis
80 |
81 | Online tool that allows analysts to interact directly with data, choose what they visualize and how etc.
82 |
83 | __Activities:__
84 | - Design / build front-end page(s) for analysts
85 | - Create back-end functionality for connecting to database and returning necessary data, facts etc.
86 |
87 |
88 | ### Data Engineering
89 |
90 | We will need to construct a data pipeline / workflow to manage the end-to-end process, both for batch processing of files as well as (potentially) real-time processing of individual urls:
91 |
92 | + Data collection from various sources, i.e. existing csv files, new file provided by analysts etc.
93 | + Data pre-processing - applying the filtering tools created to exclude broken, irrelevant and non-English articles
94 | + Article classification - applying the pre-trained classifier, or training a new classifier
95 | + Fact Extraction - using NLP tools for extracting the key facts from the articles
96 | + Data storage - saving the article along with relevenat tags and extracted facts
97 | + API for enabling data analysts to interact with the data
98 |
99 |
100 | ### All Deliverables:
101 |
102 | - URL to working version of the tool
103 | - Source code repo
104 | - Analysis of the test dataset
105 | - User guide
106 | - Admin guide
107 |
108 | __Activities:__
109 | - Create, maintain and update user guide (documentation)
110 | - Create, maintain and update admin guide (documentation)
111 |
112 |
113 | ### Possible Libraries
114 |
115 | ___NLP:___
116 | - nltk
117 | - Tensor Flow
118 | - Spacy
119 |
120 | ___Text parsing and fact extraction:___
121 | - mordecai - Geoparsing (extracting relevant country)
122 | - Newspaper module (python 3)
123 | - goose-extractor - Text + meta-data extraction (only python 2)
124 |
125 |
126 |
--------------------------------------------------------------------------------
Article 2
91 |92 | Afghanistan state news agency, Bakhtar News Agency (BNA) report that at least 7 people have been killedpeople7 people have been killed in flash floods in Faryab Province in the north of the country. Flash floods in Baghlan Province have killedBaghlan Province have killed 1 person1 person and injured around 10 others. Flash floods struck on 08 May 2015 in Faryab Province after a period of heavy rainfall. The districts of Garyzan, Pashtunkot and Belcheragh were worst affected. BNA report that at least 7 people were killedpeople7 people were killed and over 1,500 homes damaged. The Faizabada-Takhar highway have been closed to traffic and wide areas of crops and orchards have suffered damaged. Kuwaiti News Agency (KUNA) also report that flooding struck in the Baghlan-i-Markazi district of Baghlan province, where 1 person was killed and several injured early on Saturday 09 May 2015killedBaghlan province, where 1 person was killed and several injured early on Saturday 09 May 2015person1 person was killed and several injured early on Saturday 09 May 2015. “There was heavy rain in Baghlan-e-Markazi district Friday evening and the people left their houses to safer areas. It was early Saturday when a flash flood hit the area and washed away more than 500 houses500 houses,” district Governor Gohar Khan Babri told reporters in provincial capital Pul-e-Khumri, 160 km north of Kabul. 93 |
94 |