├── demo.png ├── src ├── index.css ├── index.js ├── App.test.js ├── App.css ├── App.js ├── logo.svg └── registerServiceWorker.js ├── public ├── favicon.ico ├── manifest.json └── index.html ├── .gitignore ├── package.json ├── app.js ├── readme.md └── index.html /demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hadrienj/cour_des_comptes/master/demo.png -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | padding: 0; 4 | font-family: sans-serif; 5 | } 6 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hadrienj/cour_des_comptes/master/public/favicon.ico -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import './index.css'; 4 | import App from './App'; 5 | import registerServiceWorker from './registerServiceWorker'; 6 | 7 | ReactDOM.render(, document.getElementById('root')); 8 | registerServiceWorker(); 9 | -------------------------------------------------------------------------------- /src/App.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import App from './App'; 4 | 5 | it('renders without crashing', () => { 6 | const div = document.createElement('div'); 7 | ReactDOM.render(, div); 8 | ReactDOM.unmountComponentAtNode(div); 9 | }); 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | 6 | # testing 7 | /coverage 8 | 9 | # production 10 | /build 11 | 12 | # misc 13 | .DS_Store 14 | .env.local 15 | .env.development.local 16 | .env.test.local 17 | .env.production.local 18 | 19 | npm-debug.log* 20 | yarn-debug.log* 21 | yarn-error.log* 22 | -------------------------------------------------------------------------------- /public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": "./index.html", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-header { 6 | background-color: #222; 7 | height: 150px; 8 | padding: 20px; 9 | color: white; 10 | } 11 | 12 | .App-title { 13 | font-size: 1.5em; 14 | } 15 | 16 | .App-intro { 17 | font-size: large; 18 | } 19 | 20 | @keyframes App-logo-spin { 21 | from { transform: rotate(0deg); } 22 | to { transform: rotate(360deg); } 23 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdc", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "react": "^16.3.1", 7 | "react-dom": "^16.3.1", 8 | "react-flexbox-grid": "^2.0.0", 9 | "react-scripts": "1.1.4", 10 | "react-table": "^6.8.0", 11 | "victory": "^0.25.7" 12 | }, 13 | "scripts": { 14 | "start": "react-scripts start", 15 | "build": "react-scripts build", 16 | "test": "react-scripts test --env=jsdom", 17 | "eject": "react-scripts eject" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | 3 | const path = "https://files.datapress.com/london/dataset/london-average-air-quality-levels/2018-03-23T11:58:28.61/air-quality-london-monthly-averages.csv"; 4 | 5 | const dataset = new recline.Model.Dataset({ 6 | url: path, 7 | backend: 'dataproxy', 8 | }); 9 | dataset.fetch(); 10 | 11 | 12 | // Show data table 13 | const grid = new recline.View.SlickGrid({ 14 | model: dataset, 15 | el: $('#tableContainer'), 16 | }); 17 | grid.visible = true; 18 | grid.render(); 19 | 20 | 21 | // Plot data 22 | const graph = new recline.View.Graph({ 23 | model: dataset, 24 | state: { 25 | graphType: "lines-and-points", 26 | group: "Month", 27 | series: ["London Mean Background:PM10 Particulate (ug/m3)", 28 | "London Mean Background:Nitrogen Dioxide (ug/m3)"] 29 | } 30 | }); 31 | $('#plotContainer').append(graph.el); 32 | graph.render(); 33 | graph.redraw(); 34 | 35 | }); -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 11 | 12 | 13 | 22 | React App 23 | 24 | 25 | 28 |
29 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Data for Good (2018) - #4: Cour des Comptes 2 | 3 | Open dataset visualization and previsualization. The goal of this project is to build visualization tools from tabular data (.csv, .xls...) in order to facilitate access, filtering and previsualization of open datasets. 4 | 5 | ## Goals 6 | 7 | - Previsualization: get the first few lines of the dataset to see its shape (column names, type of values etc.) before downloading it 8 | - Filtering: select only a subset of the dataset to download 9 | - Visualization: draw few charts from the data (eventually after filtering) 10 | 11 | ## Getting started 12 | 13 | The app can be started with `yarn start` in the root directory. 14 | 15 | This example takes a `.json` file from the [London air quality dataset](https://data.london.gov.uk/dataset/london-average-air-quality-levels) and shows an interactive table. A scatter plot of two variables is also displayed on the right. You should get the following result: 16 | 17 | ![demo](./demo.png) 18 | 19 | In this example, the table is created with [react-table](https://react-table.js.org/). The plot is made with [Victory](https://github.com/FormidableLabs/victory) from FormidableLabs. 20 | 21 | ## First attempt 22 | 23 | - Get the data with url request 24 | - Use [recline.js](http://okfnlabs.org/recline/) to do the previsualization and the visualization 25 | 26 | It seems that people working on recline.js moved to other projects. Also, the use of remote url to fetch csv data is not working well (503 errors). 27 | 28 | ## New directions 29 | 30 | ### Server side 31 | 32 | The current idea is to have an intermediate server written in Flask that will give the right `.csv` to the client. This could be done with [datasette](https://github.com/simonw/datasette) that can convert `.csv` files to API's. 33 | 34 | If it can't be done, the server will: 35 | 36 | - Get the right csv file according to request parameters 37 | - Preprocess the file 38 | - Convert it to json 39 | - Sending back the json to the client 40 | 41 | ### Client side 42 | 43 | On the client side, a pop-up window will be opened for previsualization containing small React components (for instance) with: 44 | 45 | - Table display for data previsualization 46 | - Some custom charts 47 | 48 | -------------------------------------------------------------------------------- /src/App.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import logo from './logo.svg'; 3 | import './App.css'; 4 | import { Grid, Row, Col } from 'react-flexbox-grid'; 5 | 6 | import { VictoryScatter } from 'victory'; 7 | import ReactTable from "react-table"; 8 | import "react-table/react-table.css"; 9 | 10 | // the data file is in public which is not good I think... 11 | const url = "./air-quality-london-time-of-day.json"; 12 | 13 | class App extends Component { 14 | constructor(props) { 15 | super(props); 16 | this.state = { 17 | data: [], 18 | columns: [], 19 | isLoading: true, 20 | }; 21 | this.getData(); 22 | } 23 | getData() { 24 | // get data from url (local in this case) 25 | fetch(url).then( (response) => { 26 | return response.json() 27 | }).then((data) => { 28 | // real data is in 'fields' 29 | const columns = this.getHeader(data); 30 | console.log(columns); 31 | return this.setState({ 32 | data: data, 33 | columns: columns, isLoading: false 34 | }); 35 | }); 36 | } 37 | getHeader(data) { 38 | // to do: check that all objects have the same keys 39 | const dataKeys = Object.keys(data[0]); 40 | const columns = []; 41 | dataKeys.map(x=>{ 42 | columns.push({ 43 | Header: x, 44 | accessor: x, 45 | }); 46 | }); 47 | return columns; 48 | } 49 | render() { 50 | if (this.state.isLoading) { 51 | return 'Loading...'; 52 | } 53 | 54 | return ( 55 |
56 | 57 | 58 | 64 | 65 | 66 | 73 | 74 | 75 |
76 | ); 77 | } 78 | } 79 | 80 | export default App; 81 | -------------------------------------------------------------------------------- /src/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 |

Example file from the London air quality dataset

46 |

Table:

47 |
48 |

Plot:

49 |
50 | 51 | -------------------------------------------------------------------------------- /src/registerServiceWorker.js: -------------------------------------------------------------------------------- 1 | // In production, we register a service worker to serve assets from local cache. 2 | 3 | // This lets the app load faster on subsequent visits in production, and gives 4 | // it offline capabilities. However, it also means that developers (and users) 5 | // will only see deployed updates on the "N+1" visit to a page, since previously 6 | // cached resources are updated in the background. 7 | 8 | // To learn more about the benefits of this model, read https://goo.gl/KwvDNy. 9 | // This link also includes instructions on opting out of this behavior. 10 | 11 | const isLocalhost = Boolean( 12 | window.location.hostname === 'localhost' || 13 | // [::1] is the IPv6 localhost address. 14 | window.location.hostname === '[::1]' || 15 | // 127.0.0.1/8 is considered localhost for IPv4. 16 | window.location.hostname.match( 17 | /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/ 18 | ) 19 | ); 20 | 21 | export default function register() { 22 | if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) { 23 | // The URL constructor is available in all browsers that support SW. 24 | const publicUrl = new URL(process.env.PUBLIC_URL, window.location); 25 | if (publicUrl.origin !== window.location.origin) { 26 | // Our service worker won't work if PUBLIC_URL is on a different origin 27 | // from what our page is served on. This might happen if a CDN is used to 28 | // serve assets; see https://github.com/facebookincubator/create-react-app/issues/2374 29 | return; 30 | } 31 | 32 | window.addEventListener('load', () => { 33 | const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`; 34 | 35 | if (isLocalhost) { 36 | // This is running on localhost. Lets check if a service worker still exists or not. 37 | checkValidServiceWorker(swUrl); 38 | 39 | // Add some additional logging to localhost, pointing developers to the 40 | // service worker/PWA documentation. 41 | navigator.serviceWorker.ready.then(() => { 42 | console.log( 43 | 'This web app is being served cache-first by a service ' + 44 | 'worker. To learn more, visit https://goo.gl/SC7cgQ' 45 | ); 46 | }); 47 | } else { 48 | // Is not local host. Just register service worker 49 | registerValidSW(swUrl); 50 | } 51 | }); 52 | } 53 | } 54 | 55 | function registerValidSW(swUrl) { 56 | navigator.serviceWorker 57 | .register(swUrl) 58 | .then(registration => { 59 | registration.onupdatefound = () => { 60 | const installingWorker = registration.installing; 61 | installingWorker.onstatechange = () => { 62 | if (installingWorker.state === 'installed') { 63 | if (navigator.serviceWorker.controller) { 64 | // At this point, the old content will have been purged and 65 | // the fresh content will have been added to the cache. 66 | // It's the perfect time to display a "New content is 67 | // available; please refresh." message in your web app. 68 | console.log('New content is available; please refresh.'); 69 | } else { 70 | // At this point, everything has been precached. 71 | // It's the perfect time to display a 72 | // "Content is cached for offline use." message. 73 | console.log('Content is cached for offline use.'); 74 | } 75 | } 76 | }; 77 | }; 78 | }) 79 | .catch(error => { 80 | console.error('Error during service worker registration:', error); 81 | }); 82 | } 83 | 84 | function checkValidServiceWorker(swUrl) { 85 | // Check if the service worker can be found. If it can't reload the page. 86 | fetch(swUrl) 87 | .then(response => { 88 | // Ensure service worker exists, and that we really are getting a JS file. 89 | if ( 90 | response.status === 404 || 91 | response.headers.get('content-type').indexOf('javascript') === -1 92 | ) { 93 | // No service worker found. Probably a different app. Reload the page. 94 | navigator.serviceWorker.ready.then(registration => { 95 | registration.unregister().then(() => { 96 | window.location.reload(); 97 | }); 98 | }); 99 | } else { 100 | // Service worker found. Proceed as normal. 101 | registerValidSW(swUrl); 102 | } 103 | }) 104 | .catch(() => { 105 | console.log( 106 | 'No internet connection found. App is running in offline mode.' 107 | ); 108 | }); 109 | } 110 | 111 | export function unregister() { 112 | if ('serviceWorker' in navigator) { 113 | navigator.serviceWorker.ready.then(registration => { 114 | registration.unregister(); 115 | }); 116 | } 117 | } 118 | --------------------------------------------------------------------------------