├── .gitignore ├── README.md ├── package.json ├── public └── index.html ├── src ├── AddNewCSVForm.js ├── App.js ├── App.test.js ├── Grid.js ├── LoadedTables.js ├── QueryForm.js ├── emitter.js └── index.js ├── test.csv └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | 6 | # testing 7 | /coverage 8 | 9 | # production 10 | /build 11 | 12 | # misc 13 | .DS_Store 14 | .env.local 15 | .env.development.local 16 | .env.test.local 17 | .env.production.local 18 | 19 | npm-debug.log* 20 | yarn-debug.log* 21 | yarn-error.log* 22 | *.csv -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CSV SQL Live! 2 | 3 | Run SQL queries on CSV files in your web browser. The future is now! 4 | 5 | http://dumbmatter.com/csv-sql-live/ 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "csv-sql-live", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "bootstrap": "^3.3.7", 7 | "papaparse": "^4.6.1", 8 | "react": "^16.6.0", 9 | "react-bootstrap": "^0.32.4", 10 | "react-data-grid": "^4.0.8", 11 | "react-data-grid-addons": "^4.0.8", 12 | "react-dom": "^16.6.0", 13 | "sql.js": "^0.5.0" 14 | }, 15 | "devDependencies": { 16 | "husky": "^1.1.3", 17 | "lint-staged": "^8.0.4", 18 | "prettier": "^1.14.3", 19 | "react-scripts": "^1.1.5" 20 | }, 21 | "lint-staged": { 22 | "src/**/*.{js,jsx,json,css}": [ 23 | "prettier --write", 24 | "git add" 25 | ] 26 | }, 27 | "scripts": { 28 | "precommit": "lint-staged", 29 | "start": "react-scripts start", 30 | "build": "react-scripts build", 31 | "test": "react-scripts test --env=jsdom", 32 | "eject": "react-scripts eject" 33 | }, 34 | "homepage": "http://dumbmatter.com/csv-sql-live" 35 | } 36 | -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | CSV SQL Live - Run SQL queries on CSV files right in your web browser 8 | 38 | 39 | 40 | 43 |
44 | 45 | 46 | 60 | 61 | -------------------------------------------------------------------------------- /src/AddNewCSVForm.js: -------------------------------------------------------------------------------- 1 | import Papa from "papaparse"; 2 | import React, { Component } from "react"; 3 | import Button from "react-bootstrap/lib/Button"; 4 | import ControlLabel from "react-bootstrap/lib/ControlLabel"; 5 | import FormControl from "react-bootstrap/lib/FormControl"; 6 | import FormGroup from "react-bootstrap/lib/FormGroup"; 7 | import Modal from "react-bootstrap/lib/Modal"; 8 | import sql from "sql.js"; 9 | import emitter from "./emitter"; 10 | 11 | const parse = file => { 12 | return new Promise((resolve, reject) => { 13 | Papa.parse(file, { 14 | complete: ({ data }) => { 15 | resolve(data); 16 | }, 17 | error: reject 18 | }); 19 | }); 20 | }; 21 | 22 | const createTable = (db, data, tableName) => { 23 | emitter.emit("updateState", { 24 | status: "creating-db" 25 | }); 26 | 27 | if (!db) { 28 | db = new sql.Database(); 29 | } 30 | 31 | const cols = data.shift(); 32 | const query = `CREATE TABLE "${tableName}" (${cols 33 | .map(col => `"${col}" TEXT`) 34 | .join(",")});`; 35 | db.run(query); 36 | 37 | try { 38 | const insertStmt = db.prepare( 39 | `INSERT INTO "${tableName}" VALUES (${cols.map(val => "?").join(",")})` 40 | ); 41 | for (const row of data) { 42 | if (row.length !== cols.length) { 43 | console.log("skipping row", row); 44 | continue; 45 | } 46 | 47 | insertStmt.run(row); 48 | } 49 | insertStmt.free(); 50 | } catch (err) { 51 | db.run(`DROP TABLE IF EXISTS "${tableName}"`); 52 | throw err; 53 | } 54 | 55 | return db; 56 | }; 57 | 58 | const getTableName = fileName => { 59 | const parts = fileName.split("."); 60 | if (parts.length < 2) { 61 | return fileName; 62 | } 63 | 64 | return parts.slice(0, -1).join("."); 65 | }; 66 | 67 | class AddNewCSVForm extends Component { 68 | constructor(props) { 69 | super(props); 70 | 71 | this.state = { 72 | errorMsg: undefined, 73 | file: undefined, 74 | fileName: undefined, 75 | tableName: "" 76 | }; 77 | } 78 | 79 | handleFile = e => { 80 | const file = e.target.files[0]; 81 | if (!file) { 82 | return; 83 | } 84 | 85 | // http://stackoverflow.com/q/4851595/786644 86 | const fileName = e.target.value.replace("C:\\fakepath\\", ""); 87 | this.setState({ 88 | file, 89 | fileName, 90 | tableName: getTableName(fileName) 91 | }); 92 | }; 93 | 94 | handleSubmit = async e => { 95 | const initialStatus = this.props.status; 96 | 97 | try { 98 | emitter.emit("updateState", { 99 | status: "parsing-data" 100 | }); 101 | 102 | if (!this.state.file) { 103 | return; 104 | } 105 | 106 | const data = await parse(this.state.file); 107 | const db = createTable(this.props.db, data, this.state.tableName); 108 | 109 | emitter.emit("updateState", { 110 | db, 111 | status: "loaded" 112 | }); 113 | emitter.emit("newTable", this.state.tableName); 114 | 115 | this.props.closeModal(); 116 | } catch (err) { 117 | console.error(err); 118 | this.setState({ 119 | errorMsg: err.message 120 | }); 121 | emitter.emit("updateState", { 122 | status: initialStatus 123 | }); 124 | } 125 | }; 126 | 127 | handleTableNameChange = async e => { 128 | this.setState({ tableName: e.target.value }); 129 | }; 130 | 131 | render() { 132 | return ( 133 |
134 | 135 | Add New CSV 136 | 137 | 138 |
139 |
140 |

141 | With CSV SQL Live you can{" "} 142 | 143 | run SQL queries on data from CSV files 144 | 145 | , right in your browser! 146 |

147 |
148 |
149 |

150 | 151 | Your data will not leave your computer. 152 | {" "} 153 | Processing is done in your browser. No servers involved. 154 |

155 |
156 |
157 | 158 |
159 | 160 | 161 | Select CSV File 162 | 167 | 168 | {this.state.fileName} 169 | 170 |
171 | 172 |
173 | 174 | Table Name 175 | 180 | 181 |
182 | 183 | {this.state.errorMsg !== undefined ? ( 184 |

185 | Error! {this.state.errorMsg} 186 |

187 | ) : null} 188 | 189 | 190 | 191 | 206 | 207 |
208 | ); 209 | } 210 | } 211 | 212 | export default AddNewCSVForm; 213 | -------------------------------------------------------------------------------- /src/App.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from "react"; 2 | import Button from "react-bootstrap/lib/Button"; 3 | import Modal from "react-bootstrap/lib/Modal"; 4 | import AddNewCSVForm from "./AddNewCSVForm"; 5 | import Grid from "./Grid"; 6 | import LoadedTables from "./LoadedTables"; 7 | import QueryForm from "./QueryForm"; 8 | import emitter from "./emitter"; 9 | import "bootstrap/dist/css/bootstrap.css"; 10 | 11 | // This mutates rows! 12 | const parseFloats = rows => { 13 | if (rows.length === 0) { 14 | return; 15 | } 16 | 17 | const allFloats = new Set(); 18 | for (let i = 0; i < rows[0].length; i++) { 19 | allFloats.add(i); 20 | } 21 | 22 | for (const row of rows) { 23 | for (let i = 0; i < row.length; i++) { 24 | if (row[i] !== parseFloat(row[i]).toString()) { 25 | allFloats.delete(i); 26 | } 27 | 28 | if (allFloats.size === 0) { 29 | return; 30 | } 31 | } 32 | } 33 | 34 | for (const row of rows) { 35 | for (const i of allFloats) { 36 | row[i] = parseFloat(row[i]); 37 | } 38 | } 39 | }; 40 | 41 | const initialState = { 42 | db: undefined, 43 | errorMsg: undefined, 44 | query: "", 45 | result: undefined, 46 | showAddModal: true, 47 | showTablesModal: false, 48 | status: "init" // init, parsing-data, creating-db, loaded, running-query, query-error 49 | }; 50 | 51 | class App extends Component { 52 | constructor(props) { 53 | super(props); 54 | 55 | this.state = initialState; 56 | } 57 | 58 | closeAddModal = () => { 59 | this.setState({ 60 | showAddModal: false 61 | }); 62 | }; 63 | 64 | closeTablesModal = () => { 65 | this.setState({ 66 | showTablesModal: false 67 | }); 68 | }; 69 | 70 | handleAddClick = () => { 71 | this.setState({ 72 | showAddModal: true 73 | }); 74 | }; 75 | 76 | handleTablesClick = () => { 77 | this.setState({ 78 | showTablesModal: true 79 | }); 80 | }; 81 | 82 | runQuery = query => { 83 | this.setState({ 84 | status: "running-query" 85 | }); 86 | 87 | try { 88 | const res = this.state.db.exec(query); 89 | const result = 90 | res.length === 0 91 | ? { 92 | cols: [], 93 | rows: [] 94 | } 95 | : { 96 | cols: res[res.length - 1].columns, 97 | rows: res[res.length - 1].values 98 | }; 99 | 100 | parseFloats(result.rows); 101 | 102 | this.setState({ 103 | result, 104 | status: "loaded" 105 | }); 106 | } catch (err) { 107 | this.setState({ 108 | errorMsg: err.message, 109 | status: "query-error" 110 | }); 111 | } 112 | }; 113 | 114 | updateState = state => { 115 | this.setState(state); 116 | }; 117 | 118 | componentDidMount() { 119 | emitter.addListener("runQuery", this.runQuery); 120 | emitter.addListener("updateState", this.updateState); 121 | } 122 | 123 | componentWillUnmount() { 124 | emitter.removeListener("runQuery", this.runQuery); 125 | emitter.removeListener("updateState", this.updateState); 126 | } 127 | 128 | render() { 129 | return ( 130 |
131 | 152 | 153 |
154 | 155 | {this.state.status === "init" ? ( 156 |

163 | To get started, first you need to{" "} 164 | load a CSV file. 165 |

166 | ) : null} 167 | {["query-error"].includes(this.state.status) ? ( 168 |

169 | Error! {this.state.errorMsg} 170 |

171 | ) : null} 172 | {this.state.result !== undefined ? ( 173 | 174 | ) : null} 175 |
176 | 177 |
178 |
179 | 180 | 224 |
225 | 226 | 227 | 232 | 233 | 234 | 239 | 240 | 241 |
242 | ); 243 | } 244 | } 245 | 246 | export default App; 247 | -------------------------------------------------------------------------------- /src/App.test.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import App from "./App"; 4 | 5 | it("renders without crashing", () => { 6 | const div = document.createElement("div"); 7 | ReactDOM.render(, div); 8 | }); 9 | -------------------------------------------------------------------------------- /src/Grid.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from "react"; 2 | import ReactDataGrid from "react-data-grid"; 3 | import { Toolbar } from "react-data-grid-addons"; 4 | 5 | class Grid extends Component { 6 | constructor(props) { 7 | super(props); 8 | 9 | this.state = { 10 | filters: {}, 11 | sortColumn: undefined, 12 | sortDirection: undefined, 13 | originalRows: [], 14 | filteredRows: [], 15 | rows: [] 16 | }; 17 | 18 | this.gridRef = React.createRef(); 19 | } 20 | 21 | static getDerivedStateFromProps(props, state) { 22 | if (props.rows !== state.originalRows) { 23 | return { 24 | filters: {}, 25 | sortColumn: undefined, 26 | sortDirection: undefined, 27 | 28 | // originalRows - straight from props, so retains all rows in original order 29 | // filteredRows - filtered version of originalRows, so in original order 30 | // rows - after filtering and sorting is applied 31 | originalRows: props.rows, 32 | filteredRows: props.rows.slice(), 33 | rows: props.rows.slice() 34 | }; 35 | } 36 | 37 | return null; 38 | } 39 | 40 | sortRows = (rows, sortColumn, sortDirection) => { 41 | const sortInd = this.props.cols.indexOf(sortColumn); 42 | const comparer = (a, b) => { 43 | if (sortDirection === "ASC") { 44 | return a[sortInd] > b[sortInd] ? 1 : -1; 45 | } else if (sortDirection === "DESC") { 46 | return a[sortInd] < b[sortInd] ? 1 : -1; 47 | } 48 | }; 49 | 50 | const sortedRows = 51 | sortDirection === "NONE" ? rows.slice() : rows.sort(comparer); 52 | return sortedRows; 53 | }; 54 | 55 | handleFilterChange = filter => { 56 | const newFilters = Object.assign({}, this.state.filters); 57 | if (filter.filterTerm) { 58 | newFilters[filter.column.key] = filter; 59 | } else { 60 | delete newFilters[filter.column.key]; 61 | } 62 | 63 | let filteredRows = this.state.originalRows.slice(); 64 | 65 | // Filter rows 66 | for (const col of Object.keys(newFilters)) { 67 | const i = this.props.cols.indexOf(col); 68 | if (i < 0) { 69 | continue; 70 | } 71 | 72 | const filterTerm = newFilters[col].filterTerm; 73 | filteredRows = filteredRows.filter(row => 74 | String(row[i]).includes(filterTerm) 75 | ); 76 | } 77 | 78 | // Also sort 79 | let rows; 80 | if ( 81 | this.state.sortColumn !== undefined && 82 | this.state.sortDirection !== undefined 83 | ) { 84 | rows = this.sortRows( 85 | filteredRows, 86 | this.state.sortColumn, 87 | this.state.sortDirection 88 | ); 89 | } else { 90 | rows = filteredRows.slice(); 91 | } 92 | 93 | this.setState({ filters: newFilters, filteredRows, rows }); 94 | }; 95 | 96 | handleGridSort = (sortColumn, sortDirection) => { 97 | const rows = this.sortRows( 98 | this.state.filteredRows.slice(), 99 | sortColumn, 100 | sortDirection 101 | ); 102 | this.setState({ rows, sortColumn, sortDirection }); 103 | }; 104 | 105 | rowGetter = i => { 106 | return this.state.rows[i].reduce((row, value, j) => { 107 | row[this.props.cols[j]] = value; 108 | return row; 109 | }, {}); 110 | }; 111 | 112 | componentDidMount() { 113 | // https://stackoverflow.com/a/45597682/786644 114 | if ( 115 | this.gridRef && 116 | this.gridRef.current && 117 | this.gridRef.current.onToggleFilter 118 | ) { 119 | this.gridRef.current.onToggleFilter(); 120 | } 121 | } 122 | 123 | render() { 124 | if (this.props.cols.length === 0 && this.state.rows.length === 0) { 125 | return

No rows returned.

; 126 | } 127 | 128 | return ( 129 | { 131 | return { 132 | key: col, 133 | name: col, 134 | filterable: true, 135 | sortable: true, 136 | resizable: true 137 | }; 138 | })} 139 | minHeight={window.innerHeight - 200} 140 | onAddFilter={this.handleFilterChange} 141 | onGridSort={this.handleGridSort} 142 | toolbar={} 143 | ref={this.gridRef} 144 | rowGetter={this.rowGetter} 145 | rowsCount={this.state.rows.length} 146 | width={100} 147 | /> 148 | ); 149 | } 150 | } 151 | 152 | export default Grid; 153 | -------------------------------------------------------------------------------- /src/LoadedTables.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from "react"; 2 | import Modal from "react-bootstrap/lib/Modal"; 3 | 4 | class LoadedTables extends Component { 5 | constructor(props) { 6 | super(props); 7 | 8 | this.state = { 9 | status: "closed", // closed, loading, loaded 10 | tables: undefined 11 | }; 12 | } 13 | 14 | static getDerivedStateFromProps(props) { 15 | if (!props.show) { 16 | return { 17 | status: "closed", 18 | tables: undefined 19 | }; 20 | } 21 | 22 | return null; 23 | } 24 | 25 | componentDidUpdate() { 26 | // If this update is to show the modal, update the tables list 27 | if (this.props.show && this.state.status === "closed") { 28 | this.setState({ 29 | status: "loading", 30 | tables: undefined 31 | }); 32 | 33 | if (this.props.db) { 34 | let tables = undefined; 35 | 36 | const res = this.props.db.exec( 37 | "SELECT name FROM sqlite_master WHERE type='table'" 38 | ); 39 | const tableNames = 40 | res.length === 0 41 | ? [] 42 | : res[res.length - 1].values.map(cols => cols[0]); 43 | 44 | if (tableNames.length > 0) { 45 | tables = {}; 46 | 47 | for (const tableName of tableNames) { 48 | const res2 = this.props.db.exec( 49 | `PRAGMA table_info("${tableName}");` 50 | ); 51 | const colNames = 52 | res2.length === 0 53 | ? [] 54 | : res2[res2.length - 1].values.map(cols => cols[1]); 55 | 56 | tables[tableName] = colNames; 57 | } 58 | } 59 | 60 | this.setState({ 61 | status: "loaded", 62 | tables 63 | }); 64 | } else { 65 | this.setState({ 66 | status: "loaded", 67 | tables: undefined 68 | }); 69 | } 70 | } 71 | } 72 | 73 | render() { 74 | return ( 75 |
76 | 77 | Loaded Tables 78 | 79 | 80 | {this.state.tables === undefined ? ( 81 | "No tables loaded." 82 | ) : ( 83 |
84 | {Object.keys(this.state.tables) 85 | .sort() 86 | .map(tableName => ( 87 |
88 |
    89 |
  • 90 | {tableName} 91 |
    • 92 | {this.state.tables[tableName].map(colName => ( 93 |
  • 94 | {colName} 95 |
    • 96 | ))} 97 |
98 |
99 | ))} 100 |
101 | )} 102 | 103 |
104 | ); 105 | } 106 | } 107 | 108 | export default LoadedTables; 109 | -------------------------------------------------------------------------------- /src/QueryForm.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from "react"; 2 | import Button from "react-bootstrap/lib/Button"; 3 | import FormControl from "react-bootstrap/lib/FormControl"; 4 | import FormGroup from "react-bootstrap/lib/FormGroup"; 5 | import HelpBlock from "react-bootstrap/lib/HelpBlock"; 6 | import emitter from "./emitter"; 7 | 8 | class QueryForm extends Component { 9 | constructor(props) { 10 | super(props); 11 | this.state = { queryText: "" }; 12 | } 13 | 14 | handleChange = e => { 15 | this.setState({ queryText: e.target.value }); 16 | }; 17 | 18 | handleSubmit = e => { 19 | e.preventDefault(); 20 | emitter.emit("runQuery", this.state.queryText); 21 | }; 22 | 23 | newTable = tableName => { 24 | if (this.state.queryText === "") { 25 | this.setState({ 26 | queryText: `SELECT * FROM "${tableName}"` 27 | }); 28 | } 29 | }; 30 | 31 | componentDidMount() { 32 | emitter.addListener("newTable", this.newTable); 33 | document.addEventListener("keydown", this.handleKeyDown); 34 | } 35 | 36 | componentWillUnmount() { 37 | emitter.removeListener("newTable", this.newTable); 38 | document.removeEventListener("keydown", this.handleKeyDown); 39 | } 40 | 41 | handleKeyDown = event => { 42 | if (event.ctrlKey && event.key === "Enter") { 43 | this.handleSubmit(event); 44 | } 45 | }; 46 | 47 | render() { 48 | if (this.props.status === "init") { 49 | return null; 50 | } 51 | 52 | return ( 53 |
54 | 55 | 64 | 76 | 77 | Any{" "} 78 | 83 | valid SQLite query 84 | {" "} 85 | is supported. 86 | 87 | 88 |
89 | ); 90 | } 91 | } 92 | 93 | export default QueryForm; 94 | -------------------------------------------------------------------------------- /src/emitter.js: -------------------------------------------------------------------------------- 1 | import EventEmitter from "events"; 2 | 3 | const emitter = new EventEmitter(); 4 | 5 | export default emitter; 6 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import App from "./App"; 4 | 5 | ReactDOM.render(, document.getElementById("root")); 6 | -------------------------------------------------------------------------------- /test.csv: -------------------------------------------------------------------------------- 1 | number',-quotedstring,ip 2 | 1,"12"" vinyl",192.168.0.123 3 | 3,"42 is answer",10.42.42.42 4 | --------------------------------------------------------------------------------