├── dev └── api │ ├── logs │ ├── querylog.csv │ └── responselog.csv │ ├── orm_resources │ ├── __init__.py │ ├── divergence_languages.json │ ├── emojis.bin │ ├── ngrams.bin │ ├── storywrangler.png │ ├── client.json │ ├── realtime_languages.json │ ├── page_defaults.json │ ├── page_option_types.json │ ├── page_options.json │ ├── popular_language_codes.csv │ ├── indexed_languages.json │ ├── language_support.json │ ├── iso_codes.csv │ ├── ngrams_languages.json │ └── supported_languages.json │ ├── __pycache__ │ ├── api.cpython-37.pyc │ └── regexr.cpython-37.pyc │ ├── uwsgi.py │ ├── init_logs.py │ └── regexr.py ├── prod └── api │ ├── orm_resources │ ├── __init__.py │ ├── divergence_languages.json │ ├── emojis.bin │ ├── ngrams.bin │ ├── storywrangler.png │ ├── client.json │ ├── realtime_languages.json │ ├── page_defaults.json │ ├── page_option_types.json │ ├── page_options.json │ ├── popular_language_codes.csv │ ├── language_support.json │ ├── indexed_languages.json │ ├── iso_codes.csv │ ├── ngrams_languages.json │ └── supported_languages.json │ ├── logs │ ├── responselog.csv │ └── querylog.csv │ ├── __pycache__ │ ├── api.cpython-37.pyc │ ├── orm.cpython-37.pyc │ └── regexr.cpython-37.pyc │ ├── uwsgi.py │ ├── init_logs.py │ └── regexr.py ├── story-viewer ├── public │ ├── robots.txt │ ├── favicon.ico │ └── index.html ├── src │ ├── img │ │ ├── fb-share-img.png │ │ ├── pattern_match.png │ │ ├── teasers │ │ │ ├── rtd-tease.png │ │ │ ├── ngram-tease.png │ │ │ ├── zipf-tease.png │ │ │ ├── realtime-tease.png │ │ │ └── languages-tease.png │ │ ├── twitter-share-img.png │ │ ├── footer │ │ │ ├── csc_roboctopus.ai │ │ │ ├── sm_roboctopus.png │ │ │ ├── tiny-roboctopus.jpg │ │ │ ├── mm_white.svg │ │ │ ├── mm.svg │ │ │ └── uvm-cems.svg │ │ ├── social_twitter.svg │ │ └── storywrangler_icon.svg │ ├── setupTests.js │ ├── index.js │ ├── components │ │ ├── index.js │ │ ├── LoadingIndicator.jsx │ │ ├── TopNgrams.jsx │ │ ├── Contact.jsx │ │ ├── Footer.jsx │ │ ├── Home.jsx │ │ ├── Navigation.jsx │ │ └── View.jsx │ ├── options │ │ ├── calendar.jsx │ │ ├── toggle.jsx │ │ ├── dropdown.jsx │ │ └── search.jsx │ ├── App.js │ ├── visualizations │ │ ├── Barchart.jsx │ │ ├── Subplots.jsx │ │ ├── Timeline.jsx │ │ ├── Subplot.jsx │ │ ├── timelineutils.js │ │ └── Table.jsx │ ├── serviceWorker.js │ ├── index.css │ ├── utils.js │ └── defaults.js ├── package.json └── README.md ├── .gitignore ├── LICENSE.txt └── README.md /dev/api/logs/querylog.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dev/api/logs/responselog.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dev/api/orm_resources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /prod/api/orm_resources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /prod/api/logs/responselog.csv: -------------------------------------------------------------------------------- 1 | pid,time,errors 2 | -------------------------------------------------------------------------------- /dev/api/orm_resources/divergence_languages.json: -------------------------------------------------------------------------------- 1 | { 2 | "en": "English" 3 | } 4 | -------------------------------------------------------------------------------- /prod/api/logs/querylog.csv: -------------------------------------------------------------------------------- 1 | pid,query,src,wordcount,lang,metrics,ip,sent,date 2 | -------------------------------------------------------------------------------- /prod/api/orm_resources/divergence_languages.json: -------------------------------------------------------------------------------- 1 | { 2 | "en": "English" 3 | } 4 | -------------------------------------------------------------------------------- /story-viewer/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /dev/api/orm_resources/emojis.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/dev/api/orm_resources/emojis.bin -------------------------------------------------------------------------------- /dev/api/orm_resources/ngrams.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/dev/api/orm_resources/ngrams.bin -------------------------------------------------------------------------------- /story-viewer/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/public/favicon.ico -------------------------------------------------------------------------------- /prod/api/orm_resources/emojis.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/prod/api/orm_resources/emojis.bin -------------------------------------------------------------------------------- /prod/api/orm_resources/ngrams.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/prod/api/orm_resources/ngrams.bin -------------------------------------------------------------------------------- /story-viewer/src/img/fb-share-img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/fb-share-img.png -------------------------------------------------------------------------------- /dev/api/__pycache__/api.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/dev/api/__pycache__/api.cpython-37.pyc -------------------------------------------------------------------------------- /dev/api/orm_resources/storywrangler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/dev/api/orm_resources/storywrangler.png -------------------------------------------------------------------------------- /prod/api/__pycache__/api.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/prod/api/__pycache__/api.cpython-37.pyc -------------------------------------------------------------------------------- /prod/api/__pycache__/orm.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/prod/api/__pycache__/orm.cpython-37.pyc -------------------------------------------------------------------------------- /story-viewer/src/img/pattern_match.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/pattern_match.png -------------------------------------------------------------------------------- /dev/api/__pycache__/regexr.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/dev/api/__pycache__/regexr.cpython-37.pyc -------------------------------------------------------------------------------- /prod/api/__pycache__/regexr.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/prod/api/__pycache__/regexr.cpython-37.pyc -------------------------------------------------------------------------------- /prod/api/orm_resources/storywrangler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/prod/api/orm_resources/storywrangler.png -------------------------------------------------------------------------------- /story-viewer/src/img/teasers/rtd-tease.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/teasers/rtd-tease.png -------------------------------------------------------------------------------- /story-viewer/src/img/twitter-share-img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/twitter-share-img.png -------------------------------------------------------------------------------- /story-viewer/src/img/teasers/ngram-tease.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/teasers/ngram-tease.png -------------------------------------------------------------------------------- /story-viewer/src/img/teasers/zipf-tease.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/teasers/zipf-tease.png -------------------------------------------------------------------------------- /story-viewer/src/img/footer/csc_roboctopus.ai: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/footer/csc_roboctopus.ai -------------------------------------------------------------------------------- /story-viewer/src/img/footer/sm_roboctopus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/footer/sm_roboctopus.png -------------------------------------------------------------------------------- /story-viewer/src/img/footer/tiny-roboctopus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/footer/tiny-roboctopus.jpg -------------------------------------------------------------------------------- /story-viewer/src/img/teasers/realtime-tease.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/teasers/realtime-tease.png -------------------------------------------------------------------------------- /story-viewer/src/img/teasers/languages-tease.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janeadams/storywrangler/HEAD/story-viewer/src/img/teasers/languages-tease.png -------------------------------------------------------------------------------- /dev/api/orm_resources/client.json: -------------------------------------------------------------------------------- 1 | { 2 | "database": "mongodb", 3 | "domain": "hydra.uvm.edu", 4 | "username": "guest", 5 | "pwd": "roboctopus", 6 | "port": "27017" 7 | } -------------------------------------------------------------------------------- /dev/api/orm_resources/realtime_languages.json: -------------------------------------------------------------------------------- 1 | { 2 | "en": "English", 3 | "es": "Spanish", 4 | "pt": "Portuguese", 5 | "ar": "Arabic", 6 | "ko": "Korean" 7 | } 8 | -------------------------------------------------------------------------------- /prod/api/orm_resources/client.json: -------------------------------------------------------------------------------- 1 | { 2 | "database": "mongodb", 3 | "domain": "hydra.uvm.edu", 4 | "username": "guest", 5 | "pwd": "roboctopus", 6 | "port": "27017" 7 | } -------------------------------------------------------------------------------- /prod/api/orm_resources/realtime_languages.json: -------------------------------------------------------------------------------- 1 | { 2 | "en": "English", 3 | "es": "Spanish", 4 | "pt": "Portuguese", 5 | "ar": "Arabic", 6 | "ko": "Korean" 7 | } 8 | -------------------------------------------------------------------------------- /dev/api/uwsgi.py: -------------------------------------------------------------------------------- 1 | from dev.api.orm import app as application 2 | if __name__ == "__main__": 3 | application.run() 4 | 5 | """ 6 | def application(env,resp): 7 | resp('200 OK',[('Content-Type','text/html')]) 8 | return [b"Hello World!"] 9 | """ 10 | -------------------------------------------------------------------------------- /prod/api/uwsgi.py: -------------------------------------------------------------------------------- 1 | from prod.api.orm import app as application 2 | if __name__ == "__main__": 3 | application.run() 4 | 5 | """ 6 | def application(env,resp): 7 | resp('200 OK',[('Content-Type','text/html')]) 8 | return [b"Hello World!"] 9 | """ 10 | -------------------------------------------------------------------------------- /story-viewer/src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom/extend-expect'; 6 | -------------------------------------------------------------------------------- /story-viewer/src/index.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import "./index.css"; 4 | import App from "./App"; 5 | import * as serviceWorker from "./serviceWorker"; 6 | 7 | ReactDOM.render(, document.getElementById("root")); 8 | 9 | serviceWorker.unregister(); 10 | -------------------------------------------------------------------------------- /story-viewer/src/components/index.js: -------------------------------------------------------------------------------- 1 | export { default as Navigation } from "./Navigation"; 2 | export { default as Footer } from "./Footer"; 3 | export { default as Home } from "./Home"; 4 | export { default as View } from "./View"; 5 | export { default as About } from "./About"; 6 | export { default as Contact } from "./Contact"; 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Don't track content of these folders 2 | node_modules/ 3 | debug/prod-*.txt 4 | .* 5 | story-viewer/build 6 | ai/ 7 | notes/ 8 | static/ 9 | package-lock.json 10 | archive/ 11 | notes/ 12 | *.sh 13 | dev/api/*log.csv 14 | prod/api/*log.csv 15 | *.log 16 | *.ipynb 17 | debug/ 18 | __pycache__/* 19 | *.pyc 20 | */api/logs/* 21 | prod/api/logs/querylog.csv 22 | -------------------------------------------------------------------------------- /story-viewer/src/options/calendar.jsx: -------------------------------------------------------------------------------- 1 | import DatePicker from "react-datepicker"; 2 | import {parseDate, formatDate, mostrecent, firstDate} from "../utils" 3 | import "react-datepicker/dist/react-datepicker.css"; 4 | 5 | const Calendar = (props) => { 6 | return ( 7 | props.setDate(formatDate(date))} 10 | maxDate={mostrecent} 11 | minDate={firstDate} 12 | /> 13 | ); 14 | }; 15 | 16 | export default Calendar -------------------------------------------------------------------------------- /story-viewer/src/options/toggle.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | 3 | const Toggle = (props) => { 4 | return ( 5 |
6 | 9 | props.setState(e.target.checked)} 15 | /> 16 |
17 | ) 18 | } 19 | 20 | export default Toggle -------------------------------------------------------------------------------- /dev/api/init_logs.py: -------------------------------------------------------------------------------- 1 | import csv 2 | with open('prod/api/logs/querylog.csv', 'w') as csvfile: 3 | filewriter = csv.writer(csvfile, delimiter=',', 4 | quotechar='|', quoting=csv.QUOTE_MINIMAL) 5 | columns=['pid','query','src','wordcount','lang','metrics','ip','sent','date'] 6 | filewriter.writerow(columns) 7 | csvfile.close() 8 | with open('prod/api/logs/responselog.csv', 'w') as csvfile: 9 | filewriter = csv.writer(csvfile, delimiter=',', 10 | quotechar='|', quoting=csv.QUOTE_MINIMAL) 11 | columns=['pid','time','errors'] 12 | filewriter.writerow(columns) 13 | csvfile.close() 14 | -------------------------------------------------------------------------------- /prod/api/init_logs.py: -------------------------------------------------------------------------------- 1 | import csv 2 | with open('prod/api/logs/querylog.csv', 'w') as csvfile: 3 | filewriter = csv.writer(csvfile, delimiter=',', 4 | quotechar='|', quoting=csv.QUOTE_MINIMAL) 5 | columns=['pid','query','src','wordcount','lang','metrics','ip','sent','date'] 6 | filewriter.writerow(columns) 7 | csvfile.close() 8 | with open('prod/api/logs/responselog.csv', 'w') as csvfile: 9 | filewriter = csv.writer(csvfile, delimiter=',', 10 | quotechar='|', quoting=csv.QUOTE_MINIMAL) 11 | columns=['pid','time','errors'] 12 | filewriter.writerow(columns) 13 | csvfile.close() 14 | -------------------------------------------------------------------------------- /story-viewer/src/components/LoadingIndicator.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { usePromiseTracker } from "react-promise-tracker" 3 | import Loader from 'react-loader-spinner' 4 | 5 | const LoadingIndicator = () => { 6 | const { promiseInProgress } = usePromiseTracker() 7 | return ( 8 | promiseInProgress &&
16 |
17 | ) 18 | } 19 | 20 | export default LoadingIndicator; -------------------------------------------------------------------------------- /story-viewer/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 16 | StoryWrangler - Twitter Ngram Search 17 | 18 | 19 | 20 |
21 | 22 | 23 | -------------------------------------------------------------------------------- /story-viewer/src/components/TopNgrams.jsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import {trackPromise} from "react-promise-tracker"; 3 | 4 | React.useEffect(async () => { 5 | const endpoint = `http://hydra.uvm.edu:3000/api/${viewer}/` 6 | let apicall = endpoint+query 7 | if (APIparams){ 8 | let formattedAPIparams = [] 9 | for (const [key, value] of Object.entries(APIparams)) { 10 | formattedAPIparams.push(key+"="+value) 11 | } 12 | apicall = apicall.concat('?'+formattedAPIparams.join('&')) 13 | } 14 | console.log('Formatted API call as:') 15 | console.log(apicall) 16 | const response = await trackPromise(fetch(apicall)); 17 | const json = await trackPromise(response.json()); 18 | if (json) { setData(json.data) } 19 | }, [query, APIparams]); -------------------------------------------------------------------------------- /story-viewer/src/options/dropdown.jsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import Select from 'react-select'; 3 | 4 | const Dropdown = (props) => { 5 | 6 | let dropdownOptions = [] 7 | Object.entries(props.options).forEach(([key, value]) => 8 | dropdownOptions.push({'value': key, 'label': value}) 9 | ) 10 | 11 | return ( 12 |
13 |

{props.prompt}

14 |