├── .babelrc ├── .gitignore ├── .htmlnanorc ├── .screenshot.png ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── assets └── favicon.png ├── docs ├── clear-pipes.b3d7541c.js ├── clear-pipes.b3d7541c.js.map ├── index.html ├── main.84e2604c.css └── main.84e2604c.css.map ├── index.html ├── index.tsx ├── main.css ├── package-lock.json ├── package.json ├── src ├── physics.ts ├── process.ts ├── render.tsx ├── state.ts └── utils.ts ├── tsconfig.json └── yarn.lock /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/preset-env", 4 | "@babel/preset-typescript", 5 | "@babel/preset-react" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | -------------------------------------------------------------------------------- /.htmlnanorc: -------------------------------------------------------------------------------- 1 | { 2 | "minifySvg": false 3 | } -------------------------------------------------------------------------------- /.screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MaxBittker/clear-pipes/ad287915fe0940652e2f500cfbfa03c2de5f6ee4/.screenshot.png -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.enabledLanguageIds": [ 3 | "asciidoc", 4 | "c", 5 | "cpp", 6 | "csharp", 7 | "css", 8 | "git-commit", 9 | "go", 10 | "handlebars", 11 | "haskell", 12 | "html", 13 | "jade", 14 | "java", 15 | "javascript", 16 | "javascriptreact", 17 | "json", 18 | "jsonc", 19 | "latex", 20 | "less", 21 | "markdown", 22 | "php", 23 | "plaintext", 24 | "pug", 25 | "restructuredtext", 26 | "rust", 27 | "scala", 28 | "scss", 29 | "text", 30 | "typescriptreact", 31 | "yaml", 32 | "yml" 33 | ] 34 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Max Bittker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # clear-pipes 2 | visualize data flow 3 | 4 |  5 | -------------------------------------------------------------------------------- /assets/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MaxBittker/clear-pipes/ad287915fe0940652e2f500cfbfa03c2de5f6ee4/assets/favicon.png -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 |
speed:
-------------------------------------------------------------------------------- /docs/main.84e2604c.css: -------------------------------------------------------------------------------- 1 | body{font-family:Libre Franklin,sans-serif;background-size:40px 40px;background-image:radial-gradient(circle,#999 1px,transparent 0)}*{color:#333;margin:0}text{fill:#000}.pipetext{font-size:20px}svg{width:100%;height:100%}.tube{stroke-width:30px;stroke:rgba(0,0,200,.05)}.trash{stroke:rgba(150,0,30,.1)}textPath{letter-spacing:.1em}h1{font-size:25px}#speedbox{position:fixed;left:5;z-index:500;bottom:5;display:flex}#article-link{font-size:15px;font-weight:100}#article-link,#info{font-family:Libre Franklin}#info{font-size:25px}.credit{position:fixed;right:2px;bottom:0;z-index:500;font-weight:100;text-shadow:#fff 0 1px}:root{--spin-duration:2000ms}.pinwheel{animation-name:spin;animation-duration:var(--spin-duration);animation-iteration-count:infinite;animation-timing-function:linear;fill:#fff;stroke:#000}@keyframes spin{0%{transform:rotate(0deg)}to{transform:rotate(1turn)}}foreignObject>div{margin-bottom:10px;height:100%}.boxText{padding:5px}.infoBox{font-size:18px}.infoBox p{margin-bottom:9px}.word-bank{padding:10px 0 10px 10px;margin-right:10px;font-size:18px;margin-top:1px;height:100%}.word-span{border-radius:3px;padding:0 2px;animation:fadeInAnimation 1s ease;animation-iteration-count:1;animation-fill-mode:forwards}@keyframes fadeInAnimation{0%{opacity:0}to{opacity:1}}#scratch{visibility:hidden} 2 | /*# sourceMappingURL=/clear-pipes/main.84e2604c.css.map */ -------------------------------------------------------------------------------- /docs/main.84e2604c.css.map: -------------------------------------------------------------------------------- 1 | {"version":3,"sources":["main.css"],"names":[],"mappings":"AAAA,KACE,qCAAyC,CACzC,yBAA0B,CAC1B,+DACF,CACA,EACE,UAAW,CACX,QAEF,CAEA,KACE,SAIF,CACA,UACE,cACF,CACA,IACE,UAAW,CACX,WACF,CAKA,MACE,iBAAkB,CAClB,wBAEF,CACA,OACE,wBACF,CACA,SAEE,mBACF,CACA,GACE,cAEF,CAIA,UACE,cAAe,CACf,MAAO,CACP,WAAY,CACZ,QAAS,CACT,YACF,CACA,cACE,cAAe,CACf,eAEF,CACA,oBAFE,0BAOF,CALA,MAEE,cAGF,CACA,QACE,cAAe,CACf,SAAU,CACV,QAAS,CACT,WAAY,CACZ,eAAgB,CAEhB,sBACF,CACA,MACE,sBACF,CACA,UACE,mBAAoB,CACpB,uCAAwC,CACxC,kCAAmC,CACnC,gCAAiC,CACjC,SAAW,CACX,WAEF,CAEA,gBACE,GACE,sBACF,CACA,GACE,uBACF,CACF,CACA,kBACE,kBAAmB,CACnB,WAGF,CACA,SACE,WACF,CACA,SACE,cACF,CACA,WACE,iBACF,CACA,WAEE,wBAA2B,CAC3B,iBAAkB,CAClB,cAAe,CAEf,cAAe,CAMf,WAEF,CACA,WAIE,iBAAkB,CAClB,aAAgB,CAChB,iCAAkC,CAClC,2BAA4B,CAC5B,4BACF,CACA,2BACE,GACE,SACF,CACA,GACE,SACF,CACF,CACA,SACE,iBACF","file":"main.84e2604c.css","sourceRoot":"..","sourcesContent":["body {\n font-family: \"Libre Franklin\", sans-serif;\n background-size: 40px 40px;\n background-image: radial-gradient(circle, #999 1px, rgba(0, 0, 0, 0) 1px);\n}\n* {\n color: #333;\n margin: 0;\n /* border: 1px red solid; */\n}\n\ntext {\n fill: black;\n /* stroke: rgb(195, 195, 240); */\n\n /* stroke-width: 1px; */\n}\n.pipetext {\n font-size: 20px;\n}\nsvg {\n width: 100%;\n height: 100%;\n}\npath {\n /* stroke-width: 40px; */\n /* stroke:red; */\n}\n.tube {\n stroke-width: 30px;\n stroke: rgba(0, 0, 200, 0.05);\n /* stroke-dasharray: 20px 10px; */\n}\n.trash {\n stroke: rgba(150, 0, 30, 0.1);\n}\ntextPath {\n /* font-size: 0.4em; */\n letter-spacing: 0.1em;\n}\nh1 {\n font-size: 25px;\n /* text-align: center; */\n}\n\n@media only screen and (max-width: 700px) {\n}\n#speedbox {\n position: fixed;\n left: 5;\n z-index: 500;\n bottom: 5;\n display: flex;\n}\n#article-link {\n font-size: 15px;\n font-weight: 100;\n font-family: \"Libre Franklin\";\n}\n#info {\n font-family: \"Libre Franklin\";\n font-size: 25px;\n\n /* font-style: italic; */\n}\n.credit {\n position: fixed;\n right: 2px;\n bottom: 0;\n z-index: 500;\n font-weight: 100;\n\n text-shadow: white 0px 1px;\n}\n:root {\n --spin-duration: 2000ms;\n}\n.pinwheel {\n animation-name: spin;\n animation-duration: var(--spin-duration);\n animation-iteration-count: infinite;\n animation-timing-function: linear;\n fill: white;\n stroke: black;\n /* stroke-linecap: round; */\n}\n\n@keyframes spin {\n from {\n transform: rotate(0deg);\n }\n to {\n transform: rotate(360deg);\n }\n}\nforeignObject > div {\n margin-bottom: 10px;\n height: 100%;\n /* padding: 10px; */\n /* margin: 10px; */\n}\n.boxText {\n padding: 5px;\n}\n.infoBox {\n font-size: 18px;\n}\n.infoBox p {\n margin-bottom: 9px;\n}\n.word-bank {\n /* text-align: justify; */\n padding: 10px 0px 10px 10px;\n margin-right: 10px;\n font-size: 18px;\n /* word-wrap: no-wrap; */\n margin-top: 1px;\n\n /* display: flex; */\n /* justify-content: space-between; */\n /* align-items: center; */\n /* flex-wrap: wrap; */\n height: 100%;\n /* overflow-y: scroll; */\n}\n.word-span {\n /* position: absolute; */\n /* background-color: rgba(200, 0, 200, 0.05); */\n /* border: 1px solid rgba(200, 0, 200, 0.5); */\n border-radius: 3px;\n padding: 0px 2px;\n animation: fadeInAnimation ease 1s;\n animation-iteration-count: 1;\n animation-fill-mode: forwards;\n}\n@keyframes fadeInAnimation {\n 0% {\n opacity: 0;\n }\n 100% {\n opacity: 1;\n }\n}\n#scratch {\n visibility: hidden;\n}\n"]} -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |speed:
40 | 41 | 49 | 50 | 51 | 52 | 55 | 56 | -------------------------------------------------------------------------------- /index.tsx: -------------------------------------------------------------------------------- 1 | import Two from "two.js"; 2 | import * as React from "react"; 3 | import * as ReactDOMServer from "react-dom/server"; 4 | import { 5 | makeConnector, 6 | makeBox, 7 | makeInfoBox, 8 | makeGradient, 9 | makeHopper, 10 | } from "./src/render"; 11 | import { startPhysics } from "./src/physics"; 12 | import { processTDV } from "./src/process"; 13 | import { Vector } from "matter-js"; 14 | import { subtract } from "./src/utils"; 15 | 16 | let elem = document.getElementById("draw-animation"); 17 | let width = window.innerWidth; 18 | let height = window.innerHeight; 19 | window.width = width; 20 | window.height = height; 21 | 22 | let two = new Two({ fullscreen: true, autostart: true }).appendTo(elem); 23 | two.renderer.domElement.setAttribute("viewBox", "-0 -0 1000 1050"); 24 | 25 | window.two = two; 26 | 27 | let pHopper = new Two.Vector(200, 220); 28 | let pClean = new Two.Vector(200, 30 + 750); 29 | let pRule = new Two.Vector(500, 30 + 750); 30 | // let pCache = new Two.Vector(600, 750); 31 | let pCheck = new Two.Vector(800, 30 + 750); 32 | let pDestination = new Two.Vector(950, 350); 33 | let pTrash = new Two.Vector(0, 30 + 870); 34 | let pTrash2 = new Two.Vector(0, 30 + 905); 35 | let pTrash3 = new Two.Vector(0, 30 + 940); 36 | // let pTrash4 = new Two.Vector(0, 975); 37 | let pGradient = new Two.Vector(0, 30 + 905); 38 | 39 | let c1 = makeConnector( 40 | new Two.Vector(200, 420), 41 | pClean, 42 | "1", 43 | true, 44 | "wiggle", 45 | true 46 | ); 47 | let c2 = makeConnector(pClean, pRule, "2", false, "l"); 48 | // let c3 = makeConnector(pRule, pCache, "3", false, "l"); 49 | let c3 = makeConnector(pRule, pCheck, "3", false, "loop"); 50 | let c4 = makeConnector(pCheck, pDestination, "4"); 51 | 52 | let cTrash1 = makeConnector(pClean, pTrash, "t1", true); 53 | let cTrash2 = makeConnector(pRule, pTrash2, "t2", true); 54 | // let cTrash3 = makeConnector(pCache, pTrash3, "t3", true); 55 | let cTrash3 = makeConnector(pCheck, pTrash3, "t4", true); 56 | let trashGradient = makeGradient(pGradient.x, pGradient.y, 200); 57 | // let c7 = makeConnector(pCheck, pTrash, "5", true); 58 | 59 | let boxHopper = makeHopper(pHopper, 350, "a"); 60 | let boxClean = makeBox(pClean, 100, "b"); 61 | boxClean.setText("Trim Punctuation"); 62 | let boxRule = makeBox(pRule, 100, "c"); 63 | boxRule.setText("Capitalized?"); 64 | 65 | // let boxCache = makeBox(pCache, 100, "d"); 66 | // boxCache.setText("Not Seen by us?"); 67 | 68 | let boxCheck = makeBox(pCheck, 100, "d"); 69 | boxCheck.setText("Exists in NYT Archives?"); 70 | 71 | let boxDestination = makeBox(new Two.Vector(900, 350), 150, "f"); 72 | 73 | function formatWords(words: Array${words.map((w) => `${w} `).join(" ")}
`; 75 | } 76 | 77 | let destinationWords: ArrayThis is a visualization of the process behind @nyt_first_said.
105 | 106 |Each day, a script scrapes new articles from nytimes.com. That text is tokenized, or split into words based on whitespace and punctuation.
107 | 108 |Each word then must pass several criteria. Containing a number or special character is criteria for disqualification. To avoid proper nouns, all capitalized words are filtered.
109 | 110 |
The most important check is against the New York Time's archive search service. The archive goes back to 1851 and contains more than 13 million articles.
111 | 112 |The paper publishes many thousands of words each day, but only a very few are firsts.
113 |
114 | more information
115 | `);
116 |
117 | let group = two.makeGroup(text1, text2, text3);
118 | let group2 = two.makeGroup();
119 | two.update();
120 | let svgElem = group2._renderer.elem;
121 | svgElem.innerHTML += `
122 | {id}
*/}{id}
*/}{id}
*/}