├── site ├── public │ ├── stdlib.js │ ├── docs │ │ ├── stdlib.js │ │ ├── index.js │ │ ├── files │ │ │ ├── 9964525a8aaf4f1e1a5a32685a8639c297ecdf0b7ac5f3093bc3c5709a560a2c │ │ │ ├── eb75aeb7fd114a50db65b3763b2b35fd676c64d5eeba5fc7cc5f9eb4a2e1c60d │ │ │ ├── e17d2fef09ef58173696facb5b9857b2669d954aeb0f1198011717529ee70b4c │ │ │ ├── b70d3e71a1a87aaa614de9d5c309db8acfe1ffc9613f7384dd5dfe9e7b9bb691 │ │ │ ├── 8ac975167401d4a3f1a20fdb606e475370e4add5871446615974ab7a728e13e5 │ │ │ ├── e8e935a22713e8ea10bbbf6e17dab8c2742708c2abf8e021c136c918459592a0 │ │ │ ├── 2e23a9f910d8e8fff0ee5f32a30b464dd0a27dcc08e73282cd7f8f68d9a79349 │ │ │ ├── 5fe1038695da56b5248ee289dc2cf66fc4918ca97dfd7649e886a1bd931df3df │ │ │ ├── 810bce42e9e40e466c2341a756f30bebf482f10c0cf230dce89893407391cf89 │ │ │ └── 7e2bc186f16000498c9227c6bd4055458bd74d7204e9dd8594d81d0ccf7587fd │ │ ├── index.html │ │ └── bc54548ebb381725e6d095ab3932e77a89d4cdfccb634ba18465f0358d5a8501.js │ ├── examples │ │ ├── census-api │ │ │ ├── stdlib.js │ │ │ ├── index.js │ │ │ ├── index.html │ │ │ ├── files │ │ │ │ └── e54361c3e7be0df0f0b536e79d37e5d0ce58eccdb461006df04312ed1a8feadd │ │ │ └── 645b0415e1030853bbc34781c1b23b6989a0bee79f978217f15eab1e73514c80.js │ │ ├── github-api │ │ │ ├── stdlib.js │ │ │ ├── index.js │ │ │ ├── index.html │ │ │ ├── files │ │ │ │ └── 22dbb963d472ebe6a9bc3201082b7e3e5a74ff3e40ab302ba9aecff68d25436f │ │ │ └── 0b7ca4bb60e127b218d27d2d59e6f708d77da8fa16da661b77f587a262c8391c.js │ │ └── wiki-pageviews │ │ │ ├── stdlib.js │ │ │ ├── index.js │ │ │ ├── index.html │ │ │ ├── files │ │ │ └── 157de8e31dca7ade4cdb014615698b244f9e4b5da7d9346ab255abde595d580e │ │ │ └── e9e128ada7176e897e2b52da0a123c594db2d80911b48bd4540baadfce3b3342.js │ ├── index.js │ ├── index.html │ └── 147b7a769e856472cafb081301cb1f171d92f08e70bf38979365bb601cd37251.js └── index.ojs ├── test ├── export │ ├── data │ │ ├── x │ │ ├── y │ │ └── sub │ │ │ ├── f1 │ │ │ └── f2 │ ├── a.ojs │ ├── b.ojs │ ├── sub_with_fa.ojs │ ├── ts-me.ojs │ └── top.ojs ├── stress │ └── stdlib.ojs ├── data │ ├── a.txt │ ├── b.txt │ ├── c.txt │ ├── x.txt │ ├── y.txt │ └── live.csv ├── imports │ ├── suba.ojs │ ├── subb.ojs │ ├── subd-op.ojs │ ├── subd.ojs │ └── top.ojs ├── builtins │ └── width.ojs ├── stdlib │ ├── colors1.js │ ├── colors2.js │ ├── funcs.js │ ├── funcs.ojs │ ├── async.ojs │ ├── colors.ojs │ └── async.js ├── file-attachments │ ├── sub.ojs │ ├── normal.ojs │ └── live.ojs ├── test-suite.ojs ├── secrets │ └── normal.ojs ├── errors │ └── parsing.ojs ├── standalone.ojs └── design.ojs ├── examples ├── local │ ├── filesize-update.sh │ ├── data │ │ └── filesize.txt │ ├── filesize-live.ojs │ ├── choropleth-live.ojs │ ├── github-api.ojs │ ├── wikipedia-pageviews.ojs │ └── census-api.ojs └── preact │ ├── index.html │ ├── a.ojs │ ├── package.json │ ├── App.jsx │ ├── build.js │ └── yarn.lock ├── CHANGELOG.md ├── src ├── utils.js ├── client │ ├── core.js │ ├── index.js │ └── run.js ├── dataflow ├── content │ └── index.html ├── compile.js └── run.js ├── docs ├── secrets.md ├── importing.md ├── file-attachments.md ├── reference.md ├── quickstart.md ├── production.md ├── compiling.md ├── stdlib.md ├── shebang.md └── site.ojs ├── LICENSE ├── TESTING.md ├── .gitignore ├── package.json └── README.md /site/public/stdlib.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/export/data/x: -------------------------------------------------------------------------------- 1 | XXX -------------------------------------------------------------------------------- /test/export/data/y: -------------------------------------------------------------------------------- 1 | YYY -------------------------------------------------------------------------------- /test/stress/stdlib.ojs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /site/public/docs/stdlib.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/data/a.txt: -------------------------------------------------------------------------------- 1 | AAAAAAAAAA -------------------------------------------------------------------------------- /test/data/b.txt: -------------------------------------------------------------------------------- 1 | BBBBBBBBBB -------------------------------------------------------------------------------- /test/data/c.txt: -------------------------------------------------------------------------------- 1 | CCCCCCCCCC -------------------------------------------------------------------------------- /test/data/x.txt: -------------------------------------------------------------------------------- 1 | XXXXX 2 | -------------------------------------------------------------------------------- /test/data/y.txt: -------------------------------------------------------------------------------- 1 | YYYYY 2 | -------------------------------------------------------------------------------- /test/export/a.ojs: -------------------------------------------------------------------------------- 1 | a = 100; 2 | -------------------------------------------------------------------------------- /test/export/b.ojs: -------------------------------------------------------------------------------- 1 | b = 200; 2 | -------------------------------------------------------------------------------- /test/export/data/sub/f1: -------------------------------------------------------------------------------- 1 | file 1 bby -------------------------------------------------------------------------------- /test/export/data/sub/f2: -------------------------------------------------------------------------------- 1 | f2 bbyyyyy -------------------------------------------------------------------------------- /site/public/examples/census-api/stdlib.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /site/public/examples/github-api/stdlib.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /site/public/examples/wiki-pageviews/stdlib.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/imports/suba.ojs: -------------------------------------------------------------------------------- 1 | a = 3 2 | 3 | b = 3 4 | 5 | c = a - b -------------------------------------------------------------------------------- /test/imports/subb.ojs: -------------------------------------------------------------------------------- 1 | a = 4; 2 | 3 | b = 4; 4 | 5 | c = a * b; -------------------------------------------------------------------------------- /test/imports/subd-op.ojs: -------------------------------------------------------------------------------- 1 | function op(a, b){ 2 | return a / b; 3 | } -------------------------------------------------------------------------------- /test/builtins/width.ojs: -------------------------------------------------------------------------------- 1 | md`# Testing builtins: width` 2 | 3 | w = width; 4 | 5 | w, Date.now(); -------------------------------------------------------------------------------- /site/public/index.js: -------------------------------------------------------------------------------- 1 | export {default} from "./147b7a769e856472cafb081301cb1f171d92f08e70bf38979365bb601cd37251.js"; -------------------------------------------------------------------------------- /test/imports/subd.ojs: -------------------------------------------------------------------------------- 1 | import { op } from "./subd-op.ojs"; 2 | 3 | a = 1; 4 | 5 | b = 2; 6 | 7 | c = op(a, b); 8 | -------------------------------------------------------------------------------- /site/public/docs/index.js: -------------------------------------------------------------------------------- 1 | export {default} from "./bc54548ebb381725e6d095ab3932e77a89d4cdfccb634ba18465f0358d5a8501.js"; -------------------------------------------------------------------------------- /examples/local/filesize-update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | find $1 -type f -exec wc -c {} + \ 3 | > $(dirname "$0")/data/filesize.txt -------------------------------------------------------------------------------- /site/public/examples/census-api/index.js: -------------------------------------------------------------------------------- 1 | export {default} from "./645b0415e1030853bbc34781c1b23b6989a0bee79f978217f15eab1e73514c80.js"; -------------------------------------------------------------------------------- /site/public/examples/github-api/index.js: -------------------------------------------------------------------------------- 1 | export {default} from "./0b7ca4bb60e127b218d27d2d59e6f708d77da8fa16da661b77f587a262c8391c.js"; -------------------------------------------------------------------------------- /site/public/examples/wiki-pageviews/index.js: -------------------------------------------------------------------------------- 1 | export {default} from "./e9e128ada7176e897e2b52da0a123c594db2d80911b48bd4540baadfce3b3342.js"; -------------------------------------------------------------------------------- /test/stdlib/colors1.js: -------------------------------------------------------------------------------- 1 | window.DATAFLOW_STDLIB = { 2 | constants: { 3 | red: "#ee2222", 4 | blue: "#1122ee", 5 | green: "#33ff33", 6 | }, 7 | }; 8 | -------------------------------------------------------------------------------- /test/stdlib/colors2.js: -------------------------------------------------------------------------------- 1 | window.DATAFLOW_STDLIB = { 2 | constants: { 3 | red: "darkred", 4 | blue: "darkblue", 5 | green: "green", 6 | }, 7 | }; 8 | -------------------------------------------------------------------------------- /test/stdlib/funcs.js: -------------------------------------------------------------------------------- 1 | window.DATAFLOW_STDLIB = { 2 | constants: { 3 | upper: () => (s) => s.toUpperCase(), 4 | lower: () => (s) => s.toLowerCase(), 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /test/file-attachments/sub.ojs: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | x: ../data/x.txt 4 | y: ../data/y.txt 5 | */ 6 | 7 | x = FileAttachment("x"); 8 | 9 | y = FileAttachment("y"); 10 | -------------------------------------------------------------------------------- /test/test-suite.ojs: -------------------------------------------------------------------------------- 1 | md` 2 | # Test Suite 3 | `; 4 | 5 | function equals(actual, expected) { 6 | if (actual !== expected) return html`
FAIL
`; 7 | return html`
PASS
`; 8 | } 9 | -------------------------------------------------------------------------------- /test/export/sub_with_fa.ojs: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | f1: ./data/sub/f1 4 | f2: ./data/sub/f2 5 | */ 6 | 7 | f1 = FileAttachment("f1").text(); 8 | 9 | f2 = FileAttachment("f2").text(); 10 | 11 | content = f1 + f2; 12 | -------------------------------------------------------------------------------- /test/export/ts-me.ojs: -------------------------------------------------------------------------------- 1 | md` 2 | ## omg dont appear 3 | `; 4 | 5 | one_cell = 4545; 6 | 7 | md` 8 | asdlfjasdlfkjad 9 | asd 10 | fa 11 | sdf 12 | asd 13 | fa 14 | d 15 | df 16 | `; 17 | 18 | function x(x) { 19 | return x; 20 | } 21 | -------------------------------------------------------------------------------- /test/stdlib/funcs.ojs: -------------------------------------------------------------------------------- 1 | md` 2 | # Testing custom stdlib: functions 3 | `; 4 | 5 | md` 6 | ## Should be in uppercase 7 | `; 8 | 9 | upper("alex garcia"); 10 | 11 | md` 12 | ## Should be in lowercase 13 | `; 14 | 15 | lower("ALEX GARCIA"); 16 | -------------------------------------------------------------------------------- /test/stdlib/async.ojs: -------------------------------------------------------------------------------- 1 | md` 2 | # Async stdlib testing 3 | `; 4 | 5 | md` 6 | This should be d5 version 5 7 | `; 8 | 9 | d3v5; 10 | 11 | d3v5.version; 12 | 13 | md` 14 | This should be d5 version 6 15 | `; 16 | 17 | d3v6; 18 | 19 | d3v6.version; 20 | -------------------------------------------------------------------------------- /examples/preact/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | -------------------------------------------------------------------------------- /test/stdlib/colors.ojs: -------------------------------------------------------------------------------- 1 | md` 2 | # Testing custom stdlib: colors 3 | `; 4 | 5 | color = (color) => 6 | html`
9 | ${color} 10 |
`; 11 | 12 | r = color(red); 13 | 14 | g = color(green); 15 | 16 | b = color(blue); 17 | -------------------------------------------------------------------------------- /examples/local/data/filesize.txt: -------------------------------------------------------------------------------- 1 | 2920 ./src/dataflow 2 | 9787 ./src/run.js 3 | 3157 ./src/content/index.html 4 | 45385 ./src/content/core.js 5 | 176989 ./src/content/run.js 6 | 1329 ./src/client/core.js 7 | 9262 ./src/client/run.js 8 | 7718 ./src/client/index.js 9 | 8991 ./src/compile.js 10 | 265538 total 11 | -------------------------------------------------------------------------------- /examples/preact/a.ojs: -------------------------------------------------------------------------------- 1 | //import { chart } from "https://observablehq.com/@d3/horizontal-bar-chart"; 2 | 3 | a = 1; 4 | 5 | b = 2; 6 | 7 | c = a - b; 8 | 9 | d3 = require("d3"); 10 | 11 | svg` 12 | 13 | ${d3 14 | .range(100) 15 | .map( 16 | (d) => svg`` 17 | )}`; 18 | -------------------------------------------------------------------------------- /test/secrets/normal.ojs: -------------------------------------------------------------------------------- 1 | md` 2 | # Secrets Testing 3 | `; 4 | 5 | md` 6 | With no --allow-secrets option, this should fail 7 | `; 8 | 9 | Secret("PASSWORD"); 10 | 11 | md` 12 | ## Quick env var test 13 | `; 14 | 15 | md`Token: \`${await Secret("TOKEN")}\``; 16 | 17 | md` 18 | ## These should fail always 19 | `; 20 | Secret("not exist"); 21 | 22 | Secret("not a key"); 23 | -------------------------------------------------------------------------------- /examples/preact/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "preact", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@observablehq/runtime": "^4.8.2", 13 | "preact": "^10.5.13" 14 | }, 15 | "devDependencies": { 16 | "esbuild": "^0.11.15" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test/errors/parsing.ojs: -------------------------------------------------------------------------------- 1 | //const lol = 123 2 | 3 | md` 4 | # test - parsing errors 5 | 6 | ~~~bash 7 | code ./test/errors/parsing.ojs 8 | ~~~ 9 | `; 10 | 11 | md` 12 | ## 1) semicolons 13 | 14 | Remove the semicolon after width, parsing error should be displayed. 15 | `; 16 | 17 | width; 18 | 19 | { 20 | return 1; 21 | } 22 | 23 | md` 24 | ## 2) .ojs differences 25 | 26 | Uncomment this, see error message. 27 | `; 28 | 29 | //const name = "Alex"; 30 | -------------------------------------------------------------------------------- /examples/preact/App.jsx: -------------------------------------------------------------------------------- 1 | import { h, render } from "preact"; 2 | import { useEffect, useRef } from "preact/hooks"; 3 | import define from "./a.ojs"; 4 | import { Runtime, Inspector } from "@observablehq/runtime"; 5 | 6 | function App() { 7 | const ref = useRef(); 8 | useEffect(() => { 9 | if (!ref.current) return; 10 | const runtime = new Runtime(); 11 | const observer = Inspector.into(ref.current); 12 | runtime.module(define, observer); 13 | return () => runtime.dispose(); 14 | }, [ref]); 15 | return
Yolo
; 16 | } 17 | render(, document.body); 18 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.0.11 - 2021-10-21 4 | 5 | ### New 6 | 7 | - Upgrade observablehq runtime and stdlib. Includes SQLite file attachment support. 8 | 9 | ## v0.0.10 - 2021-07-11 10 | 11 | ### New 12 | 13 | - Shebang support (`#!/usr/bin/env dataflow run`) for `.ojs` files. 14 | 15 | ## v0.0.9 - 2021-05-22 16 | 17 | ### Fixed 18 | 19 | - Fix a bug where the `width` builtin cell in `dataflow run` erronously updated when the value had not changed. 20 | 21 | ## v0.0.8 - 2021-05-20 22 | 23 | ### Fixed 24 | 25 | - `dataflow run` on https sites (websocket uses `wss://` instead of `ws://`) 26 | -------------------------------------------------------------------------------- /test/file-attachments/normal.ojs: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | file: ../data/a.txt 4 | notexist: ../data/notexist.txt 5 | */ 6 | 7 | md` 8 | # Existing File Attachments 9 | 10 | Toggle \`file: ./data/a.txt\` above to \`b.txt\` or \`c.txt\` and this should update 11 | 12 | ~~~bash 13 | code ./test/file-attachments/normal.ojs 14 | ~~~ 15 | `; 16 | 17 | fileContents = FileAttachment("file").text(); 18 | 19 | md` 20 | ## Not existant FA 21 | `; 22 | 23 | FileAttachment("notexist").text(); 24 | 25 | md` 26 | ## Imported 27 | `; 28 | 29 | import { x, y } from "./sub.ojs"; 30 | 31 | x.text(); 32 | y.text(); 33 | -------------------------------------------------------------------------------- /test/file-attachments/live.ojs: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | csv: ../data/live.csv 4 | */ 5 | 6 | md` 7 | # Testing: Live File Attachments 8 | `; 9 | 10 | 11 | md`~~~bash 12 | node -e 'process.stdout.write(JSON.stringify(Array.from({length:25}, (_,i)=>i)))' | ndjson-split | ndjson-map '{name: "n" + d, value: Math.random() * 100}' | ndjson-reduce | json2csv > ./test/data/live.csv 13 | ~~~` 14 | 15 | LiveFileAttachment; 16 | 17 | fa = LiveFileAttachment("csv"); 18 | 19 | data = fa.csv({typed:true}); 20 | 21 | import {chart} with {data} from "https://observablehq.com/@d3/bar-chart" 22 | 23 | chart 24 | 25 | 26 | -------------------------------------------------------------------------------- /test/stdlib/async.js: -------------------------------------------------------------------------------- 1 | window.DATAFLOW_STDLIB = { 2 | dependency: { 3 | require: { 4 | d3v5: (require) => require("d3@5"), 5 | d3v6: (require) => require("d3@6"), 6 | _: (require) => require("lodash"), 7 | }, 8 | svg: { 9 | logo: (svg) => 10 | svg` 11 | 12 | 13 | 14 | `, 15 | }, 16 | d3v6: { 17 | fakeData: (d3) => d3.range(200), 18 | }, 19 | }, 20 | }; 21 | -------------------------------------------------------------------------------- /examples/local/filesize-live.ojs: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | filesize: ./data/filesize.txt 4 | */ 5 | 6 | md`# File Size Visualizer - Live!` 7 | 8 | md`~~~ 9 | find . -type f -exec wc -c {} + 10 | ~~~` 11 | 12 | f = LiveFileAttachment("filesize"); 13 | 14 | data = f.text(); 15 | 16 | fileColor 17 | 18 | fileColor = "#75F4F4" 19 | 20 | folderColor = "blue" 21 | 22 | import {chart} with { 23 | data as source, 24 | prettybytes as format, 25 | fileColor, 26 | folderColor 27 | } from "https://observablehq.com/@mbostock/file-size-visualizer-bubbles" 28 | 29 | chart 30 | 31 | md`## Appendix` 32 | 33 | prettybytes = (await import("https://cdn.skypack.dev/pretty-bytes")).default -------------------------------------------------------------------------------- /test/data/live.csv: -------------------------------------------------------------------------------- 1 | name,value 2 | n0,52.179743889602584 3 | n1,43.698913315603605 4 | n2,24.142692588125534 5 | n3,17.072075309409684 6 | n4,68.99474444109268 7 | n5,91.13652046567677 8 | n6,21.077864435737382 9 | n7,70.63678989330009 10 | n8,59.5196497393065 11 | n9,56.843170386652254 12 | n10,59.283442706790424 13 | n11,97.33394381683291 14 | n12,56.33922809892371 15 | n13,25.842072590845767 16 | n14,74.70547469042003 17 | n15,89.51596134292437 18 | n16,49.7042315622505 19 | n17,51.713098513934106 20 | n18,49.22392989383364 21 | n19,33.79208654520287 22 | n20,70.45653596813908 23 | n21,29.820378801555925 24 | n22,32.78030957515847 25 | n23,23.970796301817288 26 | n24,77.51144957050475 27 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | 2 | const { readFileSync } = require("rw").dash; 3 | const { readFile } = require("fs").promises; 4 | 5 | function maybeStripShebang(source) { 6 | if(source.startsWith("#!")) 7 | return source.substring(source.indexOf("\n")+1) 8 | 9 | return source 10 | } 11 | 12 | function readSourceCodeSync(path) { 13 | let source = readFileSync(path, "utf8"); 14 | return maybeStripShebang(source); 15 | } 16 | 17 | async function readSourceCode(path) { 18 | let source = await readFile(path, "utf8"); 19 | return maybeStripShebang(source) 20 | } 21 | 22 | async function readBinary(path) { 23 | return await readFile(path); 24 | } 25 | 26 | module.exports = { 27 | readSourceCodeSync,readSourceCode,readBinary 28 | } 29 | -------------------------------------------------------------------------------- /test/standalone.ojs: -------------------------------------------------------------------------------- 1 | md`## Standalone 2 | 3 | exporting this to JS/HTML should work as expected` 4 | 5 | 6 | viewof a = html`
11 | 12 | 13 | ` 14 | 15 | 16 | c = a + b 17 | 18 | 19 | svg` 20 | 21 | yoo` 22 | 23 | 24 | d3 = require("d3@6") 25 | 26 | d3.range(2000) 27 | 28 | import {chart} from "https://observablehq.com/@d3/bar-chart" 29 | 30 | chart -------------------------------------------------------------------------------- /site/index.ojs: -------------------------------------------------------------------------------- 1 | html` 2 |
3 |

Dataflow

4 | 5 |
A self-hosted Observable notebook editor.
6 | 7 |

Github

8 |

Documentation

9 | 10 |

Examples

11 | ` 16 | 17 | html` 97 | 98 | 102 | 108 | 109 | 110 |
111 |
112 | 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /site/public/examples/wiki-pageviews/files/157de8e31dca7ade4cdb014615698b244f9e4b5da7d9346ab255abde595d580e: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | source_ojs: ./wikipedia-pageviews.ojs 4 | */ 5 | 6 | md`# Wikipedia Pageviews 7 | 8 | Explore pageviews data for specific Wikipedia pages!` 9 | 10 | 11 | start = new Date("2020-10-01"); 12 | 13 | end = new Date("2021-01-01"); 14 | 15 | //viewof start = html``; 16 | //viewof end = html``; 17 | 18 | viewof pagesSource = Inputs.textarea({ 19 | rows: 6, 20 | width: 220, 21 | label: md`Pages"`, 22 | value: `Donald_Trump 23 | Mike_Pence 24 | Joe_Biden 25 | Kamala_Harris`, 26 | }); 27 | 28 | viewof labelsSource = Inputs.textarea({value:`2020-10-02,⬅ Trump tests positive for COVID19,115,20 29 | 2020-11-03,Election Day ➡,-50,120 30 | 2020-11-07,⬅AP Reports Biden wins,80,10`, label: md`Labels`}); 31 | legend = html`
32 | ${Object.keys(colors).map(k=>html` 33 |
34 | ${k}`)}` 35 | 36 | Plot.plot({ 37 | grid: true, 38 | width, 39 | marginLeft: 60, 40 | color: { 41 | domain: Object.keys(colors), 42 | range: Object.values(colors), 43 | }, 44 | y: { 45 | tickFormat: d3.format(".2s"), 46 | domain: yDomain, 47 | label: " ⬆ pageviews", 48 | nice: true, 49 | }, 50 | marks: [ 51 | Plot.line(data, { x: "date", y: "value", stroke: (d) => d.page, strokeWidth: 3 }), 52 | ...labels.map(d=>Plot.line([ 53 | [d.date, yDomain[0]], 54 | [d.date, yDomain[1]] 55 | ], {stroke: "#999"})), 56 | ...labels.map(label=>Plot.text([label], { 57 | x: label.date, 58 | y: yDomain[1], 59 | text: d => d.label, 60 | dx: label.dx, 61 | dy: label.dy, 62 | fontSize: 14 63 | })) 64 | ], 65 | }); 66 | 67 | md`## Source Code 68 | 69 | This is the original source code for this notebook, the \`wikipedia-pageviews.ojs\` file.` 70 | 71 | md`~~~javascript 72 | ${(await FileAttachment("source_ojs").text()).replace(/~~~/g, "```")} 73 | ~~~` 74 | 75 | 76 | md`## Appendix` 77 | 78 | yDomain = d3.scaleLinear() 79 | .domain(d3.extent(data, d=>d.value)) 80 | .nice() 81 | .domain(); 82 | 83 | labels = d3.csvParseRows(labelsSource, d=> ({ 84 | date: new Date(d[0]), 85 | label: d[1], 86 | dx: d[2], 87 | dy: d[3] 88 | })).filter(d=>d.date >= start && d.date <= end) 89 | 90 | 91 | pages = pagesSource.split("\n"); 92 | 93 | data = d3.merge( 94 | await Promise.all( 95 | pages.map(async (page) => 96 | Object.assign(await pageviews(page, start, end), { page }) 97 | ) 98 | ) 99 | ); 100 | 101 | colors = ({ 102 | Joe_Biden: "steelblue", 103 | Kamala_Harris: "lightsteelblue", 104 | Donald_Trump: "red", 105 | Mike_Pence: "pink", 106 | }); 107 | 108 | pageviews = (page, start, end) => { 109 | const site = "en.wikipedia"; 110 | if (typeof start === "string") 111 | start = d3.utcFormat("%Y%m%d00")(new Date(start)); 112 | if (start instanceof Date) start = d3.utcFormat("%Y%m%d00")(start); 113 | if (typeof end === "string") end = d3.utcFormat("%Y%m%d")(new Date(end)); 114 | if (end instanceof Date) end = d3.utcFormat("%Y%m%d")(end); 115 | 116 | return fetch( 117 | `https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/${site}/all-access/user/${page}/daily/${start}/${end}` 118 | ) 119 | .then((r) => r.json()) 120 | .then((d) => 121 | d.items.map((item) => ({ 122 | date: d3.utcParse("%Y%m%d00")(item.timestamp), 123 | value: item.views, 124 | page, 125 | })) 126 | ); 127 | }; 128 | 129 | document.title = "Wikipedia Pageviews API Example / Dataflow"; -------------------------------------------------------------------------------- /examples/local/census-api.ojs: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | source_ojs: ./census-api.ojs 4 | counties-albers-10m.json: ./data/counties-albers-10m.json 5 | */ 6 | md`# Census API 7 | 8 | Choropleth adapted from [Choropleth](https://observablehq.com/@d3/choropleth) on Observable (ISC License). 9 | 10 | Download the counties map with: 11 | 12 | ~~~bash 13 | wget -O data/counties-albers-10m.json \\ 14 | https://cdn.jsdelivr.net/npm/us-atlas@3/counties-albers-10m.json 15 | ~~~ 16 | 17 | This notebook is a bit messier than the other examples, but look at is as a light exporer of "Census variables" inside the [B01001](https://api.census.gov/data/2016/acs/acs1/groups/B01001.html) group. ` 18 | 19 | md`## ${variable.name} - ${variable.label}` 20 | 21 | viewof selVar = Inputs.select(vars, {format: ([variable, subVars]) => `${subVars.get("estimate").label} (${variable})`}) 22 | 23 | 24 | 25 | chart = { 26 | const svg = d3.create("svg") 27 | .attr("viewBox", [0, 0, 975, 610]); 28 | 29 | svg.append("g") 30 | .attr("transform", "translate(580,20)") 31 | .append(() => legend({color, title: data.title, width: 320, tickFormat:d3.format(".2%")})); 32 | 33 | svg.append("g") 34 | .selectAll("path") 35 | .data(topojson.feature(us, us.objects.counties).features) 36 | .join("path") 37 | .attr("fill", d => color(data.get(d.id))) 38 | .attr("d", path) 39 | .append("title") 40 | .text(d => `${d.properties.name}, ${states.get(d.id.slice(0, 2)).name} 41 | ${format(data.get(d.id))}`); 42 | 43 | svg.append("path") 44 | .datum(topojson.mesh(us, us.objects.states, (a, b) => a !== b)) 45 | .attr("fill", "none") 46 | .attr("stroke", "white") 47 | .attr("stroke-linejoin", "round") 48 | .attr("d", path); 49 | 50 | return svg.node(); 51 | } 52 | 53 | //B01001_003E 54 | rawData = { 55 | const raw = await fetch( 56 | `https://api.census.gov/data/2018/acs/acs5?get=B01001_001E,${variable.name}&for=county:*` 57 | ).then((r) => r.json()); 58 | 59 | const header = raw[0]; 60 | 61 | return raw.slice(1).map(d => { 62 | const ret = {}; 63 | for(const i in header) { 64 | ret[header[i]] = d[i]; 65 | } 66 | return ret; 67 | }) 68 | } 69 | 70 | function valueOf(d) { 71 | return d[variable.name] / d.B01001_001E; 72 | } 73 | 74 | data = new Map(rawData.map(d=>[`${d.state}${d.county}`, valueOf(d)])) 75 | 76 | B01001 = fetch("https://api.census.gov/data/2018/acs/acs5/groups/B01001/").then(r=>r.json()) 77 | 78 | 79 | 80 | vars = d3.rollup 81 | (Object.entries(B01001.variables).map(d=> ({ 82 | name: d[0], 83 | ...d[1] 84 | })),//.sort((a,b)=>d3.ascending(a[0], b[0]))), 85 | v => { 86 | return new Map([ 87 | ["estimate_annotation", v.find(d=>d.name.endsWith("EA"))], 88 | ["moe_annotation", v.find(d=>d.name.endsWith("MA"))], 89 | ["moe", v.find(d=>d.name.endsWith("M"))], 90 | ["estimate", v.find(d=>d.name.endsWith("E"))], 91 | ]) 92 | }, 93 | d => d.name.substring(0, "B01001_XXX".length)) 94 | 95 | variable = fetch( 96 | `https://api.census.gov/data/2018/acs/acs5/variables/${selVar.get("estimate").name}.json` 97 | ).then((r) => r.json()); 98 | 99 | 100 | 101 | 102 | color = d3.scaleQuantize(d3.extent(data.values()), d3.schemeBlues[9]) 103 | 104 | path = d3.geoPath() 105 | 106 | format = d => `${d}%` 107 | 108 | states = new Map(us.objects.states.geometries.map(d => [d.id, d.properties])) 109 | 110 | 111 | us = FileAttachment("counties-albers-10m.json").json() 112 | 113 | topojson = require("topojson-client@3") 114 | 115 | 116 | import {legend} from "https://observablehq.com/@d3/color-legend" 117 | 118 | 119 | md`## Source Code 120 | 121 | This is the original source code for this notebook, the \`census-api.ojs\` file.` 122 | 123 | md`~~~javascript 124 | ${(await FileAttachment("source_ojs").text()).replace(/~~~/g, "```")} 125 | ~~~` 126 | 127 | document.title = "Census API Example / Dataflow"; -------------------------------------------------------------------------------- /site/public/examples/census-api/files/e54361c3e7be0df0f0b536e79d37e5d0ce58eccdb461006df04312ed1a8feadd: -------------------------------------------------------------------------------- 1 | /* 2 | FileAttachments: 3 | source_ojs: ./census-api.ojs 4 | counties-albers-10m.json: ./data/counties-albers-10m.json 5 | */ 6 | md`# Census API 7 | 8 | Choropleth adapted from [Choropleth](https://observablehq.com/@d3/choropleth) on Observable (ISC License). 9 | 10 | Download the counties map with: 11 | 12 | ~~~bash 13 | wget -O data/counties-albers-10m.json \\ 14 | https://cdn.jsdelivr.net/npm/us-atlas@3/counties-albers-10m.json 15 | ~~~ 16 | 17 | This notebook is a bit messier than the other examples, but look at is as a light exporer of "Census variables" inside the [B01001](https://api.census.gov/data/2016/acs/acs1/groups/B01001.html) group. ` 18 | 19 | md`## ${variable.name} - ${variable.label}` 20 | 21 | viewof selVar = Inputs.select(vars, {format: ([variable, subVars]) => `${subVars.get("estimate").label} (${variable})`}) 22 | 23 | 24 | 25 | chart = { 26 | const svg = d3.create("svg") 27 | .attr("viewBox", [0, 0, 975, 610]); 28 | 29 | svg.append("g") 30 | .attr("transform", "translate(580,20)") 31 | .append(() => legend({color, title: data.title, width: 320, tickFormat:d3.format(".2%")})); 32 | 33 | svg.append("g") 34 | .selectAll("path") 35 | .data(topojson.feature(us, us.objects.counties).features) 36 | .join("path") 37 | .attr("fill", d => color(data.get(d.id))) 38 | .attr("d", path) 39 | .append("title") 40 | .text(d => `${d.properties.name}, ${states.get(d.id.slice(0, 2)).name} 41 | ${format(data.get(d.id))}`); 42 | 43 | svg.append("path") 44 | .datum(topojson.mesh(us, us.objects.states, (a, b) => a !== b)) 45 | .attr("fill", "none") 46 | .attr("stroke", "white") 47 | .attr("stroke-linejoin", "round") 48 | .attr("d", path); 49 | 50 | return svg.node(); 51 | } 52 | 53 | //B01001_003E 54 | rawData = { 55 | const raw = await fetch( 56 | `https://api.census.gov/data/2018/acs/acs5?get=B01001_001E,${variable.name}&for=county:*` 57 | ).then((r) => r.json()); 58 | 59 | const header = raw[0]; 60 | 61 | return raw.slice(1).map(d => { 62 | const ret = {}; 63 | for(const i in header) { 64 | ret[header[i]] = d[i]; 65 | } 66 | return ret; 67 | }) 68 | } 69 | 70 | function valueOf(d) { 71 | return d[variable.name] / d.B01001_001E; 72 | } 73 | 74 | data = new Map(rawData.map(d=>[`${d.state}${d.county}`, valueOf(d)])) 75 | 76 | B01001 = fetch("https://api.census.gov/data/2018/acs/acs5/groups/B01001/").then(r=>r.json()) 77 | 78 | 79 | 80 | vars = d3.rollup 81 | (Object.entries(B01001.variables).map(d=> ({ 82 | name: d[0], 83 | ...d[1] 84 | })),//.sort((a,b)=>d3.ascending(a[0], b[0]))), 85 | v => { 86 | return new Map([ 87 | ["estimate_annotation", v.find(d=>d.name.endsWith("EA"))], 88 | ["moe_annotation", v.find(d=>d.name.endsWith("MA"))], 89 | ["moe", v.find(d=>d.name.endsWith("M"))], 90 | ["estimate", v.find(d=>d.name.endsWith("E"))], 91 | ]) 92 | }, 93 | d => d.name.substring(0, "B01001_XXX".length)) 94 | 95 | variable = fetch( 96 | `https://api.census.gov/data/2018/acs/acs5/variables/${selVar.get("estimate").name}.json` 97 | ).then((r) => r.json()); 98 | 99 | 100 | 101 | 102 | color = d3.scaleQuantize(d3.extent(data.values()), d3.schemeBlues[9]) 103 | 104 | path = d3.geoPath() 105 | 106 | format = d => `${d}%` 107 | 108 | states = new Map(us.objects.states.geometries.map(d => [d.id, d.properties])) 109 | 110 | 111 | us = FileAttachment("counties-albers-10m.json").json() 112 | 113 | topojson = require("topojson-client@3") 114 | 115 | 116 | import {legend} from "https://observablehq.com/@d3/color-legend" 117 | 118 | 119 | md`## Source Code 120 | 121 | This is the original source code for this notebook, the \`census-api.ojs\` file.` 122 | 123 | md`~~~javascript 124 | ${(await FileAttachment("source_ojs").text()).replace(/~~~/g, "```")} 125 | ~~~` 126 | 127 | document.title = "Census API Example / Dataflow"; -------------------------------------------------------------------------------- /docs/shebang.md: -------------------------------------------------------------------------------- 1 | ## Executable Notebooks (aka shebang) 2 | 3 | Observable notebooks in Dataflow can be defined in executable files, making it easier to manage a fleet of notebooks at the same time. If you're tired to re-typing `dataflow run notebook.ojs --allow-file-attachments` all the time, then this will make your life a tad easier! 4 | 5 | 6 | For example, let's say that you have a directory of notebooks that each explore a different aspect of your personal finances: `loans.ojs`, `investments.ojs`, `loan.ojs`, and `spending.ojs`. If you wanted to run `dataflow run` on each of these, you'd have to type a lot: `dataflow run .ojs`, unique ports with `-p $PORT`, `--allow-file-attachments` and whatever other flags you need. 7 | 8 | Or instead, you can add the follow shebangs to the first line of those notebooks: 9 | 10 | ``` 11 | $ head * 12 | ==> income.ojs <== 13 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2001 14 | /* 15 | FileAttachments: 16 | jobs.csv: ./data/jobs.csv 17 | */ 18 | 19 | ==> investments.ojs <== 20 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2002 21 | /* 22 | FileAttachments: 23 | stocks-2021.csv: ./data/stocks-2021.csv 24 | */ 25 | 26 | ==> loans.ojs <== 27 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2003 28 | /* 29 | FileAttachments: 30 | student_loans.csv: ./data/student_loans.csv 31 | mortgage_payments.csv: ./data/mortgage_payments.csv 32 | */ 33 | 34 | ==> spending.ojs <== 35 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2004 36 | /* 37 | spending2021.csv: ./data/spending-2021.csv 38 | spending-historical.csv: ./data/spending-historical.csv 39 | */ 40 | ``` 41 | 42 | Give those files user execution permissions: 43 | ``` 44 | chmod u+x *.ojs 45 | ``` 46 | 47 | And then run those files directly, no extra calls necessary! 48 | 49 | ``` 50 | ./income.ojs 51 | ./investments.ojs 52 | ./loans.ojs 53 | ./spending.ojs 54 | ``` 55 | 56 | There will now be 4 seperate `dataflow run` devservers running, on ports 2001 (`income.ojs`), 2002 (`investments.ojs`), 2003 (`loans.ojs`), and 2004 (`spending.ojs`). 57 | 58 | Note: I don't think Windows support executing files with a shebang, but I'd love to be proven wrong! 59 | 60 | ### Shebang Guide 61 | 62 | #### 1) Add the `#!/usr/bin/env dataflow run` interpretive directive 63 | 64 | At the top of your `.ojs` file, add the following: 65 | 66 | ``` 67 | #!/usr/bin/env dataflow run 68 | ``` 69 | 70 | This is the "interpretive directive", aka the [shebang](https://en.wikipedia.org/wiki/Shebang_%28Unix%29), that your shell (bash, fish, zsh, etc.) will use to determine how to run a given file. `/usr/bin/env` will execute the `dataflow` script that's on your `$PATH` with whatever arguments are passed in. If your notebook requires extra arguments, like `--allow-file-attachments`, than can be added after `dataflow run` on the same line. It is reccommended that you also add value for the `--port/-p` option, to ensure that your executable notebooks doesn't conflict with another notebook using the default port. 71 | 72 | ``` 73 | #!/usr/bin/env dataflow run --allow-file-attachments -p 3304 74 | /* 75 | FileAttachments: 76 | whatever.csv: whatever.csv 77 | */ 78 | 79 | md`# title` 80 | 81 | data = FileAttachments('whatever.csv').csv() 82 | ``` 83 | 84 | It's important that the shebang line is the very first line. Any Dataflow-specific configuration comments (ie the `FileAttachments` YAML config) should appear in the 2nd line right after. 85 | 86 | #### 2) Give your `.ojs` file user executable permissions 87 | 88 | Your shell will expect executable files to have execute permissions, which can be given like so: 89 | 90 | ``` 91 | chmod u+x notebook.ojs 92 | ``` 93 | 94 | *Translation: "change the file mode of the 'notebook.ojs' file and grant e**x**ecutable permission to the **u**ser."* 95 | 96 | 97 | #### 3) Execute your notebooks! 98 | 99 | Now you can run your notebooks with just the filename: 100 | 101 | ``` 102 | $ ./notebook.ojs 103 | ``` 104 | 105 | And it will run `dataflow run` with whatever arguments you passed in! Less clutter wow ✨ -------------------------------------------------------------------------------- /site/public/docs/files/7e2bc186f16000498c9227c6bd4055458bd74d7204e9dd8594d81d0ccf7587fd: -------------------------------------------------------------------------------- 1 | ## Executable Notebooks (aka shebang) 2 | 3 | Observable notebooks in Dataflow can be defined in executable files, making it easier to manage a fleet of notebooks at the same time. If you're tired to re-typing `dataflow run notebook.ojs --allow-file-attachments` all the time, then this will make your life a tad easier! 4 | 5 | 6 | For example, let's say that you have a directory of notebooks that each explore a different aspect of your personal finances: `loans.ojs`, `investments.ojs`, `loan.ojs`, and `spending.ojs`. If you wanted to run `dataflow run` on each of these, you'd have to type a lot: `dataflow run .ojs`, unique ports with `-p $PORT`, `--allow-file-attachments` and whatever other flags you need. 7 | 8 | Or instead, you can add the follow shebangs to the first line of those notebooks: 9 | 10 | ``` 11 | $ head * 12 | ==> income.ojs <== 13 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2001 14 | /* 15 | FileAttachments: 16 | jobs.csv: ./data/jobs.csv 17 | */ 18 | 19 | ==> investments.ojs <== 20 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2002 21 | /* 22 | FileAttachments: 23 | stocks-2021.csv: ./data/stocks-2021.csv 24 | */ 25 | 26 | ==> loans.ojs <== 27 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2003 28 | /* 29 | FileAttachments: 30 | student_loans.csv: ./data/student_loans.csv 31 | mortgage_payments.csv: ./data/mortgage_payments.csv 32 | */ 33 | 34 | ==> spending.ojs <== 35 | #!/usr/bin/env dataflow run --allow-file-attachments -p 2004 36 | /* 37 | spending2021.csv: ./data/spending-2021.csv 38 | spending-historical.csv: ./data/spending-historical.csv 39 | */ 40 | ``` 41 | 42 | Give those files user execution permissions: 43 | ``` 44 | chmod u+x *.ojs 45 | ``` 46 | 47 | And then run those files directly, no extra calls necessary! 48 | 49 | ``` 50 | ./income.ojs 51 | ./investments.ojs 52 | ./loans.ojs 53 | ./spending.ojs 54 | ``` 55 | 56 | There will now be 4 seperate `dataflow run` devservers running, on ports 2001 (`income.ojs`), 2002 (`investments.ojs`), 2003 (`loans.ojs`), and 2004 (`spending.ojs`). 57 | 58 | Note: I don't think Windows support executing files with a shebang, but I'd love to be proven wrong! 59 | 60 | ### Shebang Guide 61 | 62 | #### 1) Add the `#!/usr/bin/env dataflow run` interpretive directive 63 | 64 | At the top of your `.ojs` file, add the following: 65 | 66 | ``` 67 | #!/usr/bin/env dataflow run 68 | ``` 69 | 70 | This is the "interpretive directive", aka the [shebang](https://en.wikipedia.org/wiki/Shebang_%28Unix%29), that your shell (bash, fish, zsh, etc.) will use to determine how to run a given file. `/usr/bin/env` will execute the `dataflow` script that's on your `$PATH` with whatever arguments are passed in. If your notebook requires extra arguments, like `--allow-file-attachments`, than can be added after `dataflow run` on the same line. It is reccommended that you also add value for the `--port/-p` option, to ensure that your executable notebooks doesn't conflict with another notebook using the default port. 71 | 72 | ``` 73 | #!/usr/bin/env dataflow run --allow-file-attachments -p 3304 74 | /* 75 | FileAttachments: 76 | whatever.csv: whatever.csv 77 | */ 78 | 79 | md`# title` 80 | 81 | data = FileAttachments('whatever.csv').csv() 82 | ``` 83 | 84 | It's important that the shebang line is the very first line. Any Dataflow-specific configuration comments (ie the `FileAttachments` YAML config) should appear in the 2nd line right after. 85 | 86 | #### 2) Give your `.ojs` file user executable permissions 87 | 88 | Your shell will expect executable files to have execute permissions, which can be given like so: 89 | 90 | ``` 91 | chmod u+x notebook.ojs 92 | ``` 93 | 94 | *Translation: "change the file mode of the 'notebook.ojs' file and grant e**x**ecutable permission to the **u**ser."* 95 | 96 | 97 | #### 3) Execute your notebooks! 98 | 99 | Now you can run your notebooks with just the filename: 100 | 101 | ``` 102 | $ ./notebook.ojs 103 | ``` 104 | 105 | And it will run `dataflow run` with whatever arguments you passed in! Less clutter wow ✨ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dataflow 2 | 3 | A self-hosted Observable notebook editor, with support for FileAttachments, Secrets, custom standard libraries, and more! 4 | 5 | https://user-images.githubusercontent.com/15178711/118158592-d8fd9900-b3d0-11eb-97e1-70ae97d35038.mp4 6 | 7 | ## Examples 8 | 9 | Here are some examples of Observable notebooked created and compiled with Dataflow, along with their original source: 10 | 11 | - [Wikpedia Pageview](https://alexgarcia.xyz/dataflow/examples/wiki-pageviews/) ([source code](https://github.com/asg017/dataflow/blob/main/examples/local/wikipedia-pageviews.ojs)) 12 | - [GitHub API Notebook](https://alexgarcia.xyz/dataflow/examples/github-api/) ([source code](https://github.com/asg017/dataflow/blob/main/examples/local/github-api.ojs)) 13 | - [Census API](https://alexgarcia.xyz/dataflow/examples/census-api/) ([source code](https://github.com/asg017/dataflow/blob/main/examples/local/census-api.ojs)) 14 | 15 | ## Documentation 16 | 17 | Check out https://alexgarcia.xyz/dataflow for documentation! Fun fact, this site is entirely build with Dataflow :eyes: 18 | 19 | ## Install 20 | 21 | ```bash 22 | npm i -g @alex.garcia/dataflow 23 | 24 | dataflow --help 25 | 26 | dataflow run ./my-notebook.ojs 27 | ``` 28 | 29 | ## Background 30 | 31 | [Observable notebooks](http://observablehq.com/) are reactive, JavaScript-based computational notebooks that run inside your browser. Dataflow is one of the first fully open-sourced and fully featured Observable notebook editors, with key differences that make it easier to integrate with other developer tools! 32 | 33 | ### Edit Notebooks in "Observable JavaScript" `.ojs` files 34 | 35 | Dataflow notebooks are files on your computer, in the form of `.ojs` files. A single `.ojs` file is analagous to a single Observable notebook, and `.ojs` files can import from other `.ojs` files. 36 | 37 | `dataflow run my-notebook.ojs` will start a dev server at `localhost:8080` that shows a live rendered look at a notebook defined in `my-notebook.ojs`. Any update you make to the `my-notebook.ojs` file from any text editor will instantly update. 38 | 39 | Since notebooks are file-based, they can be easily version controlled (ie git) and appear alongside your other frontend code. Dataflow can also compile these notebooks with `dataflow compile` to plain JavaScript ES modules, and generate HTML files that run a notebook locally. 40 | 41 | ### Takes advantage of local development 42 | 43 | Since Dataflow notebooks aren't ran in a sandboxed iframe, that means you can control every aspect to how a notebook looks. Include CSS and stylesheets to change the look of it, change favicons or `document.title`, or access browser features not available for iframe like acessing Bluetooth devices or printing notebooks. 44 | 45 | Dataflow also offers easy acess to your filesystem with file attachments. Instead manually uploading files, you can simple include a configuration comment in a notebook to the path of your FileAttachment, and Dataflow will be instantly available with the same `FileAttachment` API as observablehq.com. You can also have "live" file attachments with `LiveFileAttachment`, which updates everytime a file attachment's file updates, making it even faster to rapidily test different data sources. See documentation for [Dataflow File Attachments](https://alexgarcia.xyz/dataflow/#file-attachments) for more. 46 | 47 | Finally, since Dataflow is just another service that runs on `localhost:8080`, you can build your own APIs and webservices that run local to your computer that Dataflow can access. You can build a proxy to your own database or query from local data apps with a little elbow grease! 48 | 49 | ### Customizable 50 | 51 | Dataflow aims to be extensible and customizable. [Custom Standard Library](https://alexgarcia.xyz/dataflow/#custom-standard-libraries) make it easier to define new builtin cells for your notebooks, [Secrets](https://alexgarcia.xyz/dataflow/#secrets) make it easier to pass in sensitive configuration, and working with "files as notebooks" mean you can bring in whatever text editor you want. 52 | 53 | That being said, There's still a lot of room to make Dataflow more customizable! [Custom styling](https://github.com/asg017/dataflow/issues/9), [more importing options](https://github.com/asg017/dataflow/issues/10), and [more compiling options](https://github.com/asg017/dataflow/issues/17) are planned, so watch this repo for updates! 54 | 55 | ## License 56 | 57 | Dataflow is MIT licensed, and heavily relies on these ISC licensed libraries: 58 | 59 | - https://github.com/observablehq/parser 60 | - https://github.com/observablehq/runtime 61 | - https://github.com/observablehq/stdlib 62 | -------------------------------------------------------------------------------- /test/design.ojs: -------------------------------------------------------------------------------- 1 | md`# Test Design` 2 | 3 | md`--- 4 | 5 | [Source](https://en.wikipedia.org/wiki/John_F._Kennedy) 6 | 7 | ## Level 2 8 | 9 | John Fitzgerald Kennedy (May 29, 1917 – November 22, 1963), often referred to by his initials JFK, was an American politician who served as the 35th president of the United States from 1961 until his assassination in 1963. \`Kennedy\` served at the height of the Cold War, and the majority of his work as president concerned relations with the Soviet Union and Cuba. A Democrat, Kennedy represented Massachusetts in both houses of the U.S. Congress prior to becoming president. https://github.com/d3/d3 10 | 11 | 12 | 13 | ### Level 3 14 | 15 | In September 1931, Kennedy started attending Choate, a prestigious boarding school in Wallingford, Connecticut, for 9th through 12th grade. His older brother Joe Jr. had already been at Choate for two years and was a football player and leading student. He spent his first years at Choate in his older brother's shadow and compensated with rebellious behavior that attracted a coterie. Their most notorious stunt was exploding a toilet seat with a powerful firecracker. In the next chapel assembly, the strict headmaster, George St. John, brandished the toilet seat and spoke of certain "muckers" who would "spit in our sea". Defiantly Kennedy took a cue and named his group "The Muckers Club", which included roommate and lifelong friend Kirk LeMoyne "Lem" Billings.[19] 16 | 17 | 18 | 19 | #### Level 4 20 | 21 | When Kennedy was an upperclassman at Harvard, he began to take his studies more seriously and developed an interest in political philosophy. He made the dean's list in his junior year.[32] In 1940 Kennedy completed his thesis, "Appeasement in Munich", about British negotiations during the Munich Agreement. The thesis eventually became a bestseller under the title Why England Slept.[33] In addition to addressing Britain's unwillingness to strengthen its military in the lead-up to World War II, the book also called for an Anglo-American alliance against the rising totalitarian powers. Kennedy became increasingly supportive of U.S. intervention in World War II, and his father's isolationist beliefs resulted in the latter's dismissal as ambassador to the United Kingdom. This created a split between the Kennedy and Roosevelt families.[34] 22 | 23 | 24 | 25 | ##### Level 5 26 | 27 | In 1940, Kennedy graduated cum laude from Harvard with a Bachelor of Arts in government, concentrating on international affairs. That fall, he enrolled at the Stanford Graduate School of Business and audited classes there.[35] In early 1941, Kennedy left and helped his father write a memoir of his time as an American ambassador. He then traveled throughout South America; his itinerary included Colombia, Ecuador and Peru.[36][37] 28 | 29 | 30 | ---` 31 | 32 | tex.block`\begin{aligned} 33 | \frac{\mathrm{d}🐁}{\mathrm{d}t} &= \alpha 🐁-\beta 🐁🐈 \\[2ex] 34 | \frac{\mathrm{d}🐈}{\mathrm{d}t} &= \delta 🐁🐈-\gamma 🐈 35 | \end{aligned}` 36 | 37 | 38 | md` 39 | [Source](https://github.com/d3/d3-selection) 40 | 41 | ~~~html 42 | 43 | 48 | ~~~ 49 | 50 | ~~~javascript 51 | 52 | const x = 1234; 53 | const y = 1234; 54 | 55 | const div = d3.select("body") 56 | .selectAll("div") 57 | .data([4, 8, 15, 16, 23, 42]) 58 | .enter().append("div") 59 | .text(d => d); 60 | 61 | selection.each(function(d) { foo.set(this, d.value); }); 62 | 63 | viewof x = html\`\` 64 | 65 | ~~~` 66 | 67 | md`- apples 68 | - bananas 69 | - oranges 70 | - fwuit snaccs 71 | 72 | > Roses are red, 73 | 74 | > Violets are blue, 75 | 76 | > Sugar is sweet, 77 | 78 | > And so are you.` 79 | 80 | commits = fetch('https://api.github.com/repos/streamlit/streamlit/commits').then(r=>r.json()) 81 | 82 | md`|Table|column| longer column name 83 | |-|-|- 84 | ${commits.map(d=>`|${d.sha}|${d.commit.author.name}|${d.commit.author.date}`).join("\n")}` 85 | 86 | md`---` 87 | 88 | import {viewof selection} from "https://observablehq.com/@d3/brushable-scatterplot" 89 | 90 | viewof selection 91 | 92 | md`## Tables` 93 | 94 | Table = (await require("@observablehq/inputs")).Table 95 | 96 | Table( 97 | Array.from({ length: 100 }, () => ({ 98 | a: Math.random(), 99 | b: Math.random(), 100 | c: Math.random(), 101 | d: Math.random(), 102 | x: Math.random(), 103 | y: Math.random(), 104 | z: Math.random(), 105 | zz: Math.random() 106 | })) 107 | ) -------------------------------------------------------------------------------- /site/public/examples/wiki-pageviews/e9e128ada7176e897e2b52da0a123c594db2d80911b48bd4540baadfce3b3342.js: -------------------------------------------------------------------------------- 1 | export default function define(runtime, observer) { 2 | const main = runtime.module(); 3 | const fileAttachments = new Map([["source_ojs", new URL("./files/157de8e31dca7ade4cdb014615698b244f9e4b5da7d9346ab255abde595d580e", import.meta.url)]]); 4 | main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name))); 5 | main.variable(observer()).define(["md"], function(md){return( 6 | md`# Wikipedia Pageviews 7 | 8 | Explore pageviews data for specific Wikipedia pages!` 9 | )}); 10 | main.variable(observer("start")).define("start", function(){return( 11 | new Date("2020-10-01") 12 | )}); 13 | main.variable(observer("end")).define("end", function(){return( 14 | new Date("2021-01-01") 15 | )}); 16 | main.variable(observer("viewof pagesSource")).define("viewof pagesSource", ["Inputs","md"], function(Inputs,md){return( 17 | Inputs.textarea({ 18 | rows: 6, 19 | width: 220, 20 | label: md`Pages"`, 21 | value: `Donald_Trump 22 | Mike_Pence 23 | Joe_Biden 24 | Kamala_Harris`, 25 | }) 26 | )}); 27 | main.variable(null).define("pagesSource", ["Generators", "viewof pagesSource"], (G, _) => G.input(_)); 28 | main.variable(observer("viewof labelsSource")).define("viewof labelsSource", ["Inputs","md"], function(Inputs,md){return( 29 | Inputs.textarea({value:`2020-10-02,⬅ Trump tests positive for COVID19,115,20 30 | 2020-11-03,Election Day ➡,-50,120 31 | 2020-11-07,⬅AP Reports Biden wins,80,10`, label: md`Labels`}) 32 | )}); 33 | main.variable(null).define("labelsSource", ["Generators", "viewof labelsSource"], (G, _) => G.input(_)); 34 | main.variable(observer("legend")).define("legend", ["html","colors"], function(html,colors){return( 35 | html`
36 | ${Object.keys(colors).map(k=>html` 37 |
38 | ${k}`)}` 39 | )}); 40 | main.variable(observer()).define(["Plot","width","colors","d3","yDomain","data","labels"], function(Plot,width,colors,d3,yDomain,data,labels){return( 41 | Plot.plot({ 42 | grid: true, 43 | width, 44 | marginLeft: 60, 45 | color: { 46 | domain: Object.keys(colors), 47 | range: Object.values(colors), 48 | }, 49 | y: { 50 | tickFormat: d3.format(".2s"), 51 | domain: yDomain, 52 | label: " ⬆ pageviews", 53 | nice: true, 54 | }, 55 | marks: [ 56 | Plot.line(data, { x: "date", y: "value", stroke: (d) => d.page, strokeWidth: 3 }), 57 | ...labels.map(d=>Plot.line([ 58 | [d.date, yDomain[0]], 59 | [d.date, yDomain[1]] 60 | ], {stroke: "#999"})), 61 | ...labels.map(label=>Plot.text([label], { 62 | x: label.date, 63 | y: yDomain[1], 64 | text: d => d.label, 65 | dx: label.dx, 66 | dy: label.dy, 67 | fontSize: 14 68 | })) 69 | ], 70 | }) 71 | )}); 72 | main.variable(observer()).define(["md"], function(md){return( 73 | md`## Source Code 74 | 75 | This is the original source code for this notebook, the \`wikipedia-pageviews.ojs\` file.` 76 | )}); 77 | main.variable(observer()).define(["md","FileAttachment"], async function(md,FileAttachment){return( 78 | md`~~~javascript 79 | ${(await FileAttachment("source_ojs").text()).replace(/~~~/g, "```")} 80 | ~~~` 81 | )}); 82 | main.variable(observer()).define(["md"], function(md){return( 83 | md`## Appendix` 84 | )}); 85 | main.variable(observer("yDomain")).define("yDomain", ["d3","data"], function(d3,data){return( 86 | d3.scaleLinear() 87 | .domain(d3.extent(data, d=>d.value)) 88 | .nice() 89 | .domain() 90 | )}); 91 | main.variable(observer("labels")).define("labels", ["d3","labelsSource","start","end"], function(d3,labelsSource,start,end){return( 92 | d3.csvParseRows(labelsSource, d=> ({ 93 | date: new Date(d[0]), 94 | label: d[1], 95 | dx: d[2], 96 | dy: d[3] 97 | })).filter(d=>d.date >= start && d.date <= end) 98 | )}); 99 | main.variable(observer("pages")).define("pages", ["pagesSource"], function(pagesSource){return( 100 | pagesSource.split("\n") 101 | )}); 102 | main.variable(observer("data")).define("data", ["d3","pages","pageviews","start","end"], async function(d3,pages,pageviews,start,end){return( 103 | d3.merge( 104 | await Promise.all( 105 | pages.map(async (page) => 106 | Object.assign(await pageviews(page, start, end), { page }) 107 | ) 108 | ) 109 | ) 110 | )}); 111 | main.variable(observer("colors")).define("colors", function(){return( 112 | { 113 | Joe_Biden: "steelblue", 114 | Kamala_Harris: "lightsteelblue", 115 | Donald_Trump: "red", 116 | Mike_Pence: "pink", 117 | } 118 | )}); 119 | main.variable(observer("pageviews")).define("pageviews", ["d3"], function(d3){return( 120 | (page, start, end) => { 121 | const site = "en.wikipedia"; 122 | if (typeof start === "string") 123 | start = d3.utcFormat("%Y%m%d00")(new Date(start)); 124 | if (start instanceof Date) start = d3.utcFormat("%Y%m%d00")(start); 125 | if (typeof end === "string") end = d3.utcFormat("%Y%m%d")(new Date(end)); 126 | if (end instanceof Date) end = d3.utcFormat("%Y%m%d")(end); 127 | 128 | return fetch( 129 | `https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/${site}/all-access/user/${page}/daily/${start}/${end}` 130 | ) 131 | .then((r) => r.json()) 132 | .then((d) => 133 | d.items.map((item) => ({ 134 | date: d3.utcParse("%Y%m%d00")(item.timestamp), 135 | value: item.views, 136 | page, 137 | })) 138 | ); 139 | } 140 | )}); 141 | main.variable(observer()).define(function(){return( 142 | document.title = "Wikipedia Pageviews API Example / Dataflow" 143 | )}); 144 | return main; 145 | } -------------------------------------------------------------------------------- /site/public/examples/census-api/645b0415e1030853bbc34781c1b23b6989a0bee79f978217f15eab1e73514c80.js: -------------------------------------------------------------------------------- 1 | import define1 from "https://api.observablehq.com/@d3/color-legend.js?v=3"; 2 | 3 | export default function define(runtime, observer) { 4 | const main = runtime.module(); 5 | const fileAttachments = new Map([["counties-albers-10m.json", new URL("./files/b82e08fb63aac373d976e0203e5b0d446c321c5e3fca0d7c772ae5900149a2fe", import.meta.url)],["source_ojs", new URL("./files/e54361c3e7be0df0f0b536e79d37e5d0ce58eccdb461006df04312ed1a8feadd", import.meta.url)]]); 6 | main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name))); 7 | main.variable(observer()).define(["md"], function(md){return( 8 | md`# Census API 9 | 10 | Choropleth adapted from [Choropleth](https://observablehq.com/@d3/choropleth) on Observable (ISC License). 11 | 12 | Download the counties map with: 13 | 14 | ~~~bash 15 | wget -O data/counties-albers-10m.json \\ 16 | https://cdn.jsdelivr.net/npm/us-atlas@3/counties-albers-10m.json 17 | ~~~ 18 | 19 | This notebook is a bit messier than the other examples, but look at is as a light exporer of "Census variables" inside the [B01001](https://api.census.gov/data/2016/acs/acs1/groups/B01001.html) group. ` 20 | )}); 21 | main.variable(observer()).define(["md","variable"], function(md,variable){return( 22 | md`## ${variable.name} - ${variable.label}` 23 | )}); 24 | main.variable(observer("viewof selVar")).define("viewof selVar", ["Inputs","vars"], function(Inputs,vars){return( 25 | Inputs.select(vars, {format: ([variable, subVars]) => `${subVars.get("estimate").label} (${variable})`}) 26 | )}); 27 | main.variable(null).define("selVar", ["Generators", "viewof selVar"], (G, _) => G.input(_)); 28 | main.variable(observer("chart")).define("chart", ["d3","legend","color","data","topojson","us","path","states","format"], function(d3,legend,color,data,topojson,us,path,states,format) 29 | { 30 | const svg = d3.create("svg") 31 | .attr("viewBox", [0, 0, 975, 610]); 32 | 33 | svg.append("g") 34 | .attr("transform", "translate(580,20)") 35 | .append(() => legend({color, title: data.title, width: 320, tickFormat:d3.format(".2%")})); 36 | 37 | svg.append("g") 38 | .selectAll("path") 39 | .data(topojson.feature(us, us.objects.counties).features) 40 | .join("path") 41 | .attr("fill", d => color(data.get(d.id))) 42 | .attr("d", path) 43 | .append("title") 44 | .text(d => `${d.properties.name}, ${states.get(d.id.slice(0, 2)).name} 45 | ${format(data.get(d.id))}`); 46 | 47 | svg.append("path") 48 | .datum(topojson.mesh(us, us.objects.states, (a, b) => a !== b)) 49 | .attr("fill", "none") 50 | .attr("stroke", "white") 51 | .attr("stroke-linejoin", "round") 52 | .attr("d", path); 53 | 54 | return svg.node(); 55 | } 56 | ); 57 | main.variable(observer("rawData")).define("rawData", ["variable"], async function(variable) 58 | { 59 | const raw = await fetch( 60 | `https://api.census.gov/data/2018/acs/acs5?get=B01001_001E,${variable.name}&for=county:*` 61 | ).then((r) => r.json()); 62 | 63 | const header = raw[0]; 64 | 65 | return raw.slice(1).map(d => { 66 | const ret = {}; 67 | for(const i in header) { 68 | ret[header[i]] = d[i]; 69 | } 70 | return ret; 71 | }) 72 | } 73 | ); 74 | main.variable(observer("valueOf")).define("valueOf", ["variable"], function(variable){return( 75 | function valueOf(d) { 76 | return d[variable.name] / d.B01001_001E; 77 | } 78 | )}); 79 | main.variable(observer("data")).define("data", ["rawData","valueOf"], function(rawData,valueOf){return( 80 | new Map(rawData.map(d=>[`${d.state}${d.county}`, valueOf(d)])) 81 | )}); 82 | main.variable(observer("B01001")).define("B01001", function(){return( 83 | fetch("https://api.census.gov/data/2018/acs/acs5/groups/B01001/").then(r=>r.json()) 84 | )}); 85 | main.variable(observer("vars")).define("vars", ["d3","B01001"], function(d3,B01001){return( 86 | d3.rollup 87 | (Object.entries(B01001.variables).map(d=> ({ 88 | name: d[0], 89 | ...d[1] 90 | })),//.sort((a,b)=>d3.ascending(a[0], b[0]))), 91 | v => { 92 | return new Map([ 93 | ["estimate_annotation", v.find(d=>d.name.endsWith("EA"))], 94 | ["moe_annotation", v.find(d=>d.name.endsWith("MA"))], 95 | ["moe", v.find(d=>d.name.endsWith("M"))], 96 | ["estimate", v.find(d=>d.name.endsWith("E"))], 97 | ]) 98 | }, 99 | d => d.name.substring(0, "B01001_XXX".length)) 100 | )}); 101 | main.variable(observer("variable")).define("variable", ["selVar"], function(selVar){return( 102 | fetch( 103 | `https://api.census.gov/data/2018/acs/acs5/variables/${selVar.get("estimate").name}.json` 104 | ).then((r) => r.json()) 105 | )}); 106 | main.variable(observer("color")).define("color", ["d3","data"], function(d3,data){return( 107 | d3.scaleQuantize(d3.extent(data.values()), d3.schemeBlues[9]) 108 | )}); 109 | main.variable(observer("path")).define("path", ["d3"], function(d3){return( 110 | d3.geoPath() 111 | )}); 112 | main.variable(observer("format")).define("format", function(){return( 113 | d => `${d}%` 114 | )}); 115 | main.variable(observer("states")).define("states", ["us"], function(us){return( 116 | new Map(us.objects.states.geometries.map(d => [d.id, d.properties])) 117 | )}); 118 | main.variable(observer("us")).define("us", ["FileAttachment"], function(FileAttachment){return( 119 | FileAttachment("counties-albers-10m.json").json() 120 | )}); 121 | main.variable(observer("topojson")).define("topojson", ["require"], function(require){return( 122 | require("topojson-client@3") 123 | )}); 124 | const child1 = runtime.module(define1); 125 | main.import("legend", "legend", child1); 126 | main.variable(observer()).define(["md"], function(md){return( 127 | md`## Source Code 128 | 129 | This is the original source code for this notebook, the \`census-api.ojs\` file.` 130 | )}); 131 | main.variable(observer()).define(["md","FileAttachment"], async function(md,FileAttachment){return( 132 | md`~~~javascript 133 | ${(await FileAttachment("source_ojs").text()).replace(/~~~/g, "```")} 134 | ~~~` 135 | )}); 136 | main.variable(observer()).define(function(){return( 137 | document.title = "Census API Example / Dataflow" 138 | )}); 139 | return main; 140 | } -------------------------------------------------------------------------------- /docs/site.ojs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env dataflow run --allow-file-attachments -p 9002 2 | /* 3 | FileAttachments: 4 | package.json: ../package.json 5 | quickstart: ./quickstart.md 6 | stdlib: ./stdlib.md 7 | file-attachments: ./file-attachments.md 8 | importing: ./importing.md 9 | secrets: ./secrets.md 10 | production: ./production.md 11 | compiling: ./compiling.md 12 | reference: ./reference.md 13 | shebang: ./shebang.md 14 | */ 15 | nav = html`