├── datasets ├── github │ ├── APIData.url │ ├── Gists.url │ ├── Meta.url │ ├── Emojis.url │ ├── Events.url │ └── index.json ├── reddit │ ├── Reddit.url │ └── index.json ├── blockchain │ ├── LatestBlock.url │ ├── UnconfirmedTransactions.url │ └── index.json ├── spotify │ ├── Profile.url │ ├── Artist.url │ ├── Album.url │ ├── Track.url │ ├── Playlist.url │ └── index.json ├── magic-the-gathering │ ├── LEAExtras.url │ └── index.json ├── astronauts-in-space │ ├── AstronautsInSpace.url │ ├── ISSCurrentLocation.url │ └── index.json ├── consumer-complaints │ ├── ConsumerComplaints.url │ └── index.json ├── earthquakes │ ├── Earthquakes.url │ └── index.json └── worldbank-gdp │ ├── GDP │ ├── China.url │ └── USA.url │ └── index.json ├── CONTRIBUTING.md ├── .gitignore ├── .vscode ├── settings.json └── launch.json ├── src ├── common.ts ├── readme.ts └── generate.ts ├── package.json └── README.md /datasets/github/APIData.url: -------------------------------------------------------------------------------- 1 | https://api.github.com/ -------------------------------------------------------------------------------- /datasets/github/Gists.url: -------------------------------------------------------------------------------- 1 | https://api.github.com/gists -------------------------------------------------------------------------------- /datasets/github/Meta.url: -------------------------------------------------------------------------------- 1 | https://api.github.com/meta -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ```bash 2 | $ brew install hub 3 | ``` 4 | -------------------------------------------------------------------------------- /datasets/github/Emojis.url: -------------------------------------------------------------------------------- 1 | https://api.github.com/emojis -------------------------------------------------------------------------------- /datasets/github/Events.url: -------------------------------------------------------------------------------- 1 | https://api.github.com/events -------------------------------------------------------------------------------- /datasets/reddit/Reddit.url: -------------------------------------------------------------------------------- 1 | https://www.reddit.com/r/all.json -------------------------------------------------------------------------------- /datasets/blockchain/LatestBlock.url: -------------------------------------------------------------------------------- 1 | https://blockchain.info/latestblock -------------------------------------------------------------------------------- /datasets/spotify/Profile.url: -------------------------------------------------------------------------------- 1 | https://api.spotify.com/v1/users/12139437815 -------------------------------------------------------------------------------- /datasets/magic-the-gathering/LEAExtras.url: -------------------------------------------------------------------------------- 1 | https://mtgjson.com/json/LEA-x.json -------------------------------------------------------------------------------- /datasets/astronauts-in-space/AstronautsInSpace.url: -------------------------------------------------------------------------------- 1 | http://api.open-notify.org/astros.json -------------------------------------------------------------------------------- /datasets/spotify/Artist.url: -------------------------------------------------------------------------------- 1 | https://api.spotify.com/v1/artists/0TnOYISbd1XYRBk9myaseg -------------------------------------------------------------------------------- /datasets/astronauts-in-space/ISSCurrentLocation.url: -------------------------------------------------------------------------------- 1 | http://api.open-notify.org/iss-now.json -------------------------------------------------------------------------------- /datasets/spotify/Album.url: -------------------------------------------------------------------------------- 1 | https://api.spotify.com/v1/albums/4aawyAB9vmqN3uQ7FjRGTy?market=ES -------------------------------------------------------------------------------- /datasets/spotify/Track.url: -------------------------------------------------------------------------------- 1 | https://api.spotify.com/v1/tracks/11dFghVXANMlKmJXsNCbNl?market=ES -------------------------------------------------------------------------------- /datasets/consumer-complaints/ConsumerComplaints.url: -------------------------------------------------------------------------------- 1 | http://data.consumerfinance.gov/api/views.json 2 | -------------------------------------------------------------------------------- /datasets/blockchain/UnconfirmedTransactions.url: -------------------------------------------------------------------------------- 1 | https://blockchain.info/unconfirmed-transactions?format=json -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /datasets-cache 3 | /src/dataset.ts 4 | /repos 5 | /setup-env 6 | .envrc 7 | -------------------------------------------------------------------------------- /datasets/earthquakes/Earthquakes.url: -------------------------------------------------------------------------------- 1 | https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson -------------------------------------------------------------------------------- /datasets/spotify/Playlist.url: -------------------------------------------------------------------------------- 1 | https://api.spotify.com/v1/users/jmperezperez/playlists/3cEYpjA9oz9GiPac4AsH4n?market=ES -------------------------------------------------------------------------------- /datasets/worldbank-gdp/GDP/China.url: -------------------------------------------------------------------------------- 1 | http://api.worldbank.org/countries/CHN/indicators/NY.GDP.MKTP.CD?per_page=5000&format=json -------------------------------------------------------------------------------- /datasets/worldbank-gdp/GDP/USA.url: -------------------------------------------------------------------------------- 1 | http://api.worldbank.org/countries/USA/indicators/NY.GDP.MKTP.CD?per_page=5000&format=json -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "search.exclude": { 4 | "**/.git": true, 5 | "**/node_modules": true 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /datasets/reddit/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Reddit", 3 | "description": "Reddit post list API", 4 | "url": "https://reddit.com/", 5 | "category": "Media" 6 | } 7 | -------------------------------------------------------------------------------- /datasets/worldbank-gdp/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Worldbank GDP", 3 | "description": "Worldbank GDP data", 4 | "url": "http://api.worldbank.org//", 5 | "category": "Economics" 6 | } 7 | -------------------------------------------------------------------------------- /datasets/magic-the-gathering/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Magic: The Gathering", 3 | "description": "MTG LEA Set + Extras", 4 | "url": "http://magic.wizards.com/", 5 | "category": "Gaming" 6 | } 7 | -------------------------------------------------------------------------------- /datasets/blockchain/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Blockchain Bitcoin APIs", 3 | "description": "Info about digital currencies", 4 | "url": "https://blockchain.info", 5 | "category": "Blockchain" 6 | } 7 | -------------------------------------------------------------------------------- /datasets/github/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "GitHub API", 3 | "description": "Data about gists, events, emoji, and more.", 4 | "url": "https://api.github.com/", 5 | "category": "Technology" 6 | } 7 | -------------------------------------------------------------------------------- /datasets/earthquakes/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "USGS Earthquake Data", 3 | "description": "Live earthquakes happening NOW!", 4 | "url": "https://earthquake.usgs.gov/", 5 | "category": "Science" 6 | } 7 | -------------------------------------------------------------------------------- /datasets/consumer-complaints/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Consumer Complaint Database", 3 | "description": "Complaints about financial products and services.", 4 | "url": "https://catalog.data.gov/dataset/consumer-complaint-database", 5 | "category": "Finance" 6 | } -------------------------------------------------------------------------------- /datasets/astronauts-in-space/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Open APIs From Space", 3 | "description": 4 | "Open Notify is an open source project to provide a simple programming interface for some of NASA’s awesome data.", 5 | "url": "http://open-notify.org/", 6 | "category": "Space" 7 | } 8 | -------------------------------------------------------------------------------- /datasets/spotify/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Spotify Web API", 3 | "description": 4 | "Lets your applications fetch data from the Spotify music catalog and manage user’s playlists and saved music.", 5 | "url": "https://developer.spotify.com/web-api/", 6 | "category": "Media", 7 | "oauth": "SPOTIFY_OAUTH_TOKEN" 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible Node.js debug attributes. 3 | "version": "0.2.0", 4 | "configurations": [ 5 | { 6 | "name": "debug", 7 | "type": "node", 8 | "request": "launch", 9 | "program": "${workspaceRoot}/node_modules/ts-node/dist/_bin.js", 10 | "runtimeArgs": ["--nolazy"], 11 | "args": ["script/build.ts"], 12 | "cwd": "${workspaceRoot}", 13 | "protocol": "inspector" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /src/common.ts: -------------------------------------------------------------------------------- 1 | import { Dataset } from "./dataset"; 2 | import { TargetLanguage } from "quicktype"; 3 | 4 | export type DatasetMeta = { 5 | slug: string; 6 | dataDir: string; 7 | dataset: Dataset; 8 | repoDir: string; 9 | }; 10 | 11 | export { Dataset } from "./dataset"; 12 | 13 | export function repoFromSlug(slug: string) { 14 | return `typeguard/typed-${slug}`; 15 | } 16 | 17 | export function languageShortname(language: TargetLanguage) { 18 | return language.names.sort((x, y) => y.length - x.length)[0]; 19 | } 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "awesome-typed-data", 3 | "version": "1.0.0", 4 | "description": "Awesome typed datasets", 5 | "main": "index.js", 6 | "scripts": { 7 | "prepare": "npm upgrade quicktype && npm run quicktype", 8 | "quicktype": "quicktype datasets/*/index.json -o src/dataset.ts", 9 | "generate": "npm run prepare && ts-node src/generate.ts" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/typeguard/awesome-typed-datasets.git" 14 | }, 15 | "author": "", 16 | "license": "ISC", 17 | "bugs": { 18 | "url": "https://github.com/typeguard/awesome-typed-datasets/issues" 19 | }, 20 | "homepage": "https://github.com/typeguard/awesome-typed-datasets#readme", 21 | "devDependencies": { 22 | "@types/lodash": "^4.14.100", 23 | "@types/shelljs": "^0.7.8", 24 | "lodash": "^4.17.5", 25 | "quicktype": "^15.0.108", 26 | "shelljs": "^0.8.1", 27 | "ts-node": "^4.1.0", 28 | "typescript": "^2.7.1" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Awesome Typed Datasets [![Awesome](https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/sindresorhus/awesome) 2 | 3 | These are public JSON datasets that have been strongly 4 | typed with [quicktype](https://github.com/quicktype/quicktype). 5 | Each is a repo with code in Go, Rust, C++, Objective-C, Java, TypeScript, JavaScript, Flow, Swift, Kotlin (beta), Elm, JSON Schema, Ruby, Dart, Python, and C# for 6 | reading and writing the JSON produced by these APIs. 7 | 8 | 9 | ## Blockchain 10 | 11 | * [Blockchain Bitcoin APIs](https://github.com/typeguard/typed-blockchain) (blockchain.info) 12 | 13 | ## Economics 14 | 15 | * [Worldbank GDP](https://github.com/typeguard/typed-worldbank-gdp) (api.worldbank.org) 16 | 17 | ## Finance 18 | 19 | * [Consumer Complaint Database](https://github.com/typeguard/typed-consumer-complaints) (catalog.data.gov/dataset/consumer-complaint-database) 20 | 21 | ## Gaming 22 | 23 | * [Magic: The Gathering](https://github.com/typeguard/typed-magic-the-gathering) (magic.wizards.com) 24 | 25 | ## Media 26 | 27 | * [Reddit](https://github.com/typeguard/typed-reddit) (reddit.com) 28 | * [Spotify Web API](https://github.com/typeguard/typed-spotify) (developer.spotify.com/web-api) 29 | 30 | ## Science 31 | 32 | * [USGS Earthquake Data](https://github.com/typeguard/typed-earthquakes) (earthquake.usgs.gov) 33 | 34 | ## Space 35 | 36 | * [Open APIs From Space](https://github.com/typeguard/typed-astronauts-in-space) (open-notify.org) 37 | 38 | ## Technology 39 | 40 | * [GitHub API](https://github.com/typeguard/typed-github) (api.github.com) 41 | 42 | ## Contributing 43 | If you want to contribute, please read the [contribution guidelines](CONTRIBUTING.md). 44 | 45 | ## License 46 | [![CC0](http://mirrors.creativecommons.org/presskit/buttons/88x31/svg/cc-zero.svg)](https://creativecommons.org/publicdomain/zero/1.0/) 47 | -------------------------------------------------------------------------------- /src/readme.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs"; 2 | import * as path from "path"; 3 | 4 | import { defaultTargetLanguages } from "quicktype/dist/quicktype-core"; 5 | 6 | import { 7 | Dataset, 8 | DatasetMeta, 9 | repoFromSlug, 10 | languageShortname 11 | } from "./common"; 12 | 13 | import * as lo from "lodash"; 14 | import { find } from "shelljs"; 15 | 16 | export function dataset(data: DatasetMeta): string { 17 | function* generate() { 18 | yield `# ${data.dataset.name} – Typed JSON API`; 19 | yield ``; 20 | yield `> ${data.dataset.description}`; 21 | yield ``; 22 | 23 | yield `## APIs`; 24 | yield ``; 25 | for (const urlFile of find(`${data.dataDir}/**/*.url`)) { 26 | const name = path.basename(urlFile).replace(".url", ""); 27 | const url = fs.readFileSync(urlFile, "utf8").trim(); 28 | 29 | // TODO support multiple samples 30 | 31 | yield `* \`${name}\`: ${url}`; 32 | } 33 | yield ``; 34 | 35 | yield `## Libraries`; 36 | yield ``; 37 | for (const language of defaultTargetLanguages) { 38 | yield `* [${language.displayName}](${languageShortname(language)})`; 39 | } 40 | yield ``; 41 | 42 | yield `# Contributing`; 43 | yield* [ 44 | ``, 45 | `This repo is generated with [quicktype](https://github.com/quicktype/quicktype) from data in [typeguard/awesome-typed-datasets](https://github.com/typeguard/awesome-typed-datasets).`, 46 | `To contribute, please visit those repos.`, 47 | `` 48 | ]; 49 | } 50 | 51 | return Array.from(generate()).join("\n"); 52 | } 53 | 54 | export function main(data: DatasetMeta[]): string { 55 | const categories = lo.groupBy(data, d => d.dataset.category); 56 | 57 | function* categoryList(name: string, category: DatasetMeta[]) { 58 | yield ``; 59 | yield `## ${name}`; 60 | yield ``; 61 | 62 | for (const meta of category) { 63 | let simpleUrl = meta.dataset.url.split("//")[1]; 64 | if (simpleUrl.endsWith("/")) { 65 | simpleUrl = simpleUrl.substr(0, simpleUrl.length - 1); 66 | } 67 | 68 | yield `* [${meta.dataset.name}](https://github.com/${repoFromSlug( 69 | meta.slug 70 | )}) (${simpleUrl})`; 71 | } 72 | } 73 | 74 | function* generate() { 75 | yield `# Awesome Typed Datasets [![Awesome](https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/sindresorhus/awesome)`; 76 | 77 | const displayNames = defaultTargetLanguages 78 | .map(l => l.displayName) 79 | .filter(d => d !== "Simple Types"); 80 | const nameList = 81 | displayNames.slice(1).join(", ") + ", and " + displayNames[0]; 82 | 83 | yield* [ 84 | ``, 85 | `These are public JSON datasets that have been strongly`, 86 | `typed with [quicktype](https://github.com/quicktype/quicktype).`, 87 | `Each is a repo with code in ${nameList} for`, 88 | `reading and writing the JSON produced by these APIs.`, 89 | `` 90 | ]; 91 | 92 | for (const name of Object.keys(categories).sort()) { 93 | yield* categoryList(name, categories[name]); 94 | } 95 | 96 | yield* [ 97 | ``, 98 | `## Contributing`, 99 | `If you want to contribute, please read the [contribution guidelines](CONTRIBUTING.md).`, 100 | ``, 101 | `## License`, 102 | `[![CC0](http://mirrors.creativecommons.org/presskit/buttons/88x31/svg/cc-zero.svg)](https://creativecommons.org/publicdomain/zero/1.0/)`, 103 | `` 104 | ]; 105 | } 106 | 107 | return Array.from(generate()).join("\n"); 108 | } 109 | -------------------------------------------------------------------------------- /src/generate.ts: -------------------------------------------------------------------------------- 1 | import { pwd, cd, exec, mv, cp, rm, cat, find, mkdir, chmod } from "shelljs"; 2 | 3 | import * as path from "path"; 4 | import * as fs from "fs"; 5 | import * as lo from "lodash"; 6 | 7 | import { Dataset, Convert } from "./dataset"; 8 | import { repoFromSlug, DatasetMeta, languageShortname } from "./common"; 9 | import * as readme from "./readme"; 10 | 11 | import { defaultTargetLanguages } from "quicktype/dist/quicktype-core"; 12 | 13 | const QUICKTYPE_BIN = path.resolve("node_modules/.bin/quicktype"); 14 | const DATASET_CACHE = "datasets-cache"; 15 | 16 | const QUICKTYPE_VERSION = JSON.parse( 17 | fs.readFileSync("package-lock.json", "utf8") 18 | ).dependencies.quicktype.version; 19 | 20 | function execho(command: string) { 21 | console.error(command); 22 | return exec(command); 23 | } 24 | 25 | function hub(command: string) { 26 | return execho(`hub ${command}`); 27 | } 28 | 29 | function quicktype(command: string) { 30 | console.error(`quicktype ${command}`); 31 | return exec(`${QUICKTYPE_BIN} ${command}`); 32 | } 33 | 34 | function remoteFromSlug(slug: string) { 35 | return `git@github.com:${repoFromSlug(slug)}`; 36 | } 37 | 38 | function precacheDataDirectory({ dataDir, dataset }: DatasetMeta) { 39 | mkdir("-p", DATASET_CACHE); 40 | const target = path.join(DATASET_CACHE, path.basename(dataDir)); 41 | rm("-rf", target); 42 | cp("-r", dataDir, target); 43 | rm(path.join(target, "index.json")); 44 | // Pre-download URLs 45 | for (const urlFile of find(`${target}/**/*.url`)) { 46 | const url = fs.readFileSync(urlFile, "utf8").trim(); 47 | const jsonFile = urlFile.replace(".url", ".json"); 48 | mkdir("-p", path.dirname(urlFile)); 49 | 50 | let authenticate = ""; 51 | if (dataset.oauth !== undefined) { 52 | const token = process.env[dataset.oauth]; 53 | authenticate = `-H "Authorization: Bearer ${token}"`; 54 | } 55 | exec( 56 | `curl -X GET "${url}" -H "Accept: application/json" -o "${jsonFile}" ${authenticate}` 57 | ); 58 | 59 | rm(urlFile); 60 | } 61 | return target; 62 | } 63 | 64 | function within(dir: string, work: () => void) { 65 | const cwd = pwd(); 66 | cd(dir); 67 | work(); 68 | cd(cwd); 69 | } 70 | 71 | function cloneOrCreateRepo({ repoDir, slug, dataset }: DatasetMeta) { 72 | const cloneResult = hub(`clone ${remoteFromSlug(slug)} ${repoDir}`); 73 | if (cloneResult.code === 0) return; 74 | 75 | mkdir("-p", repoDir); 76 | within(repoDir, () => { 77 | hub("init"); 78 | hub(`commit --allow-empty -m create`); 79 | hub( 80 | `create ${repoFromSlug(slug)} -d "${dataset.name}" -h "${dataset.url}"` 81 | ); 82 | hub("push --set-upstream origin master"); 83 | }); 84 | } 85 | 86 | function* getDatasets() { 87 | for (const index of find("datasets/**/index.json")) { 88 | const dataset = Convert.toDataset(fs.readFileSync(index, "utf8")); 89 | const dataDir = path.dirname(index); 90 | const slug = path.basename(dataDir); 91 | yield { 92 | slug, 93 | repoDir: `repos/types-${slug}`, 94 | dataDir, 95 | dataset 96 | }; 97 | } 98 | } 99 | 100 | function main(slugs: string[]) { 101 | rm("-rf", "repos"); 102 | mkdir("repos"); 103 | 104 | // TODO make this work as an iterator 105 | let datasets = Array.from(getDatasets()); 106 | fs.writeFileSync("README.md", readme.main(datasets)); 107 | 108 | if (slugs.length > 0) { 109 | datasets = datasets.filter(m => lo.includes(slugs, m.slug)); 110 | } 111 | 112 | for (const meta of datasets) { 113 | const scriptFile = path.join(meta.repoDir, "quicktype.sh"); 114 | const cachedDataDir = precacheDataDirectory(meta); 115 | 116 | cloneOrCreateRepo(meta); 117 | 118 | const targetDataDir = path.join(meta.repoDir, "data"); 119 | rm("-rf", targetDataDir); 120 | cp("-r", meta.dataDir, targetDataDir); 121 | rm(path.join(targetDataDir, "index.json")); 122 | 123 | let script = ["#!/bin/bash", ""]; 124 | 125 | for (const language of defaultTargetLanguages) { 126 | const langName = languageShortname(language); 127 | const languageDir = path.join(meta.repoDir, langName); 128 | const mainFile = path.join( 129 | languageDir, 130 | `${meta.slug}.${language.extension}` 131 | ); 132 | 133 | rm("-rf", languageDir); 134 | mkdir(languageDir); 135 | 136 | quicktype(`${cachedDataDir} -o ${mainFile}`); 137 | 138 | script.push( 139 | `quicktype data -o ${path.join(langName, path.basename(mainFile))}` 140 | ); 141 | } 142 | 143 | fs.writeFileSync(scriptFile, script.join("\n")); 144 | chmod("+x", scriptFile); 145 | 146 | fs.writeFileSync( 147 | path.join(meta.repoDir, "README.md"), 148 | readme.dataset(meta) 149 | ); 150 | 151 | within(meta.repoDir, () => { 152 | const hasChanges = exec("git status --porcelain").stdout.length > 0; 153 | if (hasChanges) { 154 | hub("add ."); 155 | hub(`commit -am "update (quicktype ${QUICKTYPE_VERSION})"`); 156 | hub("push"); 157 | } 158 | }); 159 | } 160 | } 161 | 162 | main(process.argv.slice(2)); 163 | --------------------------------------------------------------------------------