├── .gitattributes ├── test ├── source │ ├── basic_json_empty │ │ └── empty.json │ ├── basic_json_error │ │ └── error.json │ ├── double_depth │ │ ├── others │ │ │ ├── outcasts │ │ │ │ └── cats.csv │ │ │ └── malamutes.json │ │ └── corgis.json │ ├── advanced_aml │ │ └── text.aml │ ├── single_depth │ │ ├── others │ │ │ ├── malamutes.json │ │ │ └── corgis.csv │ │ └── corgis.json │ ├── basic_csv │ │ └── corgis.csv │ ├── basic_tsv │ │ └── corgis.tsv │ ├── non_match_file │ │ ├── corgis.txt │ │ └── corgis.yaml │ ├── basic_aml │ │ └── corgis.aml │ ├── basic_yaml │ │ └── corgis.yaml │ ├── basic_yml │ │ └── corgis.yml │ ├── duplicate_keys │ │ ├── corgis.yaml │ │ └── corgis.json │ ├── basic_yaml_error │ │ └── corgis.yaml │ ├── basic_json │ │ └── corgis.json │ ├── basic_cjs │ │ └── corgis.cjs │ ├── basic_js │ │ └── corgis.js │ ├── basic_mjs │ │ └── corgis.mjs │ ├── sync_cjs │ │ └── corgis.cjs │ ├── sync_js │ │ └── corgis.js │ ├── sync_mjs │ │ └── corgis.mjs │ ├── async_js │ │ └── corgis.js │ ├── async_cjs │ │ └── corgis.cjs │ └── async_mjs │ │ └── corgis.mjs └── test.ts ├── .prettierignore ├── source ├── lib.d.ts └── index.ts ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── tsconfig.json ├── LICENSE ├── package.json ├── .gitignore ├── CHANGELOG.md └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /test/source/basic_json_empty/empty.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | package-lock.json 3 | test/source 4 | -------------------------------------------------------------------------------- /test/source/basic_json_error/error.json: -------------------------------------------------------------------------------- 1 | { 2 | "hello": 5, 3 | } 4 | -------------------------------------------------------------------------------- /source/lib.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'archieml' { 2 | function load(raw: string): unknown; 3 | } 4 | -------------------------------------------------------------------------------- /test/source/double_depth/others/outcasts/cats.csv: -------------------------------------------------------------------------------- 1 | name,instagram 2 | Cream Brother,https://instagram.com/creambrother_thecat/ 3 | -------------------------------------------------------------------------------- /test/source/advanced_aml/text.aml: -------------------------------------------------------------------------------- 1 | [+prose] 2 | This is much more complicated than normal ArchieML. 3 | 4 | But also, much cooler. 5 | [] 6 | -------------------------------------------------------------------------------- /test/source/double_depth/others/malamutes.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Utah", 3 | "instagram": "https://instagram.com/malamutecalledutah/" 4 | } 5 | -------------------------------------------------------------------------------- /test/source/single_depth/others/malamutes.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Utah", 3 | "instagram": "https://instagram.com/malamutecalledutah/" 4 | } 5 | -------------------------------------------------------------------------------- /test/source/basic_csv/corgis.csv: -------------------------------------------------------------------------------- 1 | name,instagram 2 | Trinket,https://instagram.com/trinketbaby/ 3 | Ernie,https://instagram.com/sirbunnybottom/ 4 | Tibby,https://instagram.com/tibbythecorgi/ 5 | Pudge,https://instagram.com/pudgethecorgi/ 6 | -------------------------------------------------------------------------------- /test/source/basic_tsv/corgis.tsv: -------------------------------------------------------------------------------- 1 | name instagram 2 | Trinket https://instagram.com/trinketbaby/ 3 | Ernie https://instagram.com/sirbunnybottom/ 4 | Tibby https://instagram.com/tibbythecorgi/ 5 | Pudge https://instagram.com/pudgethecorgi/ 6 | -------------------------------------------------------------------------------- /test/source/non_match_file/corgis.txt: -------------------------------------------------------------------------------- 1 | name,instagram 2 | Trinket,https://instagram.com/trinketbaby/ 3 | Ernie,https://instagram.com/sirbunnybottom/ 4 | Tibby,https://instagram.com/tibbythecorgi/ 5 | Pudge,https://instagram.com/pudgethecorgi/ 6 | -------------------------------------------------------------------------------- /test/source/single_depth/others/corgis.csv: -------------------------------------------------------------------------------- 1 | name,instagram 2 | Trinket,https://instagram.com/trinketbaby/ 3 | Ernie,https://instagram.com/sirbunnybottom/ 4 | Tibby,https://instagram.com/tibbythecorgi/ 5 | Pudge,https://instagram.com/pudgethecorgi/ 6 | -------------------------------------------------------------------------------- /test/source/basic_aml/corgis.aml: -------------------------------------------------------------------------------- 1 | [accounts] 2 | name: Trinket 3 | url: https://instagram.com/trinketbaby/ 4 | name: Ernie 5 | url: https://instagram.com/sirbunnybottom/ 6 | name: Tibby 7 | url: https://instagram.com/tibbythecorgi/ 8 | name: Pudge 9 | url: https://instagram.com/pudgethecorgi/ 10 | [] 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | time: "13:00" 8 | open-pull-requests-limit: 10 9 | ignore: 10 | - dependency-name: "@types/node" 11 | versions: 12 | - 14.14.41 13 | - 15.0.0 14 | -------------------------------------------------------------------------------- /test/source/basic_yaml/corgis.yaml: -------------------------------------------------------------------------------- 1 | - name: Trinket 2 | instagram: https://instagram.com/trinketbaby/ 3 | - name: Ernie 4 | instagram: https://instagram.com/sirbunnybottom/ 5 | - name: Tibby 6 | instagram: https://instagram.com/tibbythecorgi/ 7 | - name: Pudge 8 | instagram: https://instagram.com/pudgethecorgi/ 9 | -------------------------------------------------------------------------------- /test/source/basic_yml/corgis.yml: -------------------------------------------------------------------------------- 1 | - name: Trinket 2 | instagram: https://instagram.com/trinketbaby/ 3 | - name: Ernie 4 | instagram: https://instagram.com/sirbunnybottom/ 5 | - name: Tibby 6 | instagram: https://instagram.com/tibbythecorgi/ 7 | - name: Pudge 8 | instagram: https://instagram.com/pudgethecorgi/ 9 | -------------------------------------------------------------------------------- /test/source/duplicate_keys/corgis.yaml: -------------------------------------------------------------------------------- 1 | - name: Trinket 2 | instagram: https://instagram.com/trinketbaby/ 3 | - name: Ernie 4 | instagram: https://instagram.com/sirbunnybottom/ 5 | - name: Tibby 6 | instagram: https://instagram.com/tibbythecorgi/ 7 | - name: Pudge 8 | instagram: https://instagram.com/pudgethecorgi/ 9 | -------------------------------------------------------------------------------- /test/source/non_match_file/corgis.yaml: -------------------------------------------------------------------------------- 1 | - name: Trinket 2 | instagram: https://instagram.com/trinketbaby/ 3 | - name: Ernie 4 | instagram: https://instagram.com/sirbunnybottom/ 5 | - name: Tibby 6 | instagram: https://instagram.com/tibbythecorgi/ 7 | - name: Pudge 8 | instagram: https://instagram.com/pudgethecorgi/ 9 | -------------------------------------------------------------------------------- /test/source/basic_yaml_error/corgis.yaml: -------------------------------------------------------------------------------- 1 | - name: Trinket 2 | instagram: https://instagram.com/trinketbaby/ 3 | - name: Ernie 4 | instagram: https://instagram.com/sirbunnybottom/ 5 | - name: Tibby 6 | instagram: https://instagram.com/tibbythecorgi/ 7 | - name: Pudge 8 | instagram: https://instagram.com/pudgethecorgi/ 9 | error 10 | -------------------------------------------------------------------------------- /test/source/basic_json/corgis.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Trinket", 4 | "instagram": "https://instagram.com/trinketbaby/" 5 | }, 6 | { 7 | "name": "Ernie", 8 | "instagram": "https://instagram.com/sirbunnybottom/" 9 | }, 10 | { 11 | "name": "Tibby", 12 | "instagram": "https://instagram.com/tibbythecorgi/" 13 | }, 14 | { 15 | "name": "Pudge", 16 | "instagram": "https://instagram.com/pudgethecorgi/" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /test/source/basic_cjs/corgis.cjs: -------------------------------------------------------------------------------- 1 | module.exports = [ 2 | { 3 | name: 'Trinket', 4 | instagram: 'https://instagram.com/trinketbaby/', 5 | }, 6 | { 7 | name: 'Ernie', 8 | instagram: 'https://instagram.com/sirbunnybottom/', 9 | }, 10 | { 11 | name: 'Tibby', 12 | instagram: 'https://instagram.com/tibbythecorgi/', 13 | }, 14 | { 15 | name: 'Pudge', 16 | instagram: 'https://instagram.com/pudgethecorgi/', 17 | }, 18 | ]; 19 | -------------------------------------------------------------------------------- /test/source/basic_js/corgis.js: -------------------------------------------------------------------------------- 1 | export default [ 2 | { 3 | name: 'Trinket', 4 | instagram: 'https://instagram.com/trinketbaby/', 5 | }, 6 | { 7 | name: 'Ernie', 8 | instagram: 'https://instagram.com/sirbunnybottom/', 9 | }, 10 | { 11 | name: 'Tibby', 12 | instagram: 'https://instagram.com/tibbythecorgi/', 13 | }, 14 | { 15 | name: 'Pudge', 16 | instagram: 'https://instagram.com/pudgethecorgi/', 17 | }, 18 | ]; 19 | -------------------------------------------------------------------------------- /test/source/basic_mjs/corgis.mjs: -------------------------------------------------------------------------------- 1 | export default [ 2 | { 3 | name: 'Trinket', 4 | instagram: 'https://instagram.com/trinketbaby/', 5 | }, 6 | { 7 | name: 'Ernie', 8 | instagram: 'https://instagram.com/sirbunnybottom/', 9 | }, 10 | { 11 | name: 'Tibby', 12 | instagram: 'https://instagram.com/tibbythecorgi/', 13 | }, 14 | { 15 | name: 'Pudge', 16 | instagram: 'https://instagram.com/pudgethecorgi/', 17 | }, 18 | ]; 19 | -------------------------------------------------------------------------------- /test/source/double_depth/corgis.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Trinket", 4 | "instagram": "https://instagram.com/trinketbaby/" 5 | }, 6 | { 7 | "name": "Ernie", 8 | "instagram": "https://instagram.com/sirbunnybottom/" 9 | }, 10 | { 11 | "name": "Tibby", 12 | "instagram": "https://instagram.com/tibbythecorgi/" 13 | }, 14 | { 15 | "name": "Pudge", 16 | "instagram": "https://instagram.com/pudgethecorgi/" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /test/source/duplicate_keys/corgis.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Trinket", 4 | "instagram": "https://instagram.com/trinketbaby/" 5 | }, 6 | { 7 | "name": "Ernie", 8 | "instagram": "https://instagram.com/sirbunnybottom/" 9 | }, 10 | { 11 | "name": "Tibby", 12 | "instagram": "https://instagram.com/tibbythecorgi/" 13 | }, 14 | { 15 | "name": "Pudge", 16 | "instagram": "https://instagram.com/pudgethecorgi/" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /test/source/single_depth/corgis.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Trinket", 4 | "instagram": "https://instagram.com/trinketbaby/" 5 | }, 6 | { 7 | "name": "Ernie", 8 | "instagram": "https://instagram.com/sirbunnybottom/" 9 | }, 10 | { 11 | "name": "Tibby", 12 | "instagram": "https://instagram.com/tibbythecorgi/" 13 | }, 14 | { 15 | "name": "Pudge", 16 | "instagram": "https://instagram.com/pudgethecorgi/" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /test/source/sync_cjs/corgis.cjs: -------------------------------------------------------------------------------- 1 | module.exports = () => [ 2 | { 3 | name: 'Trinket', 4 | instagram: 'https://instagram.com/trinketbaby/', 5 | }, 6 | { 7 | name: 'Ernie', 8 | instagram: 'https://instagram.com/sirbunnybottom/', 9 | }, 10 | { 11 | name: 'Tibby', 12 | instagram: 'https://instagram.com/tibbythecorgi/', 13 | }, 14 | { 15 | name: 'Pudge', 16 | instagram: 'https://instagram.com/pudgethecorgi/', 17 | }, 18 | ]; 19 | -------------------------------------------------------------------------------- /test/source/sync_js/corgis.js: -------------------------------------------------------------------------------- 1 | export default () => [ 2 | { 3 | name: 'Trinket', 4 | instagram: 'https://instagram.com/trinketbaby/', 5 | }, 6 | { 7 | name: 'Ernie', 8 | instagram: 'https://instagram.com/sirbunnybottom/', 9 | }, 10 | { 11 | name: 'Tibby', 12 | instagram: 'https://instagram.com/tibbythecorgi/', 13 | }, 14 | { 15 | name: 'Pudge', 16 | instagram: 'https://instagram.com/pudgethecorgi/', 17 | }, 18 | ]; 19 | -------------------------------------------------------------------------------- /test/source/sync_mjs/corgis.mjs: -------------------------------------------------------------------------------- 1 | export default () => [ 2 | { 3 | name: 'Trinket', 4 | instagram: 'https://instagram.com/trinketbaby/', 5 | }, 6 | { 7 | name: 'Ernie', 8 | instagram: 'https://instagram.com/sirbunnybottom/', 9 | }, 10 | { 11 | name: 'Tibby', 12 | instagram: 'https://instagram.com/tibbythecorgi/', 13 | }, 14 | { 15 | name: 'Pudge', 16 | instagram: 'https://instagram.com/pudgethecorgi/', 17 | }, 18 | ]; 19 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node12/tsconfig.json", 3 | "compilerOptions": { 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "checkJs": true, 7 | "declaration": true, 8 | "module": "ES2020", 9 | "moduleResolution": "node", 10 | "outDir": "lib", 11 | "pretty": true, 12 | "noEmitOnError": true 13 | }, 14 | "files": ["source/index.ts", "source/lib.d.ts"], 15 | "ts-node": { 16 | "transpileOnly": true, 17 | "files": true 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test/source/async_js/corgis.js: -------------------------------------------------------------------------------- 1 | export default () => 2 | Promise.resolve([ 3 | { 4 | name: 'Trinket', 5 | instagram: 'https://instagram.com/trinketbaby/', 6 | }, 7 | { 8 | name: 'Ernie', 9 | instagram: 'https://instagram.com/sirbunnybottom/', 10 | }, 11 | { 12 | name: 'Tibby', 13 | instagram: 'https://instagram.com/tibbythecorgi/', 14 | }, 15 | { 16 | name: 'Pudge', 17 | instagram: 'https://instagram.com/pudgethecorgi/', 18 | }, 19 | ]); 20 | -------------------------------------------------------------------------------- /test/source/async_cjs/corgis.cjs: -------------------------------------------------------------------------------- 1 | module.exports = () => 2 | Promise.resolve([ 3 | { 4 | name: 'Trinket', 5 | instagram: 'https://instagram.com/trinketbaby/', 6 | }, 7 | { 8 | name: 'Ernie', 9 | instagram: 'https://instagram.com/sirbunnybottom/', 10 | }, 11 | { 12 | name: 'Tibby', 13 | instagram: 'https://instagram.com/tibbythecorgi/', 14 | }, 15 | { 16 | name: 'Pudge', 17 | instagram: 'https://instagram.com/pudgethecorgi/', 18 | }, 19 | ]); 20 | -------------------------------------------------------------------------------- /test/source/async_mjs/corgis.mjs: -------------------------------------------------------------------------------- 1 | export default () => 2 | Promise.resolve([ 3 | { 4 | name: 'Trinket', 5 | instagram: 'https://instagram.com/trinketbaby/', 6 | }, 7 | { 8 | name: 'Ernie', 9 | instagram: 'https://instagram.com/sirbunnybottom/', 10 | }, 11 | { 12 | name: 'Tibby', 13 | instagram: 'https://instagram.com/tibbythecorgi/', 14 | }, 15 | { 16 | name: 'Pudge', 17 | instagram: 'https://instagram.com/pudgethecorgi/', 18 | }, 19 | ]); 20 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Node.js CI 2 | 3 | on: push 4 | 5 | jobs: 6 | test: 7 | name: Node.js v${{ matrix.node-version }} (${{ matrix.os }}) 8 | runs-on: ${{ matrix.os }} 9 | 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | node-version: [ 16, 18 ] 14 | os: [ubuntu-latest, windows-latest, macOS-latest] 15 | 16 | steps: 17 | - name: Checkout the repo 18 | uses: actions/checkout@v3 19 | 20 | - name: Use Node.js v${{ matrix.node-version }} 21 | uses: actions/setup-node@v3 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | 25 | - name: npm install 26 | run: npm ci 27 | 28 | - name: Run tests 29 | run: npm test 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Ryan Murphy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "quaff", 3 | "type": "module", 4 | "version": "5.0.0", 5 | "description": "Collect JS/JSON/YAML/YML/CSV/TSV/ArchieML files from a source folder and convert them into a single object.", 6 | "exports": "./lib/index.js", 7 | "types": "./lib/index.d.ts", 8 | "repository": { 9 | "type": "git", 10 | "url": "git+https://github.com/rdmurphy/quaff.git" 11 | }, 12 | "author": "Ryan Murphy", 13 | "license": "MIT", 14 | "files": [ 15 | "lib" 16 | ], 17 | "engines": { 18 | "node": "^12.20.0 || ^14.13.1 || >=16.0.0" 19 | }, 20 | "scripts": { 21 | "build": "tsc", 22 | "git-pre-commit": "precise-commits", 23 | "test": "tsc --noEmit && node --loader tsm test/test.ts", 24 | "release": "np --no-yarn" 25 | }, 26 | "keywords": [ 27 | "data", 28 | "javascript", 29 | "json", 30 | "yaml", 31 | "yml", 32 | "csv", 33 | "tsv" 34 | ], 35 | "devDependencies": { 36 | "@rdm/prettier-config": "^3.0.0", 37 | "@tsconfig/node12": "^1.0.8", 38 | "@types/d3-dsv": "^3.0.0", 39 | "@types/js-yaml": "^4.0.1", 40 | "@types/node": "^18.0.1", 41 | "@types/parse-json": "^4.0.0", 42 | "@vercel/git-hooks": "^1.0.0", 43 | "precise-commits": "^1.0.2", 44 | "prettier": "^2.0.5", 45 | "tsm": "^2.1.4", 46 | "typescript": "^5.0.2", 47 | "uvu": "^0.5.2" 48 | }, 49 | "dependencies": { 50 | "archieml": "^0.5.0", 51 | "d3-dsv": "^3.0.1", 52 | "dset": "^3.0.0", 53 | "js-yaml": "^4.0.0", 54 | "parse-json": "^7.0.0", 55 | "totalist": "^3.0.0" 56 | }, 57 | "prettier": "@rdm/prettier-config" 58 | } 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Microbundle cache 58 | .rpt2_cache/ 59 | .rts2_cache_cjs/ 60 | .rts2_cache_es/ 61 | .rts2_cache_umd/ 62 | 63 | # Optional REPL history 64 | .node_repl_history 65 | 66 | # Output of 'npm pack' 67 | *.tgz 68 | 69 | # Yarn Integrity file 70 | .yarn-integrity 71 | 72 | # dotenv environment variables file 73 | .env 74 | .env.test 75 | .env.production 76 | 77 | # parcel-bundler cache (https://parceljs.org/) 78 | .cache 79 | .parcel-cache 80 | 81 | # Next.js build output 82 | .next 83 | out 84 | 85 | # Nuxt.js build / generate output 86 | .nuxt 87 | dist 88 | 89 | # Gatsby files 90 | .cache/ 91 | # Comment in the public line in if your project uses Gatsby and not Next.js 92 | # https://nextjs.org/blog/next-9-1#public-directory-support 93 | # public 94 | 95 | # vuepress build output 96 | .vuepress/dist 97 | 98 | # Serverless directories 99 | .serverless/ 100 | 101 | # FuseBox cache 102 | .fusebox/ 103 | 104 | # DynamoDB Local files 105 | .dynamodb/ 106 | 107 | # TernJS port file 108 | .tern-port 109 | 110 | # Stores VSCode versions used for testing VSCode extensions 111 | .vscode-test 112 | 113 | # yarn v2 114 | .yarn/cache 115 | .yarn/unplugged 116 | .yarn/build-state.yml 117 | .yarn/install-state.gz 118 | .pnp.* 119 | 120 | # build folder 121 | lib -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [5.0.0] - 2021-06-12 11 | 12 | ### Added 13 | - The `quaff` individual file processor is now available at `loadFile`. This makes it possible to tap into all of `quaff`'s processors to load a single file. The newly named `load` export works the same as before and uses `loadFile` behind the scenes. 14 | - It is now possible to include JavaScript files using the `.cjs` and `.mjs` extensions. 15 | 16 | ### Changed 17 | - `quaff` is now a [pure ESM package](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c). It can no longer be `require()`'d from CommonJS. If this functionality is still needed please continue to use `quaff@^4`. It is also possible to [dynamically import ESM in CommonJS](https://nodejs.org/api/esm.html#esm_import_expressions) (`await import('quaff')`) if that is compatible with your use case. 18 | - `quaff` no longer has a default export and now uses two named exports - `load` and `loadFile`. 19 | 20 | ## [4.2.0] - 2020-07-16 21 | 22 | ### Added 23 | 24 | - `quaff` will now throw an error when more than one input file attempts to use the same output key. This is caused by having multiple files in a directory with the same name but different extensions. 25 | - When a `.yaml` or `.yml` file fails to parse the thrown error will now include the file path. 26 | - We are now testing `quaff` in Mac OS and Windows thanks to GitHub Actions. Don't expect that'll ever be an issue but good to know. 27 | 28 | ### Changed 29 | 30 | - `tiny-glob` has been replaced with [`totalist`](https://github.com/lukeed/totalist), which makes quaff a little faster at iterating through files. 31 | - All tests have been moved to [`uvu`](https://github.com/lukeed/uvu). 32 | 33 | ## [4.1.0] - 2019-03-04 34 | 35 | ### Added 36 | 37 | - Added TypeScript definition file. 38 | 39 | ### Changed 40 | 41 | - Some housekeeping in `index.js`, but no functional changes. 42 | 43 | ## [4.0.0] - 2019-01-25 44 | 45 | ### Added 46 | 47 | - Support for [ArchieML](http://archieml.org/) files. These are processed when they are found with the `.aml` extension. 48 | - Support for JavaScript files. This includes any JavaScript file that provides a default export (`module.exports = ...`). JavaScript files that export functions, including _async_ functions, are also supported! This makes it possible for `quaff` to load data that's fetched from an API. Load GraphQL and go to town! Do some extra-preprocessing! 49 | 50 | ### Changed 51 | 52 | - `quaff` now returns a **Promise** to enable async resolution of JavaScript files. 53 | - Moves testing to `nyc`. 54 | - Now requires Node.js **8 or later**. 55 | -------------------------------------------------------------------------------- /source/index.ts: -------------------------------------------------------------------------------- 1 | // internal 2 | import { promises as fs } from 'fs'; 3 | import path from 'path'; 4 | import { pathToFileURL } from 'url'; 5 | 6 | // packages 7 | import archieml from 'archieml'; 8 | import { dset } from 'dset'; 9 | import { csvParse, tsvParse } from 'd3-dsv'; 10 | import parseJson from 'parse-json'; 11 | import { totalist } from 'totalist'; 12 | import yaml from 'js-yaml'; 13 | 14 | /** 15 | * quaff's valid extensions. 16 | */ 17 | const validExtensions = [ 18 | '.js', 19 | '.cjs', 20 | '.mjs', 21 | '.json', 22 | '.yaml', 23 | '.yml', 24 | '.csv', 25 | '.tsv', 26 | '.aml', 27 | ]; 28 | 29 | /** 30 | * @param filePath the input file path 31 | * @returns {Promise} 32 | */ 33 | export async function loadFile(filePath: string): Promise { 34 | const ext = path.extname(filePath); 35 | 36 | // it can return absolutely anything 37 | let data: unknown; 38 | 39 | // we give JavaScript entries a special treatment 40 | if (ext === '.js' || ext === '.cjs' || ext === '.mjs') { 41 | // js path 42 | // @ts-ignore - dynamic imports *can* take URL objects, but TypeScript disagrees 43 | data = (await import(pathToFileURL(filePath))).default; 44 | 45 | if (typeof data === 'function') { 46 | data = await data(); 47 | } 48 | } else { 49 | // otherwise look for matches 50 | const fileContents = await fs.readFile(filePath, 'utf8'); 51 | 52 | switch (ext) { 53 | // json path 54 | case '.json': 55 | data = parseJson(fileContents, filePath); 56 | break; 57 | // yaml paths 58 | case '.yaml': 59 | case '.yml': 60 | data = yaml.load(fileContents, { filename: filePath }); 61 | break; 62 | // csv path 63 | case '.csv': 64 | data = csvParse(fileContents); 65 | break; 66 | // tsv path 67 | case '.tsv': 68 | data = tsvParse(fileContents); 69 | break; 70 | // aml path 71 | case '.aml': 72 | data = archieml.load(fileContents); 73 | break; 74 | default: 75 | throw new Error( 76 | `Unable to parse ${filePath} - no valid processor found for ${ext} extension`, 77 | ); 78 | } 79 | } 80 | 81 | return data; 82 | } 83 | 84 | /** 85 | * We know this will return a string-keyed Object, but that's about it. 86 | */ 87 | export type LoadReturnValue = Record; 88 | 89 | /** 90 | * @param dirPath the input directory 91 | * @returns {Promise} 92 | */ 93 | export async function load(dirPath: string): Promise { 94 | // normalize the input path 95 | const cwd = path.normalize(dirPath); 96 | 97 | // the object we will eventually return with data 98 | const output: LoadReturnValue = {}; 99 | 100 | // a set to watch out for duplicate keys 101 | const existing = new Set(); 102 | 103 | // loop through the files in the directory 104 | await totalist(cwd, async (rel, abs) => { 105 | const { name, dir, ext } = path.parse(rel); 106 | 107 | // early exit if not a valid extension 108 | if (!validExtensions.includes(ext)) return; 109 | 110 | // remove the leading path, split into a list, and filter out empty strings 111 | const dirs = dir.split(path.sep).filter(Boolean); 112 | 113 | // add the filename to the path part list 114 | dirs.push(name); 115 | 116 | // build a unique "key" for this file so we can watch out for dupes 117 | const key = dirs.join('.'); 118 | 119 | // if this key isn't unique, throw an error 120 | if (existing.has(key)) { 121 | throw new Error( 122 | `More than one file attempted to use "${key}" as its path. This error is caused by having multiple files in a directory with the same name but different extensions.`, 123 | ); 124 | } 125 | 126 | // otherwise save it for checking future inputs 127 | existing.add(key); 128 | 129 | const data = await loadFile(abs); 130 | dset(output, dirs, data); 131 | }); 132 | 133 | return output; 134 | } 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | quaff 3 |

4 |

5 | quaff 6 |

7 |

8 |

9 | npm 10 | github actions 11 | install size 12 |

13 | 14 | ## Important! 15 | 16 | `quaff` is now a [pure ESM package](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c). It can no longer be `require()`'d from CommonJS. If this functionality is still needed please continue to use `quaff@^4`. 17 | 18 | ## Key features 19 | 20 | - 🚚 A **data pipeline helper** written in Node.js that works similar to [Middleman](https://middlemanapp.com/)'s [Data Files](https://middlemanapp.com/advanced/data_files/) collector 21 | - 📦 Point the library at a folder filled with JS, AML ([ArchieML](http://archieml.org)), JSON, YAML, CSV and/or TSV files and **get a JavaScript object back that reflects the folder's structure and content/exports** 22 | - 🤓 Under the hood it uses [`parse-json`](https://github.com/sindresorhus/parse-json) (for better JSON error support), [`js-yaml`](https://github.com/nodeca/js-yaml) and [`d3-dsv`](https://github.com/d3/d3-dsv) to **read files efficiently** 23 | 24 | ## Installation 25 | 26 | ```sh 27 | npm install quaff --save-dev 28 | ``` 29 | 30 | `quaff` requires **Node.js 12.20.0 or later**. 31 | 32 | ## Usage 33 | 34 | Assume a folder with this structure. 35 | 36 | ```txt 37 | data/ 38 | mammals/ 39 | cats.json 40 | dogs.json 41 | bears.csv 42 | birds/ 43 | parrots.yml 44 | story.aml 45 | ``` 46 | 47 | After `import`'ing `quaff`: 48 | 49 | ```js 50 | import { load } from 'quaff'; 51 | 52 | const data = await load('./data/'); 53 | console.log(data); 54 | ``` 55 | 56 | And the results... 57 | 58 | ```json 59 | { 60 | "mammals": { 61 | "cats": ["Marty", "Sammy"], 62 | "dogs": ["Snazzy", "Cally"], 63 | "bears": [ 64 | { 65 | "name": "Steve", 66 | "type": "Polar bear" 67 | }, 68 | { 69 | "name": "Angelica", 70 | "type": "Sun bear" 71 | } 72 | ] 73 | }, 74 | "birds": { 75 | "parrots": { 76 | "alive": ["Buzz"], 77 | "dead": ["Moose"] 78 | }, 79 | "story": { 80 | "title": "All about birds", 81 | "prose": [ 82 | { "type": "text", "value": "Do you know how great birds are?" }, 83 | { "type": "text", "value": "Come with me on this journey." } 84 | ] 85 | } 86 | } 87 | } 88 | ``` 89 | 90 | As of `5.0.0` it's now possible to load a single file at a time, enabling more custom approaches in case `load` doesn't work exactly the way you'd like. 91 | 92 | ```js 93 | import { loadFile } from 'quaff'; 94 | 95 | const data = await loadFile('./data/mammals/bears.csv'); 96 | console.log(data); 97 | ``` 98 | 99 | And the results... 100 | 101 | ```json 102 | [ 103 | { 104 | "name": "Steve", 105 | "type": "Polar bear" 106 | }, 107 | { 108 | "name": "Angelica", 109 | "type": "Sun bear" 110 | } 111 | ] 112 | ``` 113 | 114 | ## Advanced Usage with JavaScript files 115 | 116 | One of the biggest features added in `quaff` 4.0 is the ability to load JavaScript files. But how exactly does that work? 117 | 118 | JavaScript files that are consumed by `quaff` have to follow one simple rule - they must `export default` a function, an async function or value. All three of these are valid and return the same value: 119 | 120 | ```js 121 | export default [ 122 | { 123 | name: 'Pudge', 124 | instagram: 'https://instagram.com/pudgethecorgi/', 125 | }, 126 | ]; 127 | ``` 128 | 129 | ```js 130 | export default () => [ 131 | { 132 | name: 'Pudge', 133 | instagram: 'https://instagram.com/pudgethecorgi/', 134 | }, 135 | ]; 136 | ``` 137 | 138 | ```js 139 | export default async () => [ 140 | { 141 | name: 'Pudge', 142 | instagram: 'https://instagram.com/pudgethecorgi/', 143 | }, 144 | ]; 145 | ``` 146 | 147 | The final example above is the most interesting one - `async` functions also work! This means you can write code to hit API endpoints, or do other asynchronous work, and `quaff` will wait for those to resolve. 148 | 149 | ```js 150 | import fetch from 'node-fetch'; 151 | 152 | export default async () => { 153 | const res = await fetch('https://my-cool-api/'); 154 | const data = await res.json(); 155 | 156 | // whatever the API returned will be added to the quaff object! 157 | return data; 158 | }; 159 | ``` 160 | 161 | Don't have a `Promise` to do async work with? Working with a callback interface? Just wrap it in one! 162 | 163 | ```js 164 | import {apiHelper } from 'my-callback-api'; 165 | 166 | export default () => { 167 | return new Promise((resolve, reject) => { 168 | apiHelper('people', (err, data) => { 169 | if (err) return reject(err); 170 | 171 | // quaff will take it from here! 172 | resolve(data); 173 | }); 174 | }); 175 | }; 176 | ``` 177 | 178 | ## License 179 | 180 | By [Ryan Murphy](https://twitter.com/rdmurphy). 181 | 182 | Available under the MIT license. 183 | -------------------------------------------------------------------------------- /test/test.ts: -------------------------------------------------------------------------------- 1 | // native 2 | import { test as it } from 'uvu'; 3 | import { strict as assert } from 'assert'; 4 | import { promises as fs } from 'fs'; 5 | 6 | // packages 7 | import archieml from 'archieml'; 8 | import { csvParse, tsvParse } from 'd3-dsv'; 9 | import yaml from 'js-yaml'; 10 | 11 | // internal 12 | import * as quaff from '../source/index.js'; 13 | 14 | const readJson = async (filepath: string) => 15 | JSON.parse(await fs.readFile(filepath, 'utf8')); 16 | const readYaml = async (filepath: string) => 17 | yaml.load(await fs.readFile(filepath, 'utf8')); 18 | const readCsv = async (filepath: string) => 19 | csvParse(await fs.readFile(filepath, 'utf8')); 20 | const readTsv = async (filepath: string) => 21 | tsvParse(await fs.readFile(filepath, 'utf8')); 22 | const readArchieML = async (filepath: string) => 23 | archieml.load(await fs.readFile(filepath, 'utf8')); 24 | 25 | it('should normalize a trailing extra slash', async () => { 26 | assert.deepStrictEqual(await quaff.load('./test/source/basic_json/'), { 27 | corgis: await readJson('./test/source/basic_json/corgis.json'), 28 | }); 29 | }); 30 | 31 | it('should return object generated from json', async () => { 32 | assert.deepStrictEqual(await quaff.load('./test/source/basic_json'), { 33 | corgis: await readJson('./test/source/basic_json/corgis.json'), 34 | }); 35 | }); 36 | 37 | it('should return object generated from yaml', async () => { 38 | assert.deepStrictEqual(await quaff.load('./test/source/basic_yaml'), { 39 | corgis: await readYaml('./test/source/basic_yaml/corgis.yaml'), 40 | }); 41 | }); 42 | 43 | it('should return object generated from yml', async () => { 44 | assert.deepStrictEqual(await quaff.load('./test/source/basic_yml'), { 45 | corgis: await readYaml('./test/source/basic_yml/corgis.yml'), 46 | }); 47 | }); 48 | 49 | it('should return object generated from csv', async () => { 50 | assert.deepStrictEqual(await quaff.load('./test/source/basic_csv'), { 51 | corgis: await readCsv('./test/source/basic_csv/corgis.csv'), 52 | }); 53 | }); 54 | 55 | it('should return object generated from tsv', async () => { 56 | assert.deepStrictEqual(await quaff.load('./test/source/basic_tsv'), { 57 | corgis: await readTsv('./test/source/basic_tsv/corgis.tsv'), 58 | }); 59 | }); 60 | 61 | it('should ignore files that do not match filters', async () => { 62 | assert.deepStrictEqual(await quaff.load('./test/source/non_match_file'), { 63 | corgis: await readYaml('./test/source/non_match_file/corgis.yaml'), 64 | }); 65 | }); 66 | 67 | it('should return object representing data one subdirectory deep', async () => { 68 | assert.deepStrictEqual(await quaff.load('./test/source/single_depth'), { 69 | corgis: await readJson('./test/source/single_depth/corgis.json'), 70 | others: { 71 | malamutes: await readJson( 72 | './test/source/single_depth/others/malamutes.json', 73 | ), 74 | corgis: await readCsv('./test/source/basic_csv/corgis.csv'), 75 | }, 76 | }); 77 | }); 78 | 79 | it('should return object representing data two subdirectories deep', async () => { 80 | assert.deepStrictEqual(await quaff.load('./test/source/double_depth'), { 81 | corgis: await readJson('./test/source/double_depth/corgis.json'), 82 | others: { 83 | malamutes: await readJson( 84 | './test/source/double_depth/others/malamutes.json', 85 | ), 86 | outcasts: { 87 | cats: await readCsv( 88 | './test/source/double_depth/others/outcasts/cats.csv', 89 | ), 90 | }, 91 | }, 92 | }); 93 | }); 94 | 95 | it('should throw an error when attempting to load empty JSON', async () => { 96 | await assert.rejects(quaff.load('./test/source/basic_json_empty'), { 97 | name: 'JSONError', 98 | message: /^Unexpected end of JSON input/, 99 | }); 100 | }); 101 | 102 | it('should throw an error when attempting to load bad JSON', async () => { 103 | await assert.rejects(quaff.load('./test/source/basic_json_error'), { 104 | name: 'JSONError', 105 | message: /^Unexpected token "}"/, 106 | }); 107 | }); 108 | 109 | it('should throw an error when attempting to load bad YAML', async () => { 110 | await assert.rejects(quaff.load('./test/source/basic_yaml_error'), { 111 | name: 'YAMLException', 112 | message: /^end of the stream or a document separator is expected/, 113 | }); 114 | }); 115 | 116 | it('should return what is exported from a JavaScript file (no function)', async () => { 117 | assert.deepStrictEqual(await quaff.load('./test/source/basic_js'), { 118 | corgis: (await import('./source/basic_js/corgis.js')).default, 119 | }); 120 | 121 | assert.deepStrictEqual(await quaff.load('./test/source/basic_cjs'), { 122 | // @ts-ignore 123 | corgis: (await import('./source/basic_cjs/corgis.cjs')).default, 124 | }); 125 | 126 | assert.deepStrictEqual(await quaff.load('./test/source/basic_mjs'), { 127 | // @ts-ignore 128 | corgis: (await import('./source/basic_mjs/corgis.mjs')).default, 129 | }); 130 | }); 131 | 132 | it('should return what is exported from a JavaScript file (sync function)', async () => { 133 | assert.deepStrictEqual(await quaff.load('./test/source/sync_js'), { 134 | corgis: (await import('./source/sync_js/corgis.js')).default(), 135 | }); 136 | 137 | assert.deepStrictEqual(await quaff.load('./test/source/sync_cjs'), { 138 | // @ts-ignore 139 | corgis: (await import('./source/sync_cjs/corgis.cjs')).default(), 140 | }); 141 | 142 | assert.deepStrictEqual(await quaff.load('./test/source/sync_mjs'), { 143 | // @ts-ignore 144 | corgis: (await import('./source/sync_mjs/corgis.mjs')).default(), 145 | }); 146 | }); 147 | 148 | it('should return what is exported from a JavaScript file (async function)', async () => { 149 | assert.deepStrictEqual(await quaff.load('./test/source/async_js'), { 150 | corgis: await (await import('./source/async_js/corgis.js')).default(), 151 | }); 152 | 153 | assert.deepStrictEqual(await quaff.load('./test/source/async_cjs'), { 154 | // @ts-ignore 155 | corgis: await (await import('./source/async_cjs/corgis.cjs')).default(), 156 | }); 157 | 158 | assert.deepStrictEqual(await quaff.load('./test/source/async_mjs'), { 159 | // @ts-ignore 160 | corgis: await (await import('./source/async_mjs/corgis.mjs')).default(), 161 | }); 162 | }); 163 | 164 | it('should return object generated from aml', async () => { 165 | assert.deepStrictEqual(await quaff.load('./test/source/basic_aml'), { 166 | corgis: await readArchieML('./test/source/basic_aml/corgis.aml'), 167 | }); 168 | 169 | assert.deepStrictEqual(await quaff.load('./test/source/advanced_aml'), { 170 | text: await readArchieML('./test/source/advanced_aml/text.aml'), 171 | }); 172 | }); 173 | 174 | it('should throw an error if a file key is reused', async () => { 175 | await assert.rejects(quaff.load('./test/source/duplicate_keys'), { 176 | name: 'Error', 177 | message: /^More than one file attempted/, 178 | }); 179 | }); 180 | 181 | it('should be possible to read a file directly with quaffFile', async () => { 182 | const filePath = './test/source/basic_json/corgis.json'; 183 | assert.deepStrictEqual( 184 | await quaff.loadFile(filePath), 185 | await readJson(filePath), 186 | ); 187 | }); 188 | 189 | it('should throw an error if a non-valid extension is passed to quaffFile', async () => { 190 | await assert.rejects( 191 | quaff.loadFile('./test/source/non_match_file/corgis.txt'), 192 | { 193 | name: 'Error', 194 | message: /^Unable to parse/, 195 | }, 196 | ); 197 | }); 198 | 199 | it.run(); 200 | --------------------------------------------------------------------------------