├── .babelrc ├── .github └── workflows │ └── node.js.yml ├── .gitignore ├── .prettierrc ├── README.md ├── babel.config.json ├── browser-test └── index.spec.js ├── dist ├── browser │ ├── bundle.js │ └── bundle.js.map └── node │ ├── browser-utils │ ├── index.js │ └── utils.js │ ├── data.js │ ├── dataset.js │ ├── file-base.js │ ├── file-inline.js │ ├── file-interface.js │ ├── file-local.js │ ├── file-remote.js │ ├── index.js │ └── parser │ ├── csv.js │ └── xlsx.js ├── docs └── API.md ├── index.html ├── karma.conf.js ├── package.json ├── src ├── browser-utils │ ├── index.js │ └── utils.js ├── data.js ├── dataset.js ├── file-base.js ├── file-inline.js ├── file-interface.js ├── file-local.js ├── file-remote.js ├── index.js └── parser │ ├── csv.js │ └── xlsx.js ├── test ├── data.js ├── dataset.js ├── file-base.js ├── file-inline.js ├── file-local.js ├── file-remote.js ├── fixtures │ ├── co2-ppm │ │ ├── README.md │ │ ├── data │ │ │ ├── co2-annmean-gl.csv │ │ │ ├── co2-annmean-mlo.csv │ │ │ ├── co2-gr-gl.csv │ │ │ ├── co2-gr-mlo.csv │ │ │ ├── co2-mm-gl.csv │ │ │ └── co2-mm-mlo.csv │ │ └── datapackage.json │ ├── dp-with-inline-readme │ │ └── datapackage.json │ ├── encodings │ │ ├── iso8859.csv │ │ └── western-macos-roman.csv │ ├── sample-cyrillic-encoding.csv │ ├── sample-multi-sheet.xlsx │ ├── sample.csv │ ├── sample.xls │ ├── sample.xlsx │ ├── semicolon-delimited.csv │ └── some file.name.ext └── parser │ ├── csv.js │ └── xlsx.js ├── webpack.config.js └── yarn.lock /.babelrc: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "presets": [ 4 | [ 5 | "@babel/preset-env", 6 | { 7 | "targets": { 8 | "node": "current" 9 | } 10 | } 11 | ] 12 | ] 13 | } -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: [push] 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | matrix: 15 | node-version: [10.x, 12.x, 14.x] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Use Node.js ${{ matrix.node-version }} 20 | uses: actions/setup-node@v1 21 | with: 22 | node-version: ${{ matrix.node-version }} 23 | - run: yarn 24 | - run: yarn build 25 | - run: yarn test 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | lib-cov 4 | *.seed 5 | *.log 6 | *.dat 7 | *.out 8 | *.pid 9 | *.gz 10 | 11 | pids/ 12 | logs/ 13 | results/ 14 | node_modules/ 15 | .idea/ 16 | 17 | npm-debug.log 18 | package-lock.json 19 | 20 | sandbox/* 21 | 22 | .nyc_output 23 | coverage 24 | .vscode 25 | 26 | datajs/test/_babel* 27 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 2, 4 | "semi": false, 5 | "singleQuote": true, 6 | "printWidth": 79 7 | } 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | `frictionless.js` is a lightweight, standardized "stream-plus-metadata" interface for accessing files and datasets, especially tabular ones (CSV, Excel). 2 | 3 | `frictionless.js` follows the ["Frictionless Data Lib Pattern"][fd-pattern]. 4 | 5 | [fd-pattern]: http://okfnlabs.org/blog/2018/02/15/design-pattern-for-a-core-data-library.html#dataset 6 | 7 | * **Open it fast**: simple `open` method for data on disk, online and inline 8 | * **Data plus**: data plus metadata (size, path, etc) in standardized way 9 | * **Stream it**: raw streams and object streams 10 | * **Tabular**: open CSV, Excel or arrays and get a row stream 11 | * **Frictionless**: compatible with [Frictionless Data standards][fd] 12 | 13 | [![Build Status](https://travis-ci.org/frictionlessdata/frictionless-js.svg?branch=master)](https://travis-ci.org/frictionlessdata/frictionless-js) [![Gitter](https://img.shields.io/gitter/room/frictionlessdata/chat.svg)](https://gitter.im/datahubio/chat) 14 | 15 | A line of code is worth a thousand words ... 16 | 17 | ``` 18 | const {open} = require('frictionless.js') 19 | 20 | var file = open('path/to/ons-mye-population-totals.xls') 21 | 22 | file.descriptor 23 | { 24 | path: '/path/to/ons-mye-population-totals.xls', 25 | pathType: 'local', 26 | name: 'ons-mye-population-totals', 27 | format: 'xls', 28 | mediatype: 'application/vnd.ms-excel', 29 | encoding: 'windows-1252' 30 | } 31 | 32 | file.size 33 | 67584 34 | 35 | file.rows() => stream object for rows 36 | // keyed by header row by default ... 37 | { 'col1': 1, 'col2': 2, ... } 38 | { 'col1': 10, 'col2': 20, ... } 39 | ``` 40 | 41 | 42 | 43 | 44 | **Table of Contents** 45 | 46 | - [Motivation](#motivation) 47 | - [Features](#features) 48 | - [Installation](#installation) 49 | - [Browser](#browser) 50 | - [Usage](#usage) 51 | - [API](#api) 52 | - [open](#open) 53 | - [Files](#files) 54 | - [load](#load) 55 | - [Metadata](#metadata) 56 | - [stream](#stream) 57 | - [buffer](#buffer) 58 | - [rows](#rows) 59 | - [Datasets](#datasets) 60 | - [load](#load-1) 61 | - [addResource](#addresource) 62 | - [Utilities](#utilities) 63 | - [isDataset](#isdataset) 64 | - [parseDatasetIdentifier](#parsedatasetidentifier) 65 | - [Developers](#developers) 66 | 67 | 68 | 69 | ## Motivation 70 | 71 | `frictionless.js` is motivated by the following use cases: 72 | 73 | * **Data "plus"**: when you work with data you always find yourself needing the data itself plus a little bit more -- things like where the data came from on disk (or is going to), or how large it is. This library gives you that information in a standardized way. 74 | * **Convenient open**: the same simple `open` method whether you are accessing data on disk, from a URL or inline data from a string, buffer or array. 75 | * **Streams (or strings)**: standardized iterator / object stream interface to data wherever you've loaded from online, on disk or inline 76 | * **Building block for data pipelines**: provides a standardized building block for more complex data processing. For example, suppose you want to load a csv file then write to JSON. That's simple enough. But then suppose you want to delete the first 3 rows, delete the 2nd column. Now you have a more complex processing pipeline. This library provides a simple set of "data plus metadata" objects that you can pass along your pipeline. 77 | 78 | ## Features 79 | 80 | * Easy: a single common API for local, online and inline data 81 | * Micro: The whole project is ~400 lines of code 82 | * Simple: Oriented for single purpose 83 | * Explicit: No hidden behaviours, no extra magic 84 | * Frictionlesss: uses and supports (but does not require) [Frictionless Data][fd] specs such as [Data Package][dp] so you can leverage Frictionless tooling 85 | * Minimal glue: Use on its own or as a building block for more complex data tooling (thanks to its common minimal metadata) 86 | 87 | [fd]: https://frictionlessdata.io/ 88 | [dp]: https://frictionlessdata.io/data-packages/ 89 | 90 | ## Installation 91 | 92 | `npm install frictionless.js` 93 | 94 | 95 | ## Browser 96 | 97 | If you want to use the it in the browser, first you need to build the bundle. 98 | 99 | Run the following command to generate the bundle for the necessary JS targets 100 | 101 | `yarn build` 102 | 103 | This will create two bundles in the `dist` folder. `node` sub-folder contains build for node environment, while `browser` sub-folder contains build for the browser. In a simple html file you can use it like this: 104 | ```html 105 | 106 | 107 | 113 | 114 | 115 | ``` 116 | 117 | ## Usage 118 | 119 | With a simple file: 120 | 121 | ```javascript 122 | const data = require('frictionless.js') 123 | 124 | // path can be local or remote 125 | const file = data.open(path) 126 | 127 | // descriptor with metadata e.g. name, path, format, (guessed) mimetype etc 128 | console.log(file.descriptor) 129 | 130 | // returns promise with raw stream 131 | const stream = await file.stream() 132 | 133 | // let's get an object stream of the rows 134 | // (assuming it is tabular i.e. csv, xls etc) 135 | const rows = await file.rows() 136 | 137 | // entire file as a buffer 138 | const buffer = await file.buffer 139 | 140 | //for large files you can return in chunks 141 | await file.bufferInChunks((chunk, progress)=>{ 142 | console.log(progress, chunk) 143 | }) 144 | 145 | 146 | ``` 147 | 148 | With a Dataset: 149 | 150 | ```javascript 151 | const { Dataset } = require('frictionless.js') 152 | 153 | const path = '/path/to/directory/' // must have datapackage.json in the directory atm 154 | 155 | Dataset.load(path).then(dataset => { 156 | // get a data file in this dataset 157 | const file = dataset.resources[0] 158 | 159 | const data = file.stream() 160 | }) 161 | ``` 162 | 163 | ## API 164 | 165 | 166 | ### open 167 | 168 | Load a file from a path or descriptor. 169 | 170 | ` 171 | load(pathOrDescriptor, {basePath, format}={}) 172 | ` 173 | 174 | There are 3 types of file source we support: 175 | 176 | * Local path 177 | * Remote url 178 | * Inline data 179 | 180 | ```javascript 181 | const data = require('frictionless.js') 182 | 183 | const file = data.open('/path/to/file.csv') 184 | 185 | const file = data.open('https://example.com/data.xls') 186 | 187 | // loading raw data 188 | const file = data.open({ 189 | name: 'mydata', 190 | data: { // can be any javascript - an object, an array or a string or ... 191 | a: 1, 192 | b: 2 193 | } 194 | }) 195 | 196 | // Loading with a descriptor - this allows more fine-grained configuration 197 | // The descriptor should follow the Frictionless Data Resource model 198 | // http://specs.frictionlessdata.io/data-resource/ 199 | const file = data.open({ 200 | // file or url path 201 | path: 'https://example.com/data.csv', 202 | // a Table Schema - https://specs.frictionlessdata.io/table-schema/ 203 | schema: { 204 | fields: [ 205 | ... 206 | ] 207 | } 208 | // CSV dialect - https://specs.frictionlessdata.io/csv-dialect/ 209 | dialect: { 210 | // this is tab separated CSV/DSV 211 | delimiter: '\t' 212 | } 213 | }) 214 | ``` 215 | 216 | `basePath`: use in cases where you want to create a File with a path that is relative to a base directory / path e.g. 217 | 218 | ``` 219 | const file = data.open('data.csv', {basePath: '/my/base/path'}) 220 | ``` 221 | 222 | Will open the file: `/my/base/path/data.csv` 223 | 224 | This functionality is primarily useful when using Files as part of Datasets where it can be convenient for a File to have a path relative to the directory of the Dataset. (See also Data Package and Data Resource in the Frictionless Data specs). 225 | 226 | 227 | ### Files 228 | 229 | A single data file - local or remote. 230 | 231 | #### load 232 | 233 | *DEPRECATED*. Use simple `open`. 234 | 235 | #### Metadata 236 | 237 | Main metadata is available via the `descriptor`: 238 | 239 | ``` 240 | file.descriptor 241 | ``` 242 | 243 | This metadata is a combination of the metadata passed in at File creation (if you created the File with a descriptor object) and auto-inferred information from the File path. This is the info that is auto-inferred: 244 | 245 | ``` 246 | path: path this was instantiated with - may not be same as file.path (depending on basePath) 247 | pathType: remote | local 248 | name: file name (without extension) 249 | format: the extension 250 | mediatype: mimetype based on file name and extension 251 | ``` 252 | 253 | In addition to this metadata there are certain properties which are computed on demand: 254 | 255 | ```javascript 256 | // the full path to the file (using basepath) 257 | const path = file.path 258 | 259 | const size = file.size 260 | 261 | // md5 hash of the file 262 | const hash = file.hash() 263 | 264 | // sha256 hash of the file 265 | const hash256 = file.hash(hashType='sha256') 266 | 267 | // file encoding 268 | const encoding = file.encoding 269 | ``` 270 | 271 | **Note**: size, hash are not available for remote Files (those created from urls). 272 | 273 | 274 | #### stream 275 | 276 | `stream()` 277 | 278 | Get readable stream 279 | 280 | @returns Promise with readable stream object on resolve 281 | 282 | #### buffer 283 | 284 | `File.buffer` 285 | 286 | Get this file as a buffer (async) 287 | 288 | @returns: promise which resolves to the buffer 289 | 290 | #### rows 291 | 292 | `rows({keyed}={})` 293 | 294 | Get the rows for this file as a node object stream (assumes underlying data is tabular!) 295 | 296 | @returns Promise with rows as parsed JS objects (depends on file format) 297 | 298 | * `keyed`: if `false` (default) returns rows as arrays. If `true` returns rows as objects. 299 | 300 | TODO: casting (does data get cast automatically for you or not ...) 301 | 302 | **What formats are supported?** 303 | 304 | The rows functionality is currently available for CSV and Excel files. The Tabular support incorporates supports for Table Schema and CSV Dialect e.g. you can do: 305 | 306 | ```javascript 307 | 308 | // load a CSV with a non-standard dialect e.g. tab separated or semi-colon separated 309 | const file = data.open({ 310 | path: 'mydata.tsv' 311 | // Full support for http://specs.frictionlessdata.io/csv-dialect/ 312 | dialect: { 313 | delimiter: '\t' // for tabs or ';' for semi-colons etc 314 | } 315 | }) 316 | 317 | // open a CSV with a Table Schema 318 | const file = data.open({ 319 | path: 'mydata.csv' 320 | // Full support for Table Schema https://specs.frictionlessdata.io/table-schema/ 321 | schema: { 322 | fields: [ 323 | { 324 | name: 'Column 1', 325 | type: 'integer' 326 | }, 327 | ... 328 | ] 329 | } 330 | }) 331 | ``` 332 | 333 | 334 | ### Datasets 335 | 336 | A collection of data files with optional metadata. 337 | 338 | Under the hood it heavily uses Data Package formats and it natively supports Data Package formats including loading from `datapackage.json` files. However, it does not require knowledge or use of Data Packages. 339 | 340 | A Dataset has four primary properties: 341 | 342 | * `descriptor`: key metadata. The descriptor follows the Data Package spec 343 | * `resources`: an array of the Files contained in this Dataset 344 | * `identifier`: the identifier encapsulates the location (or origin) of this Dataset 345 | * `readme`: the README for this Dataset (if it exists). The readme content is taken from the README.md file located in the Dataset root directory, or, if that does not exist from the `readme` property on the descriptor. If neither of those exist the readme will be undefined or null. 346 | 347 | In addition we provide the convenience attributes: 348 | 349 | * `path`: the path (remote or local) to this dataset 350 | * `dataPackageJsonPath`: the path to the `datapackage.json` for this Dataset (if it exists) 351 | 352 | #### load 353 | 354 | To create a new Dataset object use `Dataset.load`. It takes descriptor Object or identifier string: 355 | 356 | ``` 357 | async Dataset.load(pathOrDescriptor, {owner = null} = {}) 358 | ``` 359 | 360 | * `pathOrDescriptor` - can be one of: 361 | * local path to Dataset 362 | * remote url to Dataset 363 | * descriptor object 364 | * @returns: a fully loaded Dataset (parsed and used `datapackage.json` and `README.md` -- if README exists) 365 | 366 | For example: 367 | 368 | ```javascript 369 | const data = require('frictionless.js') 370 | 371 | const pathOrDescriptor = 'https://raw.githubusercontent.com/datasets/co2-ppm/master/datapackage.json' 372 | const dataset = await data.Dataset.load(pathOrDescriptor) 373 | ``` 374 | 375 | #### addResource 376 | 377 | Add a resource to the Dataset: 378 | 379 | ```javascript 380 | addResource(resource) 381 | ``` 382 | 383 | * `resource`: may be an already instantiated File object or it is a resource descriptor 384 | * @returns: null 385 | 386 | 387 | ### Utilities 388 | 389 | #### isDataset 390 | 391 | ```javascript 392 | // seeks to guess whether a given path is the path to a Dataset or a File 393 | // (i.e. a directory or datapackage.json) 394 | data.isDataset(path) 395 | ``` 396 | 397 | #### parseDatasetIdentifier 398 | 399 | ```javascript 400 | // parses dataset path and returns identifier dictionary 401 | // handles local paths, remote URLs as well as DataHub and GitHub specific URLs 402 | // (e.g., https://datahub.io/core/finance-vix or https://github.com/datasets/finance-vix 403 | const identifier = data.parseDatasetIdentifier(path) 404 | 405 | console.log(identifier) 406 | ``` 407 | 408 | and it prints out: 409 | 410 | ``` 411 | { 412 | name: , 413 | owner: , 414 | path: , 415 | type: , 416 | original: , 417 | version: 418 | } 419 | ``` 420 | 421 | ## Developers 422 | 423 | Requirements: 424 | 425 | * NodeJS >= v8.10.0 426 | * NPM >= v5.2.0 427 | 428 | 429 | ### Test 430 | 431 | We have two type of tests Karma based for browser testing and Mocha with Chai for Node. All node tests are in `datajs/test` folder. Since Mocha is sensitive to test namings, we have separate the folder `/browser-test` for only Karma. 432 | 433 | - To run browser test, first you need to build the library in order to have the bundle in `dist/browser` folder. Run: `yarn build:browser` to achieve this, then for browser testing use the command `yarn test:browser`, this will run Karma tests. 434 | - To test in Node: `yarn test:node` 435 | - To run all tests including Node and browser run `yarn test` 436 | - To watch Node test run: `yarn test:node:watch` 437 | 438 | ### Setup 439 | 440 | 1. Git clone the repo 441 | 2. Install dependencies: `yarn` 442 | 3. To make the browser and node test work, first run the build: `yarn build` 443 | 4. Run tests: `yarn test` 444 | 5. Do some dev work 445 | 6. Once done, make sure tests are passing. Then build distribution version of the app - `yarn build`. 446 | 447 | Run `yarn build` to compile using webpack and babel for different node and web target. To watch the build run: `yarn build:watch`. 448 | 449 | 7. Now proceed to "Deployment" stage 450 | 451 | 452 | ### Deployment 453 | 454 | 1. Update version number in `package.json`. 455 | 2. Git commit: `git commit -m "some message, eg, version"`. 456 | 3. Release: `git tag -a v0.12.0 -m "some message"`. 457 | 4. Push: `git push origin master --tags` 458 | 5. Publish to NPM: `npm publish` 459 | -------------------------------------------------------------------------------- /babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/preset-env" 4 | ] 5 | } -------------------------------------------------------------------------------- /browser-test/index.spec.js: -------------------------------------------------------------------------------- 1 | const toArray = require('stream-to-array') 2 | 3 | const genFile = () => { 4 | return new File( 5 | [ 6 | `number,string,boolean 7 | 1,two,true 8 | 3,four,false 9 | `, 10 | ], 11 | 'sample.csv', 12 | { type: 'text/csv' } 13 | ) 14 | } 15 | 16 | describe('FileInterface', function () { 17 | it('addSchema()', async () => { 18 | const file = new data.open(genFile()) 19 | 20 | expect(file.descriptor.schema).toBe(undefined) 21 | await file.addSchema() 22 | headers = file.descriptor.schema.fields.map((field) => field.name) 23 | expect(headers).toEqual(['number', 'string', 'boolean']) 24 | expect(file.descriptor.schema.fields[1]['type']).toBe('string') 25 | }) 26 | 27 | it('displayName, size and hash', async () => { 28 | const file = new data.open(genFile()) 29 | 30 | const hash = await file.hash('md5') 31 | const hashSha256 = await file.hash('sha256') 32 | expect(hash).toBe('b0661d9566498a800fbf95365ce28747') 33 | expect(hashSha256).toBe( 34 | 'd9d47b90ac9607c5111ff9a83aa37bc10e058ce6206c00b6626ade091784e098' 35 | ) 36 | expect(file.displayName).toBe('FileInterface') 37 | expect(file.size).toBe(46) 38 | }) 39 | 40 | it('rows()', async () => { 41 | const file = new data.open(genFile()) 42 | 43 | let rowStream = await file.rows({ size: -1 }) 44 | let rows = await toArray(rowStream) 45 | expect(rows[0]).toEqual(['number', 'string', 'boolean']) 46 | expect(rows[1]).toEqual(['1', 'two', 'true']) 47 | 48 | rowStream = await file.rows({ size: 1 }) 49 | rows = await toArray(rowStream) 50 | expect(rows[0]).toEqual(['number', 'string', 'boolean']) 51 | expect(rows[1]).toEqual(undefined) 52 | }) 53 | 54 | it('stream()', async () => { 55 | const file = new data.open(genFile()) 56 | 57 | // Test stream 58 | const stream = await file.stream({ size: -1 }) 59 | const out = await toArray(stream) 60 | expect(out.toString().indexOf('number,string,boolean')).toBeGreaterThan(-1) 61 | }) 62 | 63 | it('buffer()', async () => { 64 | const file = new data.open(genFile()) 65 | const buffer = await file.buffer 66 | const text = new TextDecoder('utf-8').decode(buffer) 67 | expect(text.slice(0, 21)).toBe('number,string,boolean') 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /dist/node/browser-utils/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | if (typeof window !== 'undefined') module.exports = { ...require('./utils') 4 | };else module.exports = { 5 | isFileFromBrowser: file => {}, 6 | webToNodeStream: reader => {} 7 | }; -------------------------------------------------------------------------------- /dist/node/browser-utils/utils.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.webToNodeStream = webToNodeStream; 7 | exports.isFileFromBrowser = isFileFromBrowser; 8 | 9 | var _stream = require("stream"); 10 | 11 | var _readableWebToNodeStream = require("readable-web-to-node-stream"); 12 | 13 | async function webToNodeStream(stream, size) { 14 | if (size === undefined || size === -1) { 15 | return new _readableWebToNodeStream.ReadableWebToNodeStream(stream); 16 | } else { 17 | const nodeStream = new _stream.Readable({ 18 | read: {} 19 | }); 20 | let lineCounter = 0; 21 | let lastString = ''; 22 | const decoder = new TextDecoder(); 23 | let reader = stream.getReader(); 24 | 25 | while (true) { 26 | const { 27 | done, 28 | value 29 | } = await reader.read(); 30 | 31 | if (done || lineCounter > size && size !== 0) { 32 | reader.cancel(); 33 | break; 34 | } 35 | 36 | const string = `${lastString}${decoder.decode(value)}`; 37 | const lines = string.split(/\r\n|[\r\n]/g); 38 | lastString = lines.pop() || ''; 39 | 40 | for (const line of lines) { 41 | if (lineCounter === size) { 42 | reader.cancel(); 43 | break; 44 | } 45 | 46 | nodeStream.push(line + '\r\n'); 47 | lineCounter++; 48 | } 49 | } 50 | 51 | nodeStream.push(null); 52 | return nodeStream; 53 | } 54 | } 55 | 56 | function isFileFromBrowser(file) { 57 | return file instanceof File; 58 | } -------------------------------------------------------------------------------- /dist/node/data.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.open = open; 7 | exports.isDataset = exports.isUrl = exports.parseDatasetIdentifier = exports.parsePath = exports.KNOWN_TABULAR_FORMAT = exports.PARSE_DATABASE = exports.DEFAULT_ENCODING = void 0; 8 | 9 | var _fs = _interopRequireDefault(require("fs")); 10 | 11 | var _path = _interopRequireDefault(require("path")); 12 | 13 | var _url = _interopRequireDefault(require("url")); 14 | 15 | var _nodeFetch = _interopRequireDefault(require("node-fetch")); 16 | 17 | var _lodash = require("lodash"); 18 | 19 | var _mimeTypes = _interopRequireDefault(require("mime-types")); 20 | 21 | var _csv = require("./parser/csv"); 22 | 23 | var _xlsx = require("./parser/xlsx"); 24 | 25 | var _index = require("./browser-utils/index"); 26 | 27 | var _fileInterface = require("./file-interface"); 28 | 29 | var _fileLocal = require("./file-local"); 30 | 31 | var _fileRemote = require("./file-remote"); 32 | 33 | var _fileInline = require("./file-inline"); 34 | 35 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 36 | 37 | const DEFAULT_ENCODING = 'utf-8'; 38 | exports.DEFAULT_ENCODING = DEFAULT_ENCODING; 39 | const PARSE_DATABASE = { 40 | csv: _csv.csvParser, 41 | tsv: _csv.csvParser, 42 | xlsx: _xlsx.xlsxParser, 43 | xls: _xlsx.xlsxParser 44 | }; 45 | exports.PARSE_DATABASE = PARSE_DATABASE; 46 | const KNOWN_TABULAR_FORMAT = ['csv', 'tsv', 'dsv']; 47 | exports.KNOWN_TABULAR_FORMAT = KNOWN_TABULAR_FORMAT; 48 | 49 | function open(pathOrDescriptor, { 50 | basePath, 51 | format 52 | } = {}) { 53 | let descriptor = null; 54 | 55 | if ((0, _index.isFileFromBrowser)(pathOrDescriptor)) { 56 | return new _fileInterface.FileInterface(pathOrDescriptor); 57 | } 58 | 59 | if ((0, _lodash.isPlainObject)(pathOrDescriptor)) { 60 | descriptor = (0, _lodash.cloneDeep)(pathOrDescriptor); 61 | 62 | if (descriptor.data) { 63 | return new _fileInline.FileInline(descriptor, { 64 | basePath 65 | }); 66 | } else if (descriptor.path) { 67 | descriptor = Object.assign(parsePath(descriptor.path, basePath), descriptor); 68 | } 69 | } else if ((0, _lodash.isString)(pathOrDescriptor)) { 70 | descriptor = parsePath(pathOrDescriptor, basePath, format); 71 | } else { 72 | throw new TypeError(`Cannot create File from ${pathOrDescriptor}`); 73 | } 74 | 75 | const isRemote = descriptor.pathType === 'remote' || isUrl(basePath); 76 | 77 | if (isRemote) { 78 | return new _fileRemote.FileRemote(descriptor, { 79 | basePath 80 | }); 81 | } 82 | 83 | return new _fileLocal.FileLocal(descriptor, { 84 | basePath 85 | }); 86 | } 87 | 88 | const parsePath = (path_, basePath = null, format = null) => { 89 | let fileName; 90 | const isItUrl = isUrl(path_) || isUrl(basePath); 91 | 92 | if (isItUrl) { 93 | const urlParts = _url.default.parse(path_); 94 | 95 | fileName = urlParts.pathname.replace(/^.*[\\\/]/, ''); 96 | 97 | if (!format && urlParts.query && urlParts.query.includes('format=csv')) { 98 | format = 'csv'; 99 | } 100 | } else { 101 | fileName = path_.replace(/^.*[\\\/]/, ''); 102 | } 103 | 104 | const extension = _path.default.extname(fileName); 105 | 106 | fileName = fileName.replace(extension, '').toLowerCase().trim().replace(/&/g, '-and-').replace(/[^a-z0-9-._]+/g, '-'); 107 | const descriptor = { 108 | path: path_, 109 | pathType: isItUrl ? 'remote' : 'local', 110 | name: fileName, 111 | format: format ? format : extension.slice(1).toLowerCase() 112 | }; 113 | 114 | const mediatype = _mimeTypes.default.lookup(path_); 115 | 116 | if (mediatype) { 117 | descriptor.mediatype = mediatype; 118 | } 119 | 120 | return descriptor; 121 | }; 122 | 123 | exports.parsePath = parsePath; 124 | 125 | const parseDatasetIdentifier = async path_ => { 126 | const out = { 127 | name: '', 128 | owner: null, 129 | path: '', 130 | type: '', 131 | original: path_, 132 | version: '' 133 | }; 134 | if (path_ === null || path_ === '') return out; 135 | out.type = isUrl(path_) ? 'url' : 'local'; 136 | let normalizedPath = path_.replace(/\/?datapackage\.json/, ''); 137 | normalizedPath = normalizedPath.replace(/\/$/, ''); 138 | 139 | if (out.type === 'local') { 140 | if (process.platform === 'win32') { 141 | out.path = _path.default.resolve(normalizedPath); 142 | } else { 143 | out.path = _path.default.posix.resolve(normalizedPath); 144 | } 145 | 146 | out.name = _path.default.basename(out.path); 147 | } else if (out.type === 'url') { 148 | const urlparts = _url.default.parse(normalizedPath); 149 | 150 | const parts = urlparts.pathname.split('/'); 151 | let name = parts[parts.length - 1]; 152 | let owner = null; 153 | 154 | if (urlparts.host === 'github.com') { 155 | out.type = 'github'; 156 | urlparts.host = 'raw.githubusercontent.com'; 157 | owner = parts[1]; 158 | let repoName = parts[2]; 159 | let branch = 'master'; 160 | 161 | if (parts.length < 6) { 162 | name = repoName; 163 | } 164 | 165 | if (parts.length == 3) { 166 | parts.push(branch); 167 | } else { 168 | branch = parts[4]; 169 | parts.splice(3, 1); 170 | } 171 | 172 | urlparts.pathname = parts.join('/'); 173 | out.version = branch; 174 | } else if (urlparts.host === 'datahub.io') { 175 | out.type = 'datahub'; 176 | urlparts.host = 'pkgstore.datahub.io'; 177 | owner = parts[1]; 178 | name = parts[2]; 179 | 180 | if (owner !== 'core') { 181 | let resolvedPath = await (0, _nodeFetch.default)(`https://api.datahub.io/resolver/resolve?path=${owner}/${name}`); 182 | resolvedPath = await resolvedPath.json(); 183 | parts[1] = resolvedPath.userid; 184 | } 185 | 186 | let res = await (0, _nodeFetch.default)(`https://api.datahub.io/source/${parts[1]}/${name}/successful`); 187 | 188 | if (res.status >= 400) { 189 | throw new Error('Provided URL is invalid. Expected URL to a dataset or descriptor.'); 190 | } 191 | 192 | res = await res.json(); 193 | const revisionId = parseInt(res.id.split('/').pop(), 10); 194 | parts.push(revisionId); 195 | urlparts.pathname = parts.join('/'); 196 | out.version = revisionId; 197 | } 198 | 199 | out.name = name; 200 | out.owner = owner; 201 | out.path = _url.default.format(urlparts) + '/'; 202 | } 203 | 204 | return out; 205 | }; 206 | 207 | exports.parseDatasetIdentifier = parseDatasetIdentifier; 208 | 209 | const isUrl = path_ => { 210 | const r = new RegExp('^(?:[a-z]+:)?//', 'i'); 211 | return r.test(path_); 212 | }; 213 | 214 | exports.isUrl = isUrl; 215 | 216 | const isDataset = path_ => { 217 | if (path_.endsWith('datapackage.json')) { 218 | return true; 219 | } 220 | 221 | const isItUrl = isUrl(path_); 222 | 223 | if (isItUrl) { 224 | return false; 225 | } else if (_fs.default.lstatSync(path_).isFile()) { 226 | return false; 227 | } 228 | 229 | return true; 230 | }; 231 | 232 | exports.isDataset = isDataset; -------------------------------------------------------------------------------- /dist/node/dataset.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.Dataset = void 0; 7 | 8 | var _lodash = require("lodash"); 9 | 10 | var _urlJoin = _interopRequireDefault(require("url-join")); 11 | 12 | var _fs = _interopRequireDefault(require("fs")); 13 | 14 | var _path = _interopRequireDefault(require("path")); 15 | 16 | var _nodeFetch = _interopRequireDefault(require("node-fetch")); 17 | 18 | var _data = require("./data"); 19 | 20 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 21 | 22 | class Dataset { 23 | constructor(descriptor = {}, identifier = { 24 | path: null, 25 | owner: null 26 | }) { 27 | if (!(0, _lodash.isPlainObject)(descriptor)) { 28 | throw new TypeError(`To create a new Dataset please use Dataset.load`); 29 | } 30 | 31 | this._descriptor = descriptor; 32 | this._resources = []; 33 | this._identifier = identifier; 34 | } 35 | 36 | static async load(pathOrDescriptor, { 37 | owner = null 38 | } = {}) { 39 | if (!((0, _lodash.isString)(pathOrDescriptor) || (0, _lodash.isPlainObject)(pathOrDescriptor))) { 40 | throw new TypeError('Dataset needs to be created with descriptor Object or identifier string'); 41 | } 42 | 43 | let descriptor, 44 | identifier = null; 45 | 46 | if ((0, _lodash.isPlainObject)(pathOrDescriptor)) { 47 | descriptor = pathOrDescriptor; 48 | identifier = { 49 | path: null, 50 | owner: owner 51 | }; 52 | } else { 53 | descriptor = {}; 54 | identifier = await (0, _data.parseDatasetIdentifier)(pathOrDescriptor); 55 | } 56 | 57 | const dataset = new Dataset(descriptor, identifier); 58 | await dataset._sync(); 59 | return dataset; 60 | } 61 | 62 | async _sync() { 63 | const readmePath = this._path('README.md'); 64 | 65 | switch (this.identifier.type) { 66 | case 'local': 67 | { 68 | if (_fs.default.existsSync(this.dataPackageJsonPath)) { 69 | this._descriptor = JSON.parse(_fs.default.readFileSync(this.dataPackageJsonPath)); 70 | this._originalDescriptor = (0, _lodash.cloneDeep)(this._descriptor); 71 | } else { 72 | throw new Error('No datapackage.json at destination.'); 73 | } 74 | 75 | if (_fs.default.existsSync(readmePath)) { 76 | this._descriptor.readme = _fs.default.readFileSync(readmePath).toString(); 77 | } 78 | 79 | break; 80 | } 81 | 82 | case 'url': 83 | case 'github': 84 | case 'datahub': 85 | { 86 | let res = await (0, _nodeFetch.default)(this.dataPackageJsonPath); 87 | 88 | if (res.status >= 400) { 89 | throw new Error(`${res.status}: ${res.statusText}. Requested URL: ${res.url}`); 90 | } 91 | 92 | this._descriptor = await res.json(); 93 | this._originalDescriptor = (0, _lodash.cloneDeep)(this._descriptor); 94 | 95 | if (!this._descriptor.readme) { 96 | res = await (0, _nodeFetch.default)(readmePath); 97 | 98 | if (res.status === 200) { 99 | this._descriptor.readme = await res.text(); 100 | } 101 | } 102 | 103 | break; 104 | } 105 | } 106 | 107 | this._resources = this.descriptor.resources.map(resource => { 108 | return (0, _data.open)(resource, { 109 | basePath: this.path 110 | }); 111 | }); 112 | this.descriptor.resources = this._resources.map(resource => { 113 | return resource.descriptor; 114 | }); 115 | } 116 | 117 | get identifier() { 118 | return this._identifier; 119 | } 120 | 121 | get descriptor() { 122 | return this._descriptor; 123 | } 124 | 125 | get path() { 126 | return this.identifier.path; 127 | } 128 | 129 | get dataPackageJsonPath() { 130 | return this._path('datapackage.json'); 131 | } 132 | 133 | get readme() { 134 | return this._descriptor.readme; 135 | } 136 | 137 | get resources() { 138 | return this._resources; 139 | } 140 | 141 | addResource(resource) { 142 | if ((0, _lodash.isPlainObject)(resource)) { 143 | this.descriptor.resources.push(resource); 144 | this.resources.push((0, _data.open)(resource)); 145 | } else if ((0, _lodash.isObject)(resource)) { 146 | this.descriptor.resources.push(resource.descriptor); 147 | this.resources.push(resource); 148 | } else { 149 | throw new TypeError(`addResource requires a resource descriptor or an instantiated resources but got: ${resource}`); 150 | } 151 | } 152 | 153 | _path(offset = null) { 154 | const path_ = this.path ? this.path.replace('datapackage.json', '') : this.path; 155 | 156 | switch (this.identifier.type) { 157 | case 'local': 158 | return _path.default.join(path_, offset); 159 | 160 | case 'url': 161 | return (0, _urlJoin.default)(path_, offset); 162 | 163 | case 'github': 164 | return (0, _urlJoin.default)(path_, offset); 165 | 166 | case 'datahub': 167 | return (0, _urlJoin.default)(path_, offset); 168 | 169 | case undefined: 170 | return offset; 171 | 172 | default: 173 | throw new Error(`Unknown path type: ${this.identifier.type}`); 174 | } 175 | } 176 | 177 | } 178 | 179 | exports.Dataset = Dataset; -------------------------------------------------------------------------------- /dist/node/file-base.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.computeHash = computeHash; 7 | exports.File = void 0; 8 | 9 | var _data = require("./data"); 10 | 11 | var _tableschema = require("tableschema"); 12 | 13 | var _streamToArray = _interopRequireDefault(require("stream-to-array")); 14 | 15 | var _lodash = require("lodash"); 16 | 17 | var _csv = require("./parser/csv"); 18 | 19 | var _index = require("./browser-utils/index"); 20 | 21 | var _crypto = _interopRequireDefault(require("crypto")); 22 | 23 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 24 | 25 | const { 26 | Transform 27 | } = require('stream'); 28 | 29 | class File { 30 | constructor(descriptor, { 31 | basePath 32 | } = {}) { 33 | this._descriptor = descriptor; 34 | this._basePath = basePath; 35 | this._descriptor.encoding = this.encoding || _data.DEFAULT_ENCODING; 36 | this._computedHashes = {}; 37 | } 38 | 39 | static load(pathOrDescriptor, { 40 | basePath, 41 | format 42 | } = {}) { 43 | console.warn("WARNING! Depreciated function called. Function 'load' has been deprecated, please use the 'open' function instead!"); 44 | return (0, _data.open)(pathOrDescriptor, { 45 | basePath, 46 | format 47 | }); 48 | } 49 | 50 | get descriptor() { 51 | return this._descriptor; 52 | } 53 | 54 | get path() { 55 | throw new Error('This is an abstract base class which you should not instantiate. Use open() instead'); 56 | } 57 | 58 | async bufferInChunks(getChunk) { 59 | let stream = null; 60 | 61 | if (this.displayName == 'FileInterface') { 62 | stream = (0, _index.webToNodeStream)(this.descriptor.stream()); 63 | } else { 64 | stream = await this.stream(); 65 | } 66 | 67 | let offset = 0; 68 | let totalChunkSize = 0; 69 | let chunkCount = 0; 70 | let fileSize = this.size; 71 | var percent = 0; 72 | 73 | const _reportProgress = new Transform({ 74 | transform(chunk, encoding, callback) { 75 | if (chunkCount % 100 == 0) { 76 | const runningTotal = totalChunkSize + offset; 77 | const percentComplete = Math.round(runningTotal / fileSize * 100); 78 | percent = percentComplete; 79 | } 80 | 81 | callback(null, chunk); 82 | } 83 | 84 | }); 85 | 86 | stream.pipe(_reportProgress).on('data', function (chunk) { 87 | offset += chunk.length; 88 | chunkCount += 1; 89 | let buffer = new Buffer.from(chunk); 90 | getChunk(buffer, percent); 91 | }).on('error', function (err) { 92 | throw new Error(err); 93 | }); 94 | } 95 | 96 | get buffer() { 97 | return (async () => { 98 | const stream = await this.stream(); 99 | const buffers = await (0, _streamToArray.default)(stream); 100 | return Buffer.concat(buffers); 101 | })(); 102 | } 103 | 104 | async hash(hashType = 'md5', progress, cache = true) { 105 | if (cache && hashType in this._computedHashes) { 106 | if (typeof progress === 'function') { 107 | progress(100); 108 | } 109 | 110 | return this._computedHashes[hashType]; 111 | } else { 112 | let hash = await computeHash(await this.stream(), this.size, hashType, progress); 113 | 114 | if (cache && this != null) { 115 | this._computedHashes[hashType] = hash; 116 | } 117 | 118 | return hash; 119 | } 120 | } 121 | 122 | async hashSha256(progress) { 123 | console.warn("WARNING! Depreciated function called. Function 'hashSha256' has been deprecated, use the 'hash' function and pass the algorithm type instead!"); 124 | return this.hash('sha256', progress); 125 | } 126 | 127 | rows({ 128 | keyed, 129 | sheet, 130 | size 131 | } = {}) { 132 | return this._rows({ 133 | keyed, 134 | sheet, 135 | size 136 | }); 137 | } 138 | 139 | _rows({ 140 | keyed, 141 | sheet, 142 | size 143 | } = {}) { 144 | if (this.descriptor.format in _data.PARSE_DATABASE) { 145 | const parser = _data.PARSE_DATABASE[this.descriptor.format]; 146 | return parser(this, { 147 | keyed, 148 | sheet, 149 | size 150 | }); 151 | } 152 | 153 | throw new Error(`We do not have a parser for that format: ${this.descriptor.format}`); 154 | } 155 | 156 | getSample() { 157 | return new Promise(async (resolve, reject) => { 158 | let smallStream = await this.rows({ 159 | size: 100 160 | }); 161 | resolve(await (0, _streamToArray.default)(smallStream)); 162 | }); 163 | } 164 | 165 | async addSchema() { 166 | if (this.displayName === 'FileInline') { 167 | this.descriptor.schema = await (0, _tableschema.infer)(this.descriptor.data); 168 | return; 169 | } 170 | 171 | let sample = await this.getSample(); 172 | 173 | if (this.descriptor.format === 'xlsx' && sample) { 174 | let headers = 1; 175 | 176 | if ((0, _lodash.isPlainObject)(sample[0])) { 177 | headers = Object.keys(sample[0]); 178 | } 179 | 180 | this.descriptor.schema = await (0, _tableschema.infer)(sample, { 181 | headers 182 | }); 183 | return; 184 | } 185 | 186 | if (_data.KNOWN_TABULAR_FORMAT.indexOf(this.descriptor.format) === -1) { 187 | throw new Error('File is not in known tabular format.'); 188 | } 189 | 190 | const parserOptions = await (0, _csv.guessParseOptions)(this); 191 | this.descriptor.dialect = { 192 | delimiter: parserOptions.delimiter, 193 | quoteChar: parserOptions.quote 194 | }; 195 | let thisFileStream = await this.stream({ 196 | size: 100 197 | }); 198 | this.descriptor.schema = await (0, _tableschema.infer)(thisFileStream, parserOptions); 199 | } 200 | 201 | } 202 | 203 | exports.File = File; 204 | 205 | function computeHash(fileStream, fileSize, algorithm, progress, encoding = 'hex') { 206 | return new Promise((resolve, reject) => { 207 | let hash = _crypto.default.createHash(algorithm); 208 | 209 | let offset = 0; 210 | let totalChunkSize = 0; 211 | let chunkCount = 0; 212 | 213 | if (!['hex', 'latin1', 'binary', 'base64'].includes(encoding)) { 214 | throw new Error(`Invalid encoding value: ${encoding}; Expecting 'hex', 'latin1', 'binary' or 'base64'`); 215 | } 216 | 217 | const _reportProgress = new Transform({ 218 | transform(chunk, encoding, callback) { 219 | if (chunkCount % 20 == 0) { 220 | const runningTotal = totalChunkSize + offset; 221 | const percentComplete = Math.round(runningTotal / fileSize * 100); 222 | 223 | if (typeof progress === 'function') { 224 | progress(percentComplete); 225 | } 226 | } 227 | 228 | callback(null, chunk); 229 | } 230 | 231 | }); 232 | 233 | fileStream.pipe(_reportProgress).on('error', function (err) { 234 | reject(err); 235 | }).on('data', function (chunk) { 236 | offset += chunk.length; 237 | chunkCount += 1; 238 | hash.update(chunk); 239 | }).on('end', function () { 240 | hash = hash.digest(encoding); 241 | 242 | if (typeof progress === 'function') { 243 | progress(100); 244 | } 245 | 246 | resolve(hash); 247 | }); 248 | }); 249 | } -------------------------------------------------------------------------------- /dist/node/file-inline.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.FileInline = void 0; 7 | 8 | var _stream = _interopRequireDefault(require("stream")); 9 | 10 | var _fileBase = require("./file-base"); 11 | 12 | var _lodash = require("lodash"); 13 | 14 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 15 | 16 | class FileInline extends _fileBase.File { 17 | constructor(descriptor, { 18 | basePath 19 | } = {}) { 20 | super(descriptor, { 21 | basePath 22 | }); 23 | 24 | if ((0, _lodash.isString)(this.descriptor.data)) { 25 | this._buffer = Buffer.from(this.descriptor.data); 26 | } else { 27 | this._buffer = Buffer.from(JSON.stringify(this.descriptor.data)); 28 | } 29 | } 30 | 31 | get displayName() { 32 | return 'FileInline'; 33 | } 34 | 35 | get path() { 36 | return this.descriptor.path; 37 | } 38 | 39 | get size() { 40 | return this._buffer.byteLength; 41 | } 42 | 43 | stream() { 44 | const bufferStream = new _stream.default.PassThrough(); 45 | bufferStream.end(this._buffer); 46 | return bufferStream; 47 | } 48 | 49 | rows({ 50 | keyed 51 | } = {}) { 52 | if ((0, _lodash.isArray)(this.descriptor.data)) { 53 | const rowStream = new _stream.default.PassThrough({ 54 | objectMode: true 55 | }); 56 | this.descriptor.data.forEach(row => { 57 | rowStream.write(row); 58 | }); 59 | rowStream.end(); 60 | return rowStream; 61 | } 62 | 63 | return this._rows({ 64 | keyed, 65 | size 66 | }); 67 | } 68 | 69 | } 70 | 71 | exports.FileInline = FileInline; -------------------------------------------------------------------------------- /dist/node/file-interface.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.FileInterface = void 0; 7 | 8 | var _data = require("./data"); 9 | 10 | var _index = require("./browser-utils/index"); 11 | 12 | var _fileBase = require("./file-base"); 13 | 14 | class FileInterface extends _fileBase.File { 15 | constructor(descriptor, { 16 | basePath 17 | } = {}) { 18 | super(descriptor, { 19 | basePath 20 | }); 21 | this._descriptor.format = descriptor.name.split('.').pop() || ''; 22 | } 23 | 24 | get displayName() { 25 | return 'FileInterface'; 26 | } 27 | 28 | get path() { 29 | return URL.createObjectURL(this.descriptor); 30 | } 31 | 32 | get encoding() { 33 | return this._encoding || _data.DEFAULT_ENCODING; 34 | } 35 | 36 | async stream({ 37 | size 38 | } = {}) { 39 | return (0, _index.webToNodeStream)(await this.descriptor.stream(), size); 40 | } 41 | 42 | get buffer() { 43 | return this.descriptor.arrayBuffer(); 44 | } 45 | 46 | get size() { 47 | return this.descriptor.size; 48 | } 49 | 50 | get fileName() { 51 | return this.descriptor.name; 52 | } 53 | 54 | } 55 | 56 | exports.FileInterface = FileInterface; -------------------------------------------------------------------------------- /dist/node/file-local.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.FileLocal = void 0; 7 | 8 | var _chardet = _interopRequireDefault(require("chardet")); 9 | 10 | var _fs = _interopRequireDefault(require("fs")); 11 | 12 | var _fileBase = require("./file-base"); 13 | 14 | var _path = _interopRequireDefault(require("path")); 15 | 16 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 17 | 18 | class FileLocal extends _fileBase.File { 19 | get displayName() { 20 | return 'FileLocal'; 21 | } 22 | 23 | get path() { 24 | return this._basePath ? _path.default.join(this._basePath, this.descriptor.path) : this.descriptor.path; 25 | } 26 | 27 | stream({ 28 | size 29 | } = {}) { 30 | let end = size; 31 | return _fs.default.createReadStream(this.path, { 32 | start: 0, 33 | end 34 | }); 35 | } 36 | 37 | get size() { 38 | return _fs.default.statSync(this.path).size; 39 | } 40 | 41 | get encoding() { 42 | if (this.size > 1000000) { 43 | return _chardet.default.detectFileSync(this.path, { 44 | sampleSize: 1000000 45 | }); 46 | } 47 | 48 | return _chardet.default.detectFileSync(this.path); 49 | } 50 | 51 | } 52 | 53 | exports.FileLocal = FileLocal; -------------------------------------------------------------------------------- /dist/node/file-remote.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.FileRemote = void 0; 7 | 8 | var _urlJoin = _interopRequireDefault(require("url-join")); 9 | 10 | var _nodeFetch = _interopRequireDefault(require("node-fetch")); 11 | 12 | var _fileBase = require("./file-base"); 13 | 14 | var _data = require("./data"); 15 | 16 | var _index = require("./browser-utils/index"); 17 | 18 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 19 | 20 | class FileRemote extends _fileBase.File { 21 | get displayName() { 22 | return 'FileRemote'; 23 | } 24 | 25 | get path() { 26 | const isItUrl = (0, _data.isUrl)(this.descriptor.path); 27 | 28 | if (isItUrl) { 29 | return this.descriptor.path; 30 | } else { 31 | return this._basePath ? (0, _urlJoin.default)(this._basePath, this.descriptor.path) : this.descriptor.path; 32 | } 33 | } 34 | 35 | get browserBuffer() { 36 | return (async () => { 37 | const res = await (0, _nodeFetch.default)(this.path); 38 | return await res.arrayBuffer(); 39 | })(); 40 | } 41 | 42 | stream({ 43 | size 44 | } = {}) { 45 | return (async () => { 46 | const res = await (0, _nodeFetch.default)(this.path); 47 | 48 | if (res.status === 200) { 49 | if (typeof window === 'undefined') { 50 | return res.body; 51 | } else { 52 | return (0, _index.webToNodeStream)(res.body, size); 53 | } 54 | } else { 55 | throw new Error(`${res.status}: ${res.statusText}. Requested URL: ${this.path}`); 56 | } 57 | })(); 58 | } 59 | 60 | get encoding() { 61 | return this._encoding || _data.DEFAULT_ENCODING; 62 | } 63 | 64 | } 65 | 66 | exports.FileRemote = FileRemote; -------------------------------------------------------------------------------- /dist/node/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | Object.defineProperty(exports, "open", { 7 | enumerable: true, 8 | get: function () { 9 | return _data.open; 10 | } 11 | }); 12 | Object.defineProperty(exports, "parsePath", { 13 | enumerable: true, 14 | get: function () { 15 | return _data.parsePath; 16 | } 17 | }); 18 | Object.defineProperty(exports, "parseDatasetIdentifier", { 19 | enumerable: true, 20 | get: function () { 21 | return _data.parseDatasetIdentifier; 22 | } 23 | }); 24 | Object.defineProperty(exports, "isDataset", { 25 | enumerable: true, 26 | get: function () { 27 | return _data.isDataset; 28 | } 29 | }); 30 | Object.defineProperty(exports, "isUrl", { 31 | enumerable: true, 32 | get: function () { 33 | return _data.isUrl; 34 | } 35 | }); 36 | Object.defineProperty(exports, "File", { 37 | enumerable: true, 38 | get: function () { 39 | return _fileBase.File; 40 | } 41 | }); 42 | Object.defineProperty(exports, "computeHash", { 43 | enumerable: true, 44 | get: function () { 45 | return _fileBase.computeHash; 46 | } 47 | }); 48 | Object.defineProperty(exports, "FileInterface", { 49 | enumerable: true, 50 | get: function () { 51 | return _fileInterface.FileInterface; 52 | } 53 | }); 54 | Object.defineProperty(exports, "FileLocal", { 55 | enumerable: true, 56 | get: function () { 57 | return _fileLocal.FileLocal; 58 | } 59 | }); 60 | Object.defineProperty(exports, "FileRemote", { 61 | enumerable: true, 62 | get: function () { 63 | return _fileRemote.FileRemote; 64 | } 65 | }); 66 | Object.defineProperty(exports, "FileInline", { 67 | enumerable: true, 68 | get: function () { 69 | return _fileInline.FileInline; 70 | } 71 | }); 72 | Object.defineProperty(exports, "Dataset", { 73 | enumerable: true, 74 | get: function () { 75 | return _dataset.Dataset; 76 | } 77 | }); 78 | Object.defineProperty(exports, "csvParser", { 79 | enumerable: true, 80 | get: function () { 81 | return _csv.csvParser; 82 | } 83 | }); 84 | Object.defineProperty(exports, "xlsxParser", { 85 | enumerable: true, 86 | get: function () { 87 | return _xlsx.xlsxParser; 88 | } 89 | }); 90 | 91 | var _data = require("./data"); 92 | 93 | var _fileBase = require("./file-base"); 94 | 95 | var _fileInterface = require("./file-interface"); 96 | 97 | var _fileLocal = require("./file-local"); 98 | 99 | var _fileRemote = require("./file-remote"); 100 | 101 | var _fileInline = require("./file-inline"); 102 | 103 | var _dataset = require("./dataset"); 104 | 105 | var _csv = require("./parser/csv"); 106 | 107 | var _xlsx = require("./parser/xlsx"); -------------------------------------------------------------------------------- /dist/node/parser/csv.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.csvParser = csvParser; 7 | exports.guessParseOptions = guessParseOptions; 8 | exports.getParseOptions = getParseOptions; 9 | exports.Uint8ArrayToStringsTransformer = void 0; 10 | 11 | var _csvParse = _interopRequireDefault(require("csv-parse")); 12 | 13 | var _streamToString = _interopRequireDefault(require("stream-to-string")); 14 | 15 | var _iconvLite = require("iconv-lite"); 16 | 17 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 18 | 19 | const CSVSniffer = require('csv-sniffer')(); 20 | 21 | async function csvParser(file, { 22 | keyed = false, 23 | size 24 | } = {}) { 25 | const parseOptions = await getParseOptions(file, keyed); 26 | let stream = await file.stream({ 27 | size 28 | }); 29 | 30 | if (file.descriptor.encoding.toLowerCase().replace('-', '') === 'utf8') { 31 | return stream.pipe((0, _csvParse.default)(parseOptions)); 32 | } else { 33 | return stream.pipe((0, _iconvLite.decodeStream)(file.descriptor.encoding)).pipe((0, _csvParse.default)(parseOptions)); 34 | } 35 | } 36 | 37 | async function guessParseOptions(file) { 38 | const possibleDelimiters = [',', ';', ':', '|', '\t', '^', '*', '&']; 39 | const sniffer = new CSVSniffer(possibleDelimiters); 40 | let text = ''; 41 | 42 | if (file.displayName === 'FileLocal') { 43 | const stream = await file.stream({ 44 | size: 50000 45 | }); 46 | text = await (0, _streamToString.default)(stream); 47 | } else if (file.displayName === 'FileInterface') { 48 | let stream = await file.stream({ 49 | size: 10 50 | }); 51 | text = await (0, _streamToString.default)(stream); 52 | } else if (file.displayName === 'FileRemote') { 53 | const stream = await file.stream({ 54 | size: 100 55 | }); 56 | let bytes = 0; 57 | await new Promise((resolve, reject) => { 58 | stream.on('data', chunk => { 59 | bytes += chunk.length; 60 | 61 | if (bytes > 50000) { 62 | stream.pause(); 63 | resolve(); 64 | } else { 65 | text += chunk.toString(); 66 | } 67 | }).on('end', () => { 68 | resolve(); 69 | }); 70 | }); 71 | } 72 | 73 | const results = sniffer.sniff(text); 74 | return { 75 | delimiter: results.delimiter, 76 | quote: results.quoteChar || '"' 77 | }; 78 | } 79 | 80 | async function getParseOptions(file, keyed) { 81 | let parseOptions = { 82 | columns: keyed ? true : null, 83 | ltrim: true 84 | }; 85 | 86 | if (file.descriptor.dialect) { 87 | parseOptions.delimiter = file.descriptor.dialect.delimiter || ','; 88 | parseOptions.rowDelimiter = file.descriptor.dialect.lineTerminator; 89 | parseOptions.quote = file.descriptor.dialect.quoteChar || '"'; 90 | 91 | if (file.descriptor.dialect.doubleQuote !== undefined && file.descriptor.dialect.doubleQuote === false) { 92 | parseOptions.escape = ''; 93 | } 94 | } else { 95 | const guessedParseOptions = await guessParseOptions(file); 96 | parseOptions = Object.assign(parseOptions, guessedParseOptions); 97 | } 98 | 99 | return parseOptions; 100 | } 101 | 102 | class Uint8ArrayToStringsTransformer { 103 | constructor() { 104 | this.decoder = new TextDecoder(); 105 | this.lastString = ''; 106 | } 107 | 108 | transform(chunk, controller) { 109 | const string = `${this.lastString}${this.decoder.decode(chunk)}`; 110 | const lines = string.split(/\r\n|[\r\n]/g); 111 | this.lastString = lines.pop() || ''; 112 | 113 | for (const line of lines) { 114 | controller.enqueue(line); 115 | } 116 | } 117 | 118 | flush(controller) { 119 | if (this.lastString) { 120 | controller.enqueue(this.lastString); 121 | } 122 | } 123 | 124 | } 125 | 126 | exports.Uint8ArrayToStringsTransformer = Uint8ArrayToStringsTransformer; -------------------------------------------------------------------------------- /dist/node/parser/xlsx.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | Object.defineProperty(exports, "__esModule", { 4 | value: true 5 | }); 6 | exports.xlsxParser = xlsxParser; 7 | 8 | var _stream = require("stream"); 9 | 10 | var _xlsx = require("xlsx"); 11 | 12 | var _csvParse = _interopRequireDefault(require("csv-parse")); 13 | 14 | function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } 15 | 16 | async function xlsxParser(file, keyed = false, sheetIdxOrName = 0) { 17 | let buffer; 18 | 19 | if (typeof window === 'undefined' || file.displayName === 'FileInterface') { 20 | buffer = await file.buffer; 21 | } else { 22 | buffer = await file.browserBuffer; 23 | } 24 | 25 | const workbook = (0, _xlsx.read)(buffer, { 26 | type: 'buffer' 27 | }); 28 | let selectedSheetName = sheetIdxOrName; 29 | 30 | if (sheetIdxOrName.constructor.name === 'Number') { 31 | selectedSheetName = workbook.SheetNames[sheetIdxOrName]; 32 | } 33 | 34 | const sheet = workbook.Sheets[selectedSheetName]; 35 | 36 | const csv = _xlsx.utils.sheet_to_csv(sheet); 37 | 38 | const stream = new _stream.Readable(); 39 | stream.push(csv); 40 | stream.push(null); 41 | return stream.pipe((0, _csvParse.default)({ 42 | columns: keyed ? true : null, 43 | ltrim: true 44 | })); 45 | } -------------------------------------------------------------------------------- /docs/API.md: -------------------------------------------------------------------------------- 1 | ## Classes 2 | 3 |
4 |
Dataset
5 |

A collection of data files with optional metadata. 6 | Under the hood it heavily uses Data Package formats and it natively supports 7 | Data Package formats including loading from datapackage.json files.

8 |

A Dataset has four primary properties: 9 | descriptor: key metadata. The descriptor follows the Data Package spec 10 | resources: an array of the Files contained in this Dataset 11 | identifier: the identifier encapsulates the location (or origin) of this Dataset 12 | readme: the README for this Dataset (if it exists).

13 |
14 |
File
15 |

Abstract Base instance of File

16 |
17 |
18 | 19 | ## Constants 20 | 21 |
22 |
parsePath
23 |

Parse a data source path into a descriptor object. The descriptor should follow the Frictionless Data Resource model 24 | http://specs.frictionlessdata.io/data-resource/

25 |
26 |
parseDatasetIdentifier
27 |
28 |
isUrl
29 |

Checks if path os a URL

30 |
31 |
isDataset
32 |

Checks if path is a Dataset package. Dateset follows the Frictionless Data Resource model

33 |
34 |
35 | 36 | ## Functions 37 | 38 |
39 |
open(pathOrDescriptor, options)
40 |

Load a file from a path or descriptor. Files source supported are 41 | local, remote or inline data.

42 |
43 |
computeHash(fileStream, fileSize, algorithm, progress)
44 |

Computes the streaming hash of a file

45 |
46 |
47 | 48 | 49 | 50 | ## parsePath 51 | Parse a data source path into a descriptor object. The descriptor should follow the Frictionless Data Resource model 52 | http://specs.frictionlessdata.io/data-resource/ 53 | 54 | **Kind**: global constant 55 | 56 | | Param | Type | Description | 57 | | --- | --- | --- | 58 | | path_ | string | Data source. Can be a url or local file path | 59 | | basePath | string | Base path to data source | 60 | | format | string | format of the data. | 61 | 62 | 63 | 64 | ## parseDatasetIdentifier 65 | **Kind**: global constant 66 | 67 | | Param | Type | Description | 68 | | --- | --- | --- | 69 | | path_ | string | Data source. Can be a url or local file path | 70 | 71 | 72 | 73 | ## isUrl 74 | Checks if path os a URL 75 | 76 | **Kind**: global constant 77 | 78 | | Param | Type | Description | 79 | | --- | --- | --- | 80 | | path_ | string | Data source. Can be a url or local file path | 81 | 82 | 83 | 84 | ## isDataset 85 | Checks if path is a Dataset package. Dateset follows the Frictionless Data Resource model 86 | 87 | **Kind**: global constant 88 | 89 | | Param | Type | Description | 90 | | --- | --- | --- | 91 | | path_ | string | Data source. Can be a url or local file path | 92 | 93 | 94 | 95 | ## open(pathOrDescriptor, options) 96 | Load a file from a path or descriptor. Files source supported are 97 | local, remote or inline data. 98 | 99 | **Kind**: global function 100 | 101 | | Param | Type | Description | 102 | | --- | --- | --- | 103 | | pathOrDescriptor | array | A source to load data from. Can be a local or remote file path, can be a raw data object with the format: { name: 'mydata', data: { // can be any javascript object, array or string a: 1, b: 2 } } Files can also be loaded with a descriptor object. This allows more fine-grained configuration. The descriptor should follow the Frictionless Data Resource model http://specs.frictionlessdata.io/data-resource/ { file or url path path: 'https://example.com/data.csv', // a Table Schema - https://specs.frictionlessdata.io/table-schema/ schema: { fields: [ ... ] } // CSV dialect - https://specs.frictionlessdata.io/csv-dialect/ dialect: { // this is tab separated CSV/DSV delimiter: '\t' } } | 104 | | options | object | { basePath, format } Use basepath in cases where you want to create a File with a path that is relative to a base directory / path e.g: const file = data.open('data.csv', {basePath: '/my/base/path'}) | 105 | 106 | 107 | 108 | ## computeHash(fileStream, fileSize, algorithm, progress) 109 | Computes the streaming hash of a file 110 | 111 | **Kind**: global function 112 | 113 | | Param | Type | Description | 114 | | --- | --- | --- | 115 | | fileStream | object | A node like stream | 116 | | fileSize | number | Total size of the file | 117 | | algorithm | string | sha256/md5 hashing algorithm to use | 118 | | progress | func | Callback function with progress | 119 | 120 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Document 8 | 9 | 10 | 11 |
12 | 13 | 14 | 15 |
16 | 17 | 18 | 19 | 55 | 56 | -------------------------------------------------------------------------------- /karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration 2 | // Generated on Thu Sep 10 2020 11:07:13 GMT+0400 (Armenia Standard Time) 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | // base path that will be used to resolve all patterns (eg. files, exclude) 7 | basePath: '', 8 | 9 | // frameworks to use 10 | // available frameworks: https://npmjs.org/browse/keyword/karma-adapter 11 | frameworks: ['jasmine', 'browserify'], 12 | 13 | // list of files / patterns to load in the browser 14 | files: ['dist/browser/bundle.js', 'browser-test/*.js'], 15 | 16 | // list of files / patterns to exclude 17 | exclude: [], 18 | 19 | // preprocess matching files before serving them to the browser 20 | // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor 21 | preprocessors: { 22 | 'browser-test/*.js': ['browserify'], 23 | }, 24 | 25 | // test results reporter to use 26 | // possible values: 'dots', 'progress' 27 | // available reporters: https://npmjs.org/browse/keyword/karma-reporter 28 | reporters: ['progress'], 29 | 30 | // web server port 31 | port: 9877, 32 | 33 | // enable / disable colors in the output (reporters and logs) 34 | colors: true, 35 | 36 | // level of logging 37 | // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG 38 | logLevel: config.LOG_INFO, 39 | 40 | // enable / disable watching file and executing tests whenever any file changes 41 | autoWatch: true, 42 | 43 | // start these browsers 44 | // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher 45 | browsers: ['ChromeDebugging'], 46 | 47 | // Continuous Integration mode 48 | // if true, Karma captures browsers, runs the tests and exits 49 | singleRun: true, 50 | 51 | 52 | // Concurrency level 53 | // how many browser should be started simultaneous 54 | concurrency: Infinity, 55 | customLaunchers: { 56 | ChromeDebugging: { 57 | base: 'Chrome', 58 | flags: ['--remote-debugging-port=9333'], 59 | }, 60 | }, 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frictionless.js", 3 | "version": "0.13.4", 4 | "description": "", 5 | "main": "dist/node/index.js", 6 | "directories": { 7 | "test": "src/test" 8 | }, 9 | "scripts": { 10 | "test": "yarn run test:node && yarn run test:browser", 11 | "test:browser": "yarn build:browser && karma start karma.conf.js --browsers=ChromeHeadless", 12 | "test:node": "nyc mocha --require @babel/register test/*", 13 | "build:browser": "webpack --mode production", 14 | "build:browser:watch": "webpack --watch", 15 | "build:node": "rm -rf ./dist/node && babel ./src/ -d ./dist/node --no-comments", 16 | "build": "yarn run build:node && yarn run build:browser", 17 | "coveralls": "cat ./coverage/lcov.info | ./node_modules/.bin/coveralls", 18 | "coverage": "nyc report --reporter=text-lcov | coveralls && nyc report --reporter=lcov", 19 | "lint": "eslint ./src", 20 | "docs": "jsdoc2md src/**.js > docs/API.md" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "git+https://github.com/frictionlessdata/frictionless-js.git" 25 | }, 26 | "author": "datopian", 27 | "license": "ISC", 28 | "bugs": { 29 | "url": "https://github.com/frictionlessdata/frictionless-js/issues" 30 | }, 31 | "homepage": "https://github.com/frictionlessdata/frictionless-js#readme", 32 | "dependencies": { 33 | "chardet": "^1.3.0", 34 | "csv-parse": "^4.15.3", 35 | "csv-sniffer": "^0.1.1", 36 | "iconv-lite": "^0.6.2", 37 | "lodash": "^4.17.21", 38 | "mime-types": "^2.1.29", 39 | "node-fetch": "^2.6.1", 40 | "readable-web-to-node-stream": "^3.0.1", 41 | "stream-to-array": "^2.3.0", 42 | "stream-to-string": "^1.2.0", 43 | "tableschema": "^1.12.4", 44 | "url-join": "^4.0.1", 45 | "xlsx": "^0.17.0" 46 | }, 47 | "devDependencies": { 48 | "@babel/cli": "^7.10.5", 49 | "@babel/core": "^7.10.5", 50 | "@babel/plugin-transform-modules-umd": "^7.10.4", 51 | "@babel/polyfill": "7.11.5", 52 | "@babel/preset-env": "^7.10.4", 53 | "@babel/register": "^7.10.1", 54 | "ava": "3.13.0", 55 | "babel-loader": "^8.1.0", 56 | "browser-env": "3.3.0", 57 | "browserify": "16.5.2", 58 | "chai": "4.2.0", 59 | "core-js": "3.6.5", 60 | "coveralls": "3.1.0", 61 | "eslint": "7.10.0", 62 | "jasmine": "3.6.1", 63 | "jsdoc-to-markdown": "^6.0.1", 64 | "karma": "5.2.3", 65 | "karma-browserify": "7.0.0", 66 | "karma-chrome-launcher": "3.1.0", 67 | "karma-jasmine": "4.0.1", 68 | "mocha": "8.1.3", 69 | "mocha-lcov-reporter": "1.3.0", 70 | "mocha-loader": "5.1.2", 71 | "mocha-webpack": "1.1.0", 72 | "nock": "13.0.4", 73 | "node-forge": "^0.10.0", 74 | "nyc": "15.1.0", 75 | "parcel-bundler": "^1.12.4", 76 | "source-map-loader": "1.1.0", 77 | "transform-loader": "0.2.4", 78 | "watchify": "3.11.1", 79 | "webpack": "4.44.2", 80 | "webpack-cli": "3.3.12", 81 | "xo": "0.33.1" 82 | }, 83 | "nyc": { 84 | "reporter": [ 85 | "lcov", 86 | "text" 87 | ] 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/browser-utils/index.js: -------------------------------------------------------------------------------- 1 | if (typeof window !== 'undefined') module.exports = { ...require('./utils') } 2 | else 3 | module.exports = { 4 | isFileFromBrowser: (file) => {}, 5 | webToNodeStream: (reader) => {}, 6 | } 7 | -------------------------------------------------------------------------------- /src/browser-utils/utils.js: -------------------------------------------------------------------------------- 1 | import { Readable } from 'stream' 2 | import { ReadableWebToNodeStream } from 'readable-web-to-node-stream' 3 | 4 | /** 5 | * Return node like stream from the browser 6 | * Transform browser's Reader to string, then create a nodejs stream from it 7 | * @param {object} stream A browser file stream 8 | * @param {number} size size of file to return 9 | */ 10 | export async function webToNodeStream(stream, size) { 11 | if (size === undefined || size === -1) { 12 | return new ReadableWebToNodeStream(stream) 13 | } else { 14 | const nodeStream = new Readable({ 15 | read: {}, 16 | }) 17 | 18 | let lineCounter = 0 19 | let lastString = '' 20 | 21 | const decoder = new TextDecoder() 22 | let reader = stream.getReader() 23 | 24 | while (true) { 25 | const { done, value } = await reader.read() 26 | 27 | if (done || (lineCounter > size && size !== 0)) { 28 | reader.cancel() 29 | break 30 | } 31 | 32 | // Decode the current chunk to string and prepend the last string 33 | const string = `${lastString}${decoder.decode(value)}` 34 | 35 | // Extract lines from chunk 36 | const lines = string.split(/\r\n|[\r\n]/g) 37 | 38 | // Save last line, as it might be incomplete 39 | lastString = lines.pop() || '' 40 | 41 | for (const line of lines) { 42 | if (lineCounter === size) { 43 | reader.cancel() 44 | break 45 | } 46 | // Write each string line to our nodejs stream 47 | nodeStream.push(line + '\r\n') 48 | lineCounter++ 49 | } 50 | } 51 | 52 | nodeStream.push(null) 53 | 54 | return nodeStream 55 | } 56 | } 57 | 58 | export function isFileFromBrowser(file) { 59 | return file instanceof File 60 | } 61 | -------------------------------------------------------------------------------- /src/data.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import path from 'path' 3 | import url from 'url' 4 | import fetch from 'node-fetch' 5 | import { cloneDeep, isPlainObject, isString } from 'lodash' 6 | import mime from 'mime-types' 7 | import { csvParser } from './parser/csv' 8 | import { xlsxParser } from './parser/xlsx' 9 | 10 | // for browser related functions 11 | import { isFileFromBrowser } from './browser-utils/index' 12 | 13 | import { FileInterface } from './file-interface' 14 | import { FileLocal } from './file-local' 15 | import { FileRemote } from './file-remote' 16 | import { FileInline } from './file-inline' 17 | 18 | export const DEFAULT_ENCODING = 'utf-8' 19 | 20 | // Available parsers per file format 21 | export const PARSE_DATABASE = { 22 | csv: csvParser, 23 | tsv: csvParser, 24 | xlsx: xlsxParser, 25 | xls: xlsxParser, 26 | } 27 | 28 | // List of formats that are known as tabular 29 | export const KNOWN_TABULAR_FORMAT = ['csv', 'tsv', 'dsv'] 30 | 31 | 32 | /** 33 | * Load a file from a path or descriptor. Files source supported are 34 | * local, remote or inline data. 35 | * 36 | * @param {array} pathOrDescriptor - A source to load data from. Can be a local or remote file path, can be a 37 | * raw data object with the format: 38 | * { 39 | * name: 'mydata', 40 | * data: { // can be any javascript object, array or string 41 | * a: 1, 42 | * b: 2 43 | * } 44 | * } 45 | * 46 | * Files can also be loaded with a descriptor object. This allows more fine-grained configuration. The 47 | * descriptor should follow the Frictionless Data Resource model 48 | * http://specs.frictionlessdata.io/data-resource/ 49 | * 50 | * { 51 | * file or url path 52 | * path: 'https://example.com/data.csv', 53 | * // a Table Schema - https://specs.frictionlessdata.io/table-schema/ 54 | * schema: { 55 | * fields: [ 56 | * ... 57 | * ] 58 | * } 59 | * // CSV dialect - https://specs.frictionlessdata.io/csv-dialect/ 60 | * dialect: { 61 | * // this is tab separated CSV/DSV 62 | * delimiter: '\t' 63 | * } 64 | * } 65 | * 66 | * @param {object} options - { basePath, format } Use basepath in cases where you want to create 67 | * a File with a path that is relative to a base directory / path e.g: 68 | * const file = data.open('data.csv', {basePath: '/my/base/path'}) 69 | */ 70 | export function open(pathOrDescriptor, { basePath, format } = {}) { 71 | let descriptor = null 72 | 73 | if (isFileFromBrowser(pathOrDescriptor)) { 74 | return new FileInterface(pathOrDescriptor) 75 | } 76 | 77 | if (isPlainObject(pathOrDescriptor)) { 78 | descriptor = cloneDeep(pathOrDescriptor) 79 | // NB: data must come first - we could have data and path in which path 80 | // is not used (data comes from data) 81 | if (descriptor.data) { 82 | return new FileInline(descriptor, { basePath }) 83 | } else if (descriptor.path) { 84 | // We want properties already in our descriptor to take priority over 85 | // those inferred from path so we assign in this order 86 | descriptor = Object.assign( 87 | parsePath(descriptor.path, basePath), 88 | descriptor 89 | ) 90 | } 91 | } else if (isString(pathOrDescriptor)) { 92 | descriptor = parsePath(pathOrDescriptor, basePath, format) 93 | } else { 94 | throw new TypeError(`Cannot create File from ${pathOrDescriptor}`) 95 | } 96 | 97 | const isRemote = descriptor.pathType === 'remote' || isUrl(basePath) 98 | 99 | if (isRemote) { 100 | return new FileRemote(descriptor, { basePath }) 101 | } 102 | return new FileLocal(descriptor, { basePath }) 103 | } 104 | 105 | /** 106 | * Parse a data source path into a descriptor object. The descriptor should follow the Frictionless Data Resource model 107 | * http://specs.frictionlessdata.io/data-resource/ 108 | * @param {string} path_ - Data source. Can be a url or local file path 109 | * @param {string} basePath - Base path to data source 110 | * @param {string} format - format of the data. 111 | */ 112 | export const parsePath = (path_, basePath = null, format = null) => { 113 | let fileName 114 | const isItUrl = isUrl(path_) || isUrl(basePath) 115 | if (isItUrl) { 116 | const urlParts = url.parse(path_) 117 | // eslint-disable-next-line no-useless-escape 118 | fileName = urlParts.pathname.replace(/^.*[\\\/]/, '') 119 | // Check if format=csv is provided in the query 120 | // But if format is provided explicitely by user then it'll be used 121 | if (!format && urlParts.query && urlParts.query.includes('format=csv')) { 122 | format = 'csv' 123 | } 124 | } else { 125 | // eslint-disable-next-line no-useless-escape 126 | fileName = path_.replace(/^.*[\\\/]/, '') 127 | } 128 | 129 | const extension = path.extname(fileName) 130 | fileName = fileName 131 | .replace(extension, '') 132 | .toLowerCase() 133 | .trim() 134 | .replace(/&/g, '-and-') 135 | .replace(/[^a-z0-9-._]+/g, '-') 136 | 137 | const descriptor = { 138 | path: path_, 139 | pathType: isItUrl ? 'remote' : 'local', 140 | name: fileName, 141 | format: format ? format : extension.slice(1).toLowerCase(), 142 | } 143 | 144 | const mediatype = mime.lookup(path_) 145 | 146 | if (mediatype) { 147 | descriptor.mediatype = mediatype 148 | } 149 | 150 | return descriptor 151 | } 152 | 153 | /** 154 | * 155 | * @param {string} path_ - Data source. Can be a url or local file path 156 | */ 157 | export const parseDatasetIdentifier = async (path_) => { 158 | const out = { 159 | name: '', 160 | owner: null, 161 | path: '', 162 | type: '', 163 | original: path_, 164 | version: '', 165 | } 166 | if (path_ === null || path_ === '') return out 167 | 168 | out.type = isUrl(path_) ? 'url' : 'local' 169 | let normalizedPath = path_.replace(/\/?datapackage\.json/, '') 170 | normalizedPath = normalizedPath.replace(/\/$/, '') 171 | if (out.type === 'local') { 172 | // eslint-disable-next-line no-undef 173 | if (process.platform === 'win32') { 174 | out.path = path.resolve(normalizedPath) 175 | } else { 176 | out.path = path.posix.resolve(normalizedPath) 177 | } 178 | out.name = path.basename(out.path) 179 | } else if (out.type === 'url') { 180 | const urlparts = url.parse(normalizedPath) 181 | const parts = urlparts.pathname.split('/') 182 | let name = parts[parts.length - 1] 183 | let owner = null 184 | // is this a github repository? 185 | if (urlparts.host === 'github.com') { 186 | out.type = 'github' 187 | // yes, modify url for raw file server 188 | urlparts.host = 'raw.githubusercontent.com' 189 | owner = parts[1] 190 | let repoName = parts[2] 191 | let branch = 'master' 192 | 193 | // is the path a repository root? 194 | if (parts.length < 6) { 195 | // yes, use the repository name for the package name 196 | name = repoName 197 | } 198 | 199 | // does the path contain subfolders (after the repository name)? 200 | if (parts.length == 3) { 201 | // no, add 'master' branch 202 | parts.push(branch) 203 | } else { 204 | // yes, extract the branch and remove the 'tree' part 205 | branch = parts[4] 206 | parts.splice(3, 1) 207 | } 208 | 209 | urlparts.pathname = parts.join('/') 210 | out.version = branch 211 | } else if (urlparts.host === 'datahub.io') { 212 | out.type = 'datahub' 213 | urlparts.host = 'pkgstore.datahub.io' 214 | owner = parts[1] 215 | name = parts[2] 216 | if (owner !== 'core') { 217 | let resolvedPath = await fetch( 218 | `https://api.datahub.io/resolver/resolve?path=${owner}/${name}` 219 | ) 220 | resolvedPath = await resolvedPath.json() 221 | parts[1] = resolvedPath.userid 222 | } 223 | let res = await fetch( 224 | `https://api.datahub.io/source/${parts[1]}/${name}/successful` 225 | ) 226 | if (res.status >= 400) { 227 | throw new Error( 228 | 'Provided URL is invalid. Expected URL to a dataset or descriptor.' 229 | ) 230 | } 231 | res = await res.json() 232 | const revisionId = parseInt(res.id.split('/').pop(), 10) 233 | parts.push(revisionId) 234 | urlparts.pathname = parts.join('/') 235 | out.version = revisionId 236 | } 237 | out.name = name 238 | out.owner = owner 239 | out.path = url.format(urlparts) + '/' 240 | } 241 | 242 | return out 243 | } 244 | 245 | /** 246 | * Checks if path os a URL 247 | * @param {string} path_ - Data source. Can be a url or local file path 248 | */ 249 | export const isUrl = (path_) => { 250 | const r = new RegExp('^(?:[a-z]+:)?//', 'i') 251 | return r.test(path_) 252 | } 253 | 254 | /** 255 | * Checks if path is a Dataset package. Dateset follows the Frictionless Data Resource model 256 | * @param {string} path_ - Data source. Can be a url or local file path 257 | */ 258 | export const isDataset = (path_) => { 259 | // If it is a path to file we assume it is not a Dataset 260 | // Only exception is 'datapackage.json': 261 | if (path_.endsWith('datapackage.json')) { 262 | return true 263 | } 264 | const isItUrl = isUrl(path_) 265 | if (isItUrl) { 266 | // If it is URL we assume it is a file as it does not end with 'datapackage.json' 267 | return false 268 | } else if (fs.lstatSync(path_).isFile()) { 269 | return false 270 | } 271 | // All other cases are true 272 | return true 273 | } 274 | -------------------------------------------------------------------------------- /src/dataset.js: -------------------------------------------------------------------------------- 1 | import { cloneDeep, isPlainObject, isString, isObject } from 'lodash' 2 | import urljoin from 'url-join' 3 | import fs from 'fs' 4 | import path from 'path' 5 | import fetch from 'node-fetch' 6 | import { parseDatasetIdentifier, open } from './data' 7 | 8 | /** 9 | * A collection of data files with optional metadata. 10 | * Under the hood it heavily uses Data Package formats and it natively supports 11 | * Data Package formats including loading from datapackage.json files. 12 | * 13 | * A Dataset has four primary properties: 14 | * descriptor: key metadata. The descriptor follows the Data Package spec 15 | * resources: an array of the Files contained in this Dataset 16 | * identifier: the identifier encapsulates the location (or origin) of this Dataset 17 | * readme: the README for this Dataset (if it exists). 18 | */ 19 | export class Dataset { 20 | // TODO: handle owner 21 | constructor(descriptor = {}, identifier = { path: null, owner: null }) { 22 | if (!isPlainObject(descriptor)) { 23 | throw new TypeError(`To create a new Dataset please use Dataset.load`) 24 | } 25 | 26 | this._descriptor = descriptor 27 | this._resources = [] 28 | this._identifier = identifier 29 | } 30 | 31 | // eslint-disable-next-line no-unused-vars 32 | static async load(pathOrDescriptor, { owner = null } = {}) { 33 | if (!(isString(pathOrDescriptor) || isPlainObject(pathOrDescriptor))) { 34 | throw new TypeError( 35 | 'Dataset needs to be created with descriptor Object or identifier string' 36 | ) 37 | } 38 | 39 | let descriptor, 40 | identifier = null 41 | 42 | if (isPlainObject(pathOrDescriptor)) { 43 | descriptor = pathOrDescriptor 44 | identifier = { 45 | path: null, 46 | owner: owner, 47 | } 48 | } else { 49 | // pathOrDescriptor is a path 50 | descriptor = {} 51 | // TODO: owner if provided should override anything parsed from path 52 | identifier = await parseDatasetIdentifier(pathOrDescriptor) 53 | } 54 | 55 | const dataset = new Dataset(descriptor, identifier) 56 | await dataset._sync() 57 | return dataset 58 | } 59 | 60 | // Bootstrap ourselves with {this.path}/datapackage.json and readme if exists 61 | async _sync() { 62 | const readmePath = this._path('README.md') 63 | // eslint-disable-next-line default-case 64 | switch (this.identifier.type) { 65 | case 'local': { 66 | if (fs.existsSync(this.dataPackageJsonPath)) { 67 | this._descriptor = JSON.parse( 68 | fs.readFileSync(this.dataPackageJsonPath) 69 | ) 70 | this._originalDescriptor = cloneDeep(this._descriptor) 71 | } else { 72 | throw new Error('No datapackage.json at destination.') 73 | } 74 | // Now get README from local disk if exists 75 | if (fs.existsSync(readmePath)) { 76 | this._descriptor.readme = fs.readFileSync(readmePath).toString() 77 | } 78 | break 79 | } 80 | case 'url': 81 | case 'github': 82 | case 'datahub': { 83 | let res = await fetch(this.dataPackageJsonPath) 84 | if (res.status >= 400) { 85 | throw new Error( 86 | `${res.status}: ${res.statusText}. Requested URL: ${res.url}` 87 | ) 88 | } 89 | this._descriptor = await res.json() 90 | this._originalDescriptor = cloneDeep(this._descriptor) 91 | if (!this._descriptor.readme) { 92 | res = await fetch(readmePath) 93 | // May not exist and that is ok! 94 | if (res.status === 200) { 95 | this._descriptor.readme = await res.text() 96 | } 97 | } 98 | break 99 | } 100 | } 101 | 102 | // handle case where readme was already inlined in the descriptor as readme 103 | // attribute as e.g. on the datahub 104 | // Now load each resource ... 105 | this._resources = this.descriptor.resources.map((resource) => { 106 | return open(resource, { basePath: this.path }) 107 | }) 108 | // We need to update original descriptor with metadata about resources after guessing them 109 | this.descriptor.resources = this._resources.map((resource) => { 110 | return resource.descriptor 111 | }) 112 | } 113 | 114 | get identifier() { 115 | return this._identifier 116 | } 117 | 118 | get descriptor() { 119 | return this._descriptor 120 | } 121 | 122 | get path() { 123 | return this.identifier.path 124 | } 125 | 126 | get dataPackageJsonPath() { 127 | return this._path('datapackage.json') 128 | } 129 | 130 | get readme() { 131 | return this._descriptor.readme 132 | } 133 | 134 | // Array of File objects 135 | get resources() { 136 | return this._resources 137 | } 138 | 139 | addResource(resource) { 140 | if (isPlainObject(resource)) { 141 | this.descriptor.resources.push(resource) 142 | this.resources.push(open(resource)) 143 | } else if (isObject(resource)) { 144 | // It is already a resource object! 145 | this.descriptor.resources.push(resource.descriptor) 146 | this.resources.push(resource) 147 | } else { 148 | throw new TypeError( 149 | `addResource requires a resource descriptor or an instantiated resources but got: ${resource}` 150 | ) 151 | } 152 | } 153 | 154 | // Path relative to this dataset 155 | _path(offset = null) { 156 | const path_ = this.path 157 | ? this.path.replace('datapackage.json', '') 158 | : this.path 159 | // TODO: ensure offset is relative (security etc) 160 | switch (this.identifier.type) { 161 | case 'local': 162 | return path.join(path_, offset) 163 | case 'url': 164 | return urljoin(path_, offset) 165 | case 'github': 166 | return urljoin(path_, offset) 167 | case 'datahub': 168 | return urljoin(path_, offset) 169 | case undefined: 170 | return offset 171 | default: 172 | throw new Error(`Unknown path type: ${this.identifier.type}`) 173 | } 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/file-base.js: -------------------------------------------------------------------------------- 1 | import { DEFAULT_ENCODING, PARSE_DATABASE, KNOWN_TABULAR_FORMAT } from './data' 2 | import { infer } from 'tableschema' 3 | import toArray from 'stream-to-array' 4 | import { isPlainObject } from 'lodash' 5 | import { guessParseOptions } from './parser/csv' 6 | import { webToNodeStream } from './browser-utils/index' 7 | import { open } from './data' 8 | const { Transform } = require('stream') 9 | import crypto from 'crypto' 10 | /** 11 | * Abstract Base instance of File 12 | */ 13 | export class File { 14 | constructor(descriptor, { basePath } = {}) { 15 | this._descriptor = descriptor 16 | this._basePath = basePath 17 | this._descriptor.encoding = this.encoding || DEFAULT_ENCODING 18 | this._computedHashes = {} 19 | } 20 | 21 | /** 22 | * @deprecated Use "open" instead 23 | * 2019-02-05 kept for backwards compatibility 24 | */ 25 | static load(pathOrDescriptor, { basePath, format } = {}) { 26 | console.warn( 27 | "WARNING! Depreciated function called. Function 'load' has been deprecated, please use the 'open' function instead!" 28 | ) 29 | return open(pathOrDescriptor, { basePath, format }) 30 | } 31 | 32 | get descriptor() { 33 | return this._descriptor 34 | } 35 | 36 | get path() { 37 | throw new Error( 38 | 'This is an abstract base class which you should not instantiate. Use open() instead' 39 | ) 40 | } 41 | 42 | /** 43 | * Return file buffer in chunks 44 | * @param {func} getChunk Callback function that returns current chunk and percent of progress 45 | * 46 | * Example Usage: 47 | * 48 | * await file.bufferInChunks((buf, percent)=>{ 49 | * console.log("contentBuffer :", buf); 50 | * console.log("Progress :", percent); 51 | * }) 52 | * 53 | */ 54 | async bufferInChunks(getChunk) { 55 | let stream = null 56 | 57 | if (this.displayName == 'FileInterface') { 58 | stream = webToNodeStream(this.descriptor.stream()) 59 | } else { 60 | stream = await this.stream() 61 | } 62 | 63 | let offset = 0 64 | let totalChunkSize = 0 65 | let chunkCount = 0 66 | let fileSize = this.size 67 | var percent = 0 68 | 69 | //calculates and sets the progress after every 100th chunk 70 | const _reportProgress = new Transform({ 71 | transform(chunk, encoding, callback) { 72 | if (chunkCount % 100 == 0) { 73 | const runningTotal = totalChunkSize + offset 74 | const percentComplete = Math.round((runningTotal / fileSize) * 100) 75 | percent = percentComplete 76 | } 77 | callback(null, chunk) 78 | }, 79 | }) 80 | 81 | stream 82 | .pipe(_reportProgress) 83 | .on('data', function (chunk) { 84 | offset += chunk.length 85 | chunkCount += 1 86 | 87 | let buffer = new Buffer.from(chunk) 88 | getChunk(buffer, percent) 89 | }) 90 | // .on('end', function () { 91 | // getChunk(null, 100) 92 | // }) 93 | .on('error', function (err) { 94 | throw new Error(err) 95 | }) 96 | } 97 | 98 | /** 99 | * Returns file buffer 100 | */ 101 | get buffer() { 102 | return (async () => { 103 | const stream = await this.stream() 104 | const buffers = await toArray(stream) 105 | 106 | return Buffer.concat(buffers) 107 | })() 108 | } 109 | 110 | /** 111 | * Calculates the hash of a file 112 | * @param {string} hashType - md5/sha256 type of hash algorithm to use 113 | * @param {func} progress - Callback that returns current progress 114 | * @param {boolean} cache - Set to false to disable hash caching 115 | * @returns {string} hash of file 116 | */ 117 | async hash(hashType = 'md5', progress, cache = true) { 118 | if (cache && hashType in this._computedHashes) { 119 | if (typeof progress === 'function') { 120 | progress(100) 121 | } 122 | return this._computedHashes[hashType] 123 | } else { 124 | let hash = await computeHash( 125 | await this.stream(), 126 | this.size, 127 | hashType, 128 | progress 129 | ) 130 | if (cache && this != null) { 131 | this._computedHashes[hashType] = hash 132 | } 133 | return hash 134 | } 135 | } 136 | 137 | /** 138 | * @deprecated Use "hash" function instead by passing the algorithm type 139 | * 140 | * Calculates the hash of a file usiƒng sha256 algorithm 141 | * @param {func} progress - Callback that returns current progress 142 | * @returns {string} hash of file 143 | */ 144 | async hashSha256(progress) { 145 | console.warn( 146 | "WARNING! Depreciated function called. Function 'hashSha256' has been deprecated, use the 'hash' function and pass the algorithm type instead!" 147 | ) 148 | return this.hash('sha256', progress) 149 | } 150 | 151 | /** 152 | * Return specified number of rows as stream of data 153 | * @param {boolean} keyed whether the file is keyed or not 154 | * @param {number} sheet The number of the sheet to load for excel files 155 | * @param {number} size The number of rows to return 156 | * 157 | * @returns {Stream} File stream 158 | */ 159 | rows({ keyed, sheet, size } = {}) { 160 | return this._rows({ keyed, sheet, size }) 161 | } 162 | 163 | _rows({ keyed, sheet, size } = {}) { 164 | if (this.descriptor.format in PARSE_DATABASE) { 165 | const parser = PARSE_DATABASE[this.descriptor.format] 166 | return parser(this, { keyed, sheet, size }) 167 | } 168 | throw new Error( 169 | `We do not have a parser for that format: ${this.descriptor.format}` 170 | ) 171 | } 172 | 173 | /** 174 | * Returns a small portion of a file stream as Objects 175 | */ 176 | getSample() { 177 | return new Promise(async (resolve, reject) => { 178 | let smallStream = await this.rows({ size: 100 }) 179 | resolve(await toArray(smallStream)) 180 | }) 181 | } 182 | 183 | async addSchema() { 184 | if (this.displayName === 'FileInline') { 185 | this.descriptor.schema = await infer(this.descriptor.data) 186 | return 187 | } 188 | let sample = await this.getSample() 189 | 190 | // Try to infer schema from sample data if given file is xlsx 191 | if (this.descriptor.format === 'xlsx' && sample) { 192 | let headers = 1 193 | if (isPlainObject(sample[0])) { 194 | headers = Object.keys(sample[0]) 195 | } 196 | this.descriptor.schema = await infer(sample, { headers }) 197 | return 198 | } 199 | 200 | // Ensure file is tabular 201 | if (KNOWN_TABULAR_FORMAT.indexOf(this.descriptor.format) === -1) { 202 | throw new Error('File is not in known tabular format.') 203 | } 204 | 205 | // Get parserOptions so we can use it when "infering" schema: 206 | const parserOptions = await guessParseOptions(this) 207 | // We also need to include parserOptions in "dialect" property of descriptor: 208 | this.descriptor.dialect = { 209 | delimiter: parserOptions.delimiter, 210 | quoteChar: parserOptions.quote, 211 | } 212 | // Now let's get a stream from file and infer schema: 213 | let thisFileStream = await this.stream({ size: 100 }) 214 | this.descriptor.schema = await infer(thisFileStream, parserOptions) 215 | } 216 | } 217 | 218 | /** 219 | * Computes the streaming hash of a file 220 | * @param {ReadableStream} fileStream A node like stream 221 | * @param {number} fileSize Total size of the file 222 | * @param {string} algorithm sha256/md5 hashing algorithm to use 223 | * @param {func} progress Callback function with progress 224 | * @param {"hex"|"base64"|"latin1"} encoding of resulting hash; Default is 'hex', other possible 225 | * values are 'base64' or 'binary'. 226 | * 227 | * @returns {Promise} the encoded digest value 228 | */ 229 | export function computeHash( 230 | fileStream, 231 | fileSize, 232 | algorithm, 233 | progress, 234 | encoding = 'hex' 235 | ) { 236 | return new Promise((resolve, reject) => { 237 | let hash = crypto.createHash(algorithm) 238 | let offset = 0 239 | let totalChunkSize = 0 240 | let chunkCount = 0 241 | 242 | if (!['hex', 'latin1', 'binary', 'base64'].includes(encoding)) { 243 | throw new Error( 244 | `Invalid encoding value: ${encoding}; Expecting 'hex', 'latin1', 'binary' or 'base64'` 245 | ) 246 | } 247 | 248 | //calculates progress after every 20th chunk 249 | const _reportProgress = new Transform({ 250 | transform(chunk, encoding, callback) { 251 | if (chunkCount % 20 == 0) { 252 | const runningTotal = totalChunkSize + offset 253 | const percentComplete = Math.round((runningTotal / fileSize) * 100) 254 | if (typeof progress === 'function') { 255 | progress(percentComplete) //callback with progress 256 | } 257 | } 258 | callback(null, chunk) 259 | }, 260 | }) 261 | 262 | fileStream 263 | .pipe(_reportProgress) 264 | .on('error', function (err) { 265 | reject(err) 266 | }) 267 | .on('data', function (chunk) { 268 | offset += chunk.length 269 | chunkCount += 1 270 | hash.update(chunk) 271 | }) 272 | .on('end', function () { 273 | hash = hash.digest(encoding) 274 | if (typeof progress === 'function') { 275 | progress(100) 276 | } 277 | resolve(hash) 278 | }) 279 | }) 280 | } 281 | -------------------------------------------------------------------------------- /src/file-inline.js: -------------------------------------------------------------------------------- 1 | import stream from 'stream' 2 | import { File } from './file-base' 3 | import { isString, isArray } from 'lodash' 4 | 5 | 6 | export class FileInline extends File { 7 | constructor(descriptor, { basePath } = {}) { 8 | super(descriptor, { basePath }) 9 | 10 | // JSON is special case ... 11 | if (isString(this.descriptor.data)) { 12 | // eslint-disable-next-line no-undef 13 | this._buffer = Buffer.from(this.descriptor.data) 14 | } else { 15 | // It is json/javascript 16 | // eslint-disable-next-line no-undef 17 | this._buffer = Buffer.from(JSON.stringify(this.descriptor.data)) 18 | } 19 | } 20 | 21 | get displayName() { 22 | return 'FileInline' 23 | } 24 | 25 | // Not really sure this should exist here ... - have it for tests atm 26 | get path() { 27 | return this.descriptor.path 28 | } 29 | 30 | get size() { 31 | return this._buffer.byteLength 32 | } 33 | 34 | 35 | stream() { 36 | const bufferStream = new stream.PassThrough() 37 | bufferStream.end(this._buffer) 38 | return bufferStream 39 | } 40 | 41 | rows({ keyed } = {}) { 42 | if (isArray(this.descriptor.data)) { 43 | const rowStream = new stream.PassThrough({ objectMode: true }) 44 | this.descriptor.data.forEach((row) => { 45 | rowStream.write(row) 46 | }) 47 | rowStream.end() 48 | return rowStream 49 | } 50 | return this._rows({ keyed, size }) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/file-interface.js: -------------------------------------------------------------------------------- 1 | import { DEFAULT_ENCODING } from './data' 2 | import { webToNodeStream } from './browser-utils/index' 3 | import { File } from './file-base' 4 | 5 | export class FileInterface extends File { 6 | constructor(descriptor, { basePath } = {}) { 7 | super(descriptor, { basePath }) 8 | this._descriptor.format = descriptor.name.split('.').pop() || '' 9 | } 10 | 11 | get displayName() { 12 | return 'FileInterface' 13 | } 14 | 15 | // create and return a path url 16 | get path() { 17 | return URL.createObjectURL(this.descriptor) 18 | } 19 | 20 | get encoding() { 21 | return this._encoding || DEFAULT_ENCODING 22 | } 23 | 24 | /** 25 | * Return the stream to a file 26 | * If the size is -1 then will read whole file 27 | */ 28 | async stream({ size } = {}) { 29 | return webToNodeStream(await this.descriptor.stream(), size) 30 | } 31 | 32 | get buffer() { 33 | return this.descriptor.arrayBuffer() 34 | } 35 | 36 | get size() { 37 | return this.descriptor.size 38 | } 39 | 40 | get fileName() { 41 | return this.descriptor.name 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/file-local.js: -------------------------------------------------------------------------------- 1 | import chardet from 'chardet' 2 | import fs from 'fs' 3 | import { File } from './file-base' 4 | import path from 'path' 5 | 6 | export class FileLocal extends File { 7 | get displayName() { 8 | return 'FileLocal' 9 | } 10 | 11 | get path() { 12 | return this._basePath 13 | ? path.join(this._basePath, this.descriptor.path) 14 | : this.descriptor.path 15 | } 16 | 17 | stream({ size } = {}) { 18 | let end = size 19 | return fs.createReadStream(this.path, { start: 0, end }) 20 | } 21 | 22 | get size() { 23 | return fs.statSync(this.path).size 24 | } 25 | 26 | get encoding() { 27 | // When data is huge, we want to optimize performace (in tradeoff of less accuracy): 28 | // So we are using sample of first 100K bytes here: 29 | if (this.size > 1000000) { 30 | return chardet.detectFileSync(this.path, { sampleSize: 1000000 }) 31 | } 32 | return chardet.detectFileSync(this.path) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/file-remote.js: -------------------------------------------------------------------------------- 1 | import urljoin from 'url-join' 2 | import fetch from 'node-fetch' 3 | import { File } from './file-base' 4 | import { isUrl } from './data' 5 | import { webToNodeStream } from './browser-utils/index' 6 | import { DEFAULT_ENCODING } from './data' 7 | 8 | export class FileRemote extends File { 9 | get displayName() { 10 | return 'FileRemote' 11 | } 12 | 13 | get path() { 14 | const isItUrl = isUrl(this.descriptor.path) 15 | if (isItUrl) { 16 | return this.descriptor.path 17 | } else { 18 | return this._basePath 19 | ? urljoin(this._basePath, this.descriptor.path) 20 | : this.descriptor.path 21 | } 22 | } 23 | 24 | get browserBuffer() { 25 | return (async () => { 26 | const res = await fetch(this.path) 27 | return await res.arrayBuffer() 28 | })() 29 | } 30 | 31 | stream({ size } = {}) { 32 | return (async () => { 33 | const res = await fetch(this.path) 34 | if (res.status === 200) { 35 | if (typeof window === 'undefined') { 36 | return res.body 37 | } else { 38 | return webToNodeStream(res.body, size) 39 | } 40 | } else { 41 | throw new Error( 42 | `${res.status}: ${res.statusText}. Requested URL: ${this.path}` 43 | ) 44 | } 45 | })() 46 | } 47 | 48 | get encoding() { 49 | return this._encoding || DEFAULT_ENCODING 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | export { 2 | open, 3 | parsePath, 4 | parseDatasetIdentifier, 5 | isDataset, 6 | isUrl, 7 | } from './data' 8 | export { File } from './file-base' 9 | export { FileInterface } from './file-interface' 10 | export { FileLocal } from './file-local' 11 | export { FileRemote } from './file-remote' 12 | export { FileInline } from './file-inline' 13 | export { Dataset } from './dataset' 14 | export { csvParser } from './parser/csv' 15 | export { xlsxParser } from './parser/xlsx' 16 | export { computeHash } from './file-base' 17 | -------------------------------------------------------------------------------- /src/parser/csv.js: -------------------------------------------------------------------------------- 1 | import parse from 'csv-parse' 2 | const CSVSniffer = require('csv-sniffer')() 3 | import toString from 'stream-to-string' 4 | import { decodeStream } from 'iconv-lite' 5 | 6 | export async function csvParser(file, { keyed = false, size } = {}) { 7 | const parseOptions = await getParseOptions(file, keyed) 8 | let stream = await file.stream({size}) 9 | if (file.descriptor.encoding.toLowerCase().replace('-', '') === 'utf8') { 10 | return stream.pipe(parse(parseOptions)) 11 | } else { 12 | // non utf-8 files are decoded by iconv-lite module 13 | return stream 14 | .pipe(decodeStream(file.descriptor.encoding)) 15 | .pipe(parse(parseOptions)) 16 | } 17 | } 18 | 19 | export async function guessParseOptions(file) { 20 | const possibleDelimiters = [',', ';', ':', '|', '\t', '^', '*', '&'] 21 | const sniffer = new CSVSniffer(possibleDelimiters) 22 | let text = '' 23 | 24 | // We assume that reading first 50K bytes is enough to detect delimiter, line terminator etc.: 25 | if (file.displayName === 'FileLocal') { 26 | const stream = await file.stream({ size: 50000 }) 27 | text = await toString(stream) 28 | } else if (file.displayName === 'FileInterface') { 29 | let stream = await file.stream({size: 10}) 30 | text = await toString(stream) 31 | } else if (file.displayName === 'FileRemote') { 32 | const stream = await file.stream({ size: 100 }) 33 | let bytes = 0 34 | 35 | await new Promise((resolve, reject) => { 36 | stream 37 | .on('data', (chunk) => { 38 | bytes += chunk.length 39 | if (bytes > 50000) { 40 | stream.pause() 41 | resolve() 42 | } else { 43 | text += chunk.toString() 44 | } 45 | }) 46 | .on('end', () => { 47 | resolve() 48 | }) 49 | }) 50 | } 51 | 52 | const results = sniffer.sniff(text) 53 | 54 | return { 55 | delimiter: results.delimiter, 56 | quote: results.quoteChar || '"', 57 | } 58 | } 59 | 60 | export async function getParseOptions(file, keyed) { 61 | let parseOptions = { 62 | columns: keyed ? true : null, 63 | ltrim: true, 64 | } 65 | 66 | if (file.descriptor.dialect) { 67 | parseOptions.delimiter = file.descriptor.dialect.delimiter || ',' 68 | parseOptions.rowDelimiter = file.descriptor.dialect.lineTerminator 69 | parseOptions.quote = file.descriptor.dialect.quoteChar || '"' 70 | 71 | if ( 72 | file.descriptor.dialect.doubleQuote !== undefined && 73 | file.descriptor.dialect.doubleQuote === false 74 | ) { 75 | parseOptions.escape = '' 76 | } 77 | } else { 78 | const guessedParseOptions = await guessParseOptions(file) 79 | // Merge guessed parse options with default one: 80 | parseOptions = Object.assign(parseOptions, guessedParseOptions) 81 | } 82 | 83 | return parseOptions 84 | } 85 | 86 | /** 87 | * This transformer takes binary Uint8Array chunks from a `fetch` 88 | * and translates them to chunks of strings. 89 | * 90 | * @implements {TransformStreamTransformer} 91 | */ 92 | export class Uint8ArrayToStringsTransformer { 93 | constructor() { 94 | this.decoder = new TextDecoder() 95 | this.lastString = '' 96 | } 97 | 98 | /** 99 | * Receives the next Uint8Array chunk from `fetch` and transforms it. 100 | * 101 | * @param {Uint8Array} chunk The next binary data chunk. 102 | * @param {TransformStreamDefaultController} controller The controller to enqueue the transformed chunks to. 103 | */ 104 | transform(chunk, controller) { 105 | // Decode the current chunk to string and prepend the last string 106 | const string = `${this.lastString}${this.decoder.decode(chunk)}` 107 | 108 | // Extract lines from chunk 109 | const lines = string.split(/\r\n|[\r\n]/g) 110 | 111 | // Save last line, as it might be incomplete 112 | this.lastString = lines.pop() || '' 113 | 114 | // Enqueue each line in the next chunk 115 | for (const line of lines) { 116 | controller.enqueue(line) 117 | } 118 | } 119 | 120 | /** 121 | * Is called when `fetch` has finished writing to this transform stream. 122 | * 123 | * @param {TransformStreamDefaultController} controller The controller to enqueue the transformed chunks to. 124 | */ 125 | flush(controller) { 126 | // Is there still a line left? Enqueue it 127 | if (this.lastString) { 128 | controller.enqueue(this.lastString) 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/parser/xlsx.js: -------------------------------------------------------------------------------- 1 | import { Readable } from 'stream' 2 | import { read, utils } from 'xlsx' 3 | import parse from 'csv-parse' 4 | 5 | export async function xlsxParser(file, keyed = false, sheetIdxOrName = 0) { 6 | let buffer 7 | 8 | if (typeof window === 'undefined' || file.displayName === 'FileInterface') { 9 | buffer = await file.buffer 10 | } else { 11 | // Running in browser 12 | buffer = await file.browserBuffer 13 | } 14 | 15 | const workbook = read(buffer, { type: 'buffer' }) 16 | let selectedSheetName = sheetIdxOrName 17 | 18 | if (sheetIdxOrName.constructor.name === 'Number') { 19 | selectedSheetName = workbook.SheetNames[sheetIdxOrName] 20 | } 21 | 22 | const sheet = workbook.Sheets[selectedSheetName] 23 | const csv = utils.sheet_to_csv(sheet) 24 | const stream = new Readable() 25 | 26 | stream.push(csv) 27 | stream.push(null) 28 | 29 | return stream.pipe( 30 | parse({ 31 | columns: keyed ? true : null, 32 | ltrim: true, 33 | }) 34 | ) 35 | } 36 | -------------------------------------------------------------------------------- /test/data.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import path from 'path' 3 | import nock from 'nock' 4 | import * as data from '../src/data' 5 | 6 | describe('isUrl', () => { 7 | it('Tests if given path is url or not', async () => { 8 | let notUrl = 'not/url/path' 9 | let res = data.isUrl(notUrl) 10 | assert.strictEqual(res, false) 11 | 12 | notUrl = '/not/url/path/' 13 | res = data.isUrl(notUrl) 14 | assert.strictEqual(res, false) 15 | 16 | let url = 'https://test.com' 17 | res = data.isUrl(url) 18 | assert.strictEqual(res, true) 19 | 20 | url = 'http://test.com' 21 | res = data.isUrl(url) 22 | assert.strictEqual(res, true) 23 | 24 | url = 'HTTP://TEST.COM' 25 | res = data.isUrl(url) 26 | assert.strictEqual(res, true) 27 | 28 | url = '//test.com' 29 | res = data.isUrl(url) 30 | assert.strictEqual(res, true) 31 | }) 32 | }) 33 | 34 | describe('parseDatasetIdentifier', () => { 35 | it('parseDatasetIdentifier function with local path', async () => { 36 | const paths = [ 37 | '../dir/dataset/', 38 | './', 39 | './datapackage.json', 40 | '../datapackage.json', 41 | 'datapackage.json', 42 | ] 43 | paths.forEach(async (path_) => { 44 | const res = await data.parseDatasetIdentifier(path_) 45 | const normalizedPath = path.posix 46 | .resolve(path_) 47 | .replace(/\/?datapackage\.json/, '') 48 | const exp = { 49 | name: path.basename(normalizedPath), 50 | owner: null, 51 | path: normalizedPath, 52 | type: 'local', 53 | original: path_, 54 | version: '', 55 | } 56 | assert.deepStrictEqual(res, exp) 57 | }) 58 | }) 59 | it('parseDatasetIdentifier function with random url', async () => { 60 | const url_ = 'https://example.com/datasets/co2-ppm/' 61 | const res = await data.parseDatasetIdentifier(url_) 62 | const exp = { 63 | name: 'co2-ppm', 64 | owner: null, 65 | path: 'https://example.com/datasets/co2-ppm/', 66 | type: 'url', 67 | original: url_, 68 | version: '', 69 | } 70 | assert.deepStrictEqual(res, exp) 71 | }) 72 | it('parseDatasetIdentifier function with github url', async () => { 73 | const url_ = 'https://github.com/datasets/co2-ppm' 74 | const res = await data.parseDatasetIdentifier(url_) 75 | const exp = { 76 | name: 'co2-ppm', 77 | owner: 'datasets', 78 | path: 'https://raw.githubusercontent.com/datasets/co2-ppm/master/', 79 | type: 'github', 80 | original: url_, 81 | version: 'master', 82 | } 83 | assert.deepStrictEqual(res, exp) 84 | }) 85 | it('parseDatasetIdentifier function with datahub url', async () => { 86 | nock('https://api.datahub.io') 87 | .persist() 88 | .get('/resolver/resolve?path=core/co2-ppm') 89 | .reply(200, { packageid: 'co2-ppm', userid: 'core' }) 90 | .get('/source/core/co2-ppm/successful') 91 | .reply(200, { id: 'core/co2-ppm/3' }) 92 | 93 | const url_ = 'https://datahub.io/core/co2-ppm' 94 | const res = await data.parseDatasetIdentifier(url_) 95 | const exp = { 96 | name: 'co2-ppm', 97 | owner: 'core', 98 | path: 'https://pkgstore.datahub.io/core/co2-ppm/3/', 99 | type: 'datahub', 100 | original: url_, 101 | version: 3, 102 | } 103 | assert.deepStrictEqual(res, exp) 104 | }) 105 | it('parseDatasetIdentifier function with datahub url and id different from username', async () => { 106 | nock('https://api.datahub.io') 107 | .persist() 108 | .get('/resolver/resolve?path=username/dataset') 109 | .reply(200, { packageid: 'dataset', userid: 'userid' }) 110 | .get('/source/userid/dataset/successful') 111 | .reply(200, { id: 'userid/dataset/2' }) 112 | 113 | const url_ = 'https://datahub.io/username/dataset' 114 | const res = await data.parseDatasetIdentifier(url_) 115 | const exp = { 116 | name: 'dataset', 117 | owner: 'username', 118 | path: 'https://pkgstore.datahub.io/userid/dataset/2/', 119 | type: 'datahub', 120 | original: url_, 121 | version: 2, 122 | } 123 | assert.deepStrictEqual(res, exp) 124 | }) 125 | }) 126 | 127 | describe('parsePath', () => { 128 | it("parsePath function works with local path'", async () => { 129 | const path_ = 'datajs/test/fixtures/sample.csv' 130 | const res = data.parsePath(path_) 131 | assert.strictEqual(res.path, path_) 132 | assert.strictEqual(res.pathType, 'local') 133 | assert.strictEqual(res.name, 'sample') 134 | assert.strictEqual(res.format, 'csv') 135 | assert.strictEqual(res.mediatype, 'text/csv') 136 | }) 137 | 138 | it('parsePath function works with remote url', async () => { 139 | const path_ = 140 | 'https://raw.githubusercontent.com/datasets/finance-vix/master/data/vix-daily.csv' 141 | const res = data.parsePath(path_) 142 | assert.strictEqual(res.path, path_) 143 | assert.strictEqual(res.pathType, 'remote') 144 | assert.strictEqual(res.name, 'vix-daily') 145 | assert.strictEqual(res.format, 'csv') 146 | assert.strictEqual(res.mediatype, 'text/csv') 147 | }) 148 | 149 | it('parsePath function with remote url without conventional filename', async () => { 150 | const path_ = 151 | 'http://api.worldbank.org/indicator/NY.GDP.MKTP.CD?format=csv' 152 | const res = data.parsePath(path_) 153 | assert.strictEqual(res.path, path_) 154 | assert.strictEqual(res.pathType, 'remote') 155 | assert.strictEqual(res.name, 'ny.gdp.mktp') 156 | assert.strictEqual(res.format, 'csv') 157 | assert.strictEqual(res.mediatype, undefined) 158 | }) 159 | }) 160 | 161 | describe('isDataset', () => { 162 | it('isDataset function works on different examples', async () => { 163 | let pathToDataset = 'test/fixtures/co2-ppm' 164 | let res = data.isDataset(pathToDataset) 165 | assert.strictEqual(res, true) 166 | 167 | pathToDataset = 'test/fixtures/co2-ppm/datapackage.json' 168 | res = data.isDataset(pathToDataset) 169 | assert.strictEqual(res, true) 170 | 171 | const pathToFile = 'test/fixtures/sample.csv' 172 | res = data.isDataset(pathToFile) 173 | assert.strictEqual(res, false) 174 | 175 | let urlToDataset = 'http://test.com/' 176 | res = data.isDataset(urlToDataset) 177 | assert.strictEqual(res, false) 178 | 179 | urlToDataset = 'http://test.com/dir' 180 | res = data.isDataset(urlToDataset) 181 | assert.strictEqual(res, false) 182 | 183 | urlToDataset = 'http://test.com/dir/datapackage.json' 184 | res = data.isDataset(urlToDataset) 185 | assert.strictEqual(res, true) 186 | 187 | let urlToFile = 'http://test.com/dir/file.csv' 188 | res = data.isDataset(urlToFile) 189 | assert.strictEqual(res, false) 190 | 191 | urlToFile = 'http://test.com/file.csv' 192 | res = data.isDataset(urlToFile) 193 | assert.strictEqual(res, false) 194 | }) 195 | }) 196 | -------------------------------------------------------------------------------- /test/dataset.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import path from 'path' 3 | import nock from 'nock' 4 | import * as data from '../src/data' 5 | import { Dataset } from '../src/dataset' 6 | 7 | describe('Dataset', () => { 8 | it('Dataset constructor works', async () => { 9 | const dataset = new Dataset() 10 | assert.deepStrictEqual(dataset.identifier, { 11 | path: null, 12 | owner: null, 13 | }) 14 | 15 | assert.deepStrictEqual(dataset.descriptor, {}) 16 | assert.deepStrictEqual(dataset.path, null) 17 | assert.deepStrictEqual(dataset.readme, undefined) 18 | }) 19 | 20 | it('Dataset with inline README works', async () => { 21 | const path = 'test/fixtures/dp-with-inline-readme' 22 | const dataset = await Dataset.load(path) 23 | assert.deepStrictEqual(dataset.identifier.type, 'local') 24 | assert.deepStrictEqual(dataset.readme, 'This is the README') 25 | }) 26 | 27 | it('Dataset.load works with co2-ppm', async () => { 28 | const path = 'test/fixtures/co2-ppm' 29 | const dataset2 = await Dataset.load(path) 30 | assert.strictEqual(dataset2.identifier.type, 'local') 31 | 32 | assert.strictEqual(dataset2.descriptor.name, 'co2-ppm') 33 | assert.strictEqual(dataset2.resources.length, 6) 34 | assert.strictEqual(dataset2.resources[0].descriptor.name, 'co2-mm-mlo') 35 | assert.strictEqual( 36 | dataset2.resources[0].path.includes( 37 | 'test/fixtures/co2-ppm/data/co2-mm-mlo.csv' 38 | ), 39 | true 40 | ) 41 | assert.strictEqual( 42 | dataset2.readme.includes( 43 | 'CO2 PPM - Trends in Atmospheric Carbon Dioxide.' 44 | ), 45 | true 46 | ) 47 | }) 48 | 49 | it('Dataset.load with dir/datapckage.json', async () => { 50 | const path = 'test/fixtures/co2-ppm/datapackage.json' 51 | const dataset = await Dataset.load(path) 52 | assert.strictEqual(dataset.descriptor.name, 'co2-ppm') 53 | assert.strictEqual(dataset.identifier.type, 'local') 54 | assert.strictEqual(dataset.resources.length, 6) 55 | assert.strictEqual( 56 | dataset.readme.includes( 57 | 'CO2 PPM - Trends in Atmospheric Carbon Dioxide.' 58 | ), 59 | true 60 | ) 61 | }) 62 | 63 | it('Dataset.load with url/datapackage.json', async () => { 64 | const url = 65 | 'https://raw.githubusercontent.com/datasets/co2-ppm/master/datapackage.json' 66 | nock('https://raw.githubusercontent.com') 67 | .persist() 68 | .get('/datasets/co2-ppm/master/datapackage.json') 69 | .replyWithFile( 70 | 200, 71 | path.join(__dirname, '/fixtures/co2-ppm/datapackage.json') 72 | ) 73 | 74 | nock('https://raw.githubusercontent.com') 75 | .persist() 76 | .get('/datasets/co2-ppm/master/README.md') 77 | .replyWithFile(200, path.join(__dirname, '/fixtures/co2-ppm/README.md')) 78 | 79 | const dataset = await Dataset.load(url) 80 | assert.strictEqual(dataset.descriptor.name, 'co2-ppm') 81 | assert.strictEqual(dataset.identifier.type, 'url') 82 | assert.strictEqual(dataset.resources.length, 6) 83 | assert.strictEqual( 84 | dataset.readme.includes( 85 | 'CO2 PPM - Trends in Atmospheric Carbon Dioxide.' 86 | ), 87 | true 88 | ) 89 | }) 90 | 91 | it('Dataset.addResource method works', async () => { 92 | const resourceAsPlainObj = { 93 | name: 'sample', 94 | path: 'test/fixtures/sample.csv', 95 | format: 'csv', 96 | } 97 | const resourceAsFileObj = data.open(resourceAsPlainObj) 98 | const dataset = new Dataset({ resources: [] }) 99 | assert.strictEqual(dataset.resources.length, 0) 100 | assert.strictEqual(dataset.descriptor.resources.length, 0) 101 | 102 | dataset.addResource(resourceAsPlainObj) 103 | assert.strictEqual(dataset.resources.length, 1) 104 | assert.strictEqual(dataset.descriptor.resources.length, 1) 105 | 106 | dataset.addResource(resourceAsFileObj) 107 | assert.strictEqual(dataset.resources.length, 2) 108 | assert.strictEqual(dataset.descriptor.resources.length, 2) 109 | }) 110 | }) 111 | 112 | describe('Dataset.nock', function () { 113 | this.timeout(30000) // all tests in this suite get 10 seconds before timeout 114 | 115 | const url = 'https://raw.githubusercontent.com/datasets/co2-ppm/master/' 116 | it('Dataset.load with url-directory', async () => { 117 | nock('https://raw.githubusercontent.com') 118 | .persist() 119 | .get('/datasets/co2-ppm/master/datapackage.json') 120 | .replyWithFile( 121 | 200, 122 | path.join(__dirname, '/fixtures/co2-ppm/datapackage.json') 123 | ) 124 | 125 | nock('https://raw.githubusercontent.com') 126 | .persist() 127 | .get('/datasets/co2-ppm/master/README.md') 128 | .replyWithFile(200, path.join(__dirname, '/fixtures/co2-ppm/README.md')) 129 | 130 | // Added mocking for all remote files in the test dataset 131 | // Reason: FileRemote is now probing the remote resource to define its encoding 132 | nock('https://raw.githubusercontent.com') 133 | .persist() 134 | .get('/datasets/co2-ppm/master/data/co2-mm-mlo.csv') 135 | .replyWithFile( 136 | 200, 137 | path.join(__dirname, '/fixtures/co2-ppm/data/co2-mm-mlo.csv') 138 | ) 139 | 140 | nock('https://raw.githubusercontent.com') 141 | .persist() 142 | .get('/datasets/co2-ppm/master/data/co2-annmean-mlo.csv') 143 | .replyWithFile( 144 | 200, 145 | path.join(__dirname, '/fixtures/co2-ppm/data/co2-annmean-mlo.csv') 146 | ) 147 | 148 | nock('https://raw.githubusercontent.com') 149 | .persist() 150 | .get('/datasets/co2-ppm/master/data/co2-gr-mlo.csv') 151 | .replyWithFile( 152 | 200, 153 | path.join(__dirname, '/fixtures/co2-ppm/data/co2-gr-mlo.csv') 154 | ) 155 | 156 | nock('https://raw.githubusercontent.com') 157 | .persist() 158 | .get('/datasets/co2-ppm/master/data/co2-mm-gl.csv') 159 | .replyWithFile( 160 | 200, 161 | path.join(__dirname, '/fixtures/co2-ppm/data/co2-mm-gl.csv') 162 | ) 163 | 164 | nock('https://raw.githubusercontent.com') 165 | .persist() 166 | .get('/datasets/co2-ppm/master/data/co2-annmean-gl.csv') 167 | .replyWithFile( 168 | 200, 169 | path.join(__dirname, '/fixtures/co2-ppm/data/co2-annmean-gl.csv') 170 | ) 171 | 172 | nock('https://raw.githubusercontent.com') 173 | .persist() 174 | .get('/datasets/co2-ppm/master/data/co2-gr-gl.csv') 175 | .replyWithFile( 176 | 200, 177 | path.join(__dirname, '/fixtures/co2-ppm/data/co2-gr-gl.csv') 178 | ) 179 | 180 | nock('https://raw.githubusercontent.com') 181 | .persist() 182 | .get('/datasets/co2-ppm/master/datapackage.json') 183 | .replyWithFile( 184 | 200, 185 | path.join(__dirname, '/fixtures/co2-ppm/datapackage.json') 186 | ) 187 | 188 | nock.cleanAll() 189 | 190 | const dataset = await Dataset.load(url) 191 | assert.strictEqual(dataset.descriptor.name, 'co2-ppm') 192 | assert.strictEqual(dataset.identifier.type, 'url') 193 | assert.strictEqual(dataset.resources.length, 6) 194 | assert.strictEqual( 195 | dataset.readme.includes( 196 | 'CO2 PPM - Trends in Atmospheric Carbon Dioxide.' 197 | ), 198 | true 199 | ) 200 | }) 201 | }) 202 | -------------------------------------------------------------------------------- /test/file-base.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import * as data from '../src/data' 3 | import toArray from 'stream-to-array' 4 | import { File, computeHash } from '../src/file-base' 5 | // common method to test all the functionality which we can use for all types 6 | // of files 7 | export const testFile = async (assert, file) => { 8 | assert.strictEqual(file.path, 'test/fixtures/sample.csv') 9 | assert.strictEqual(file.size, 46) 10 | await testFileStream(assert, file) 11 | } 12 | 13 | export const testFileStream = async (assert, file) => { 14 | // Test stream 15 | const stream = await file.stream() 16 | const out = await toArray(stream) 17 | assert.strictEqual(out.toString().includes('number,string,boolean'), true) 18 | 19 | // Test buffer 20 | const buffer = await file.buffer 21 | assert.strictEqual(buffer.toString().slice(0, 21), 'number,string,boolean') 22 | 23 | // Test rows 24 | const rowStream = await file.rows() 25 | const rows = await toArray(rowStream) 26 | assert.deepStrictEqual(rows[0], ['number', 'string', 'boolean']) 27 | assert.deepStrictEqual(rows[1], ['1', 'two', 'true']) 28 | 29 | // Test rows with keyed option (rows as objects) 30 | const rowStreamKeyed = await file.rows({ keyed: true }) 31 | const rowsAsObjects = await toArray(rowStreamKeyed) 32 | assert.deepStrictEqual(rowsAsObjects[0], { 33 | number: '1', 34 | string: 'two', 35 | boolean: 'true', 36 | }) 37 | assert.deepStrictEqual(rowsAsObjects[1], { 38 | number: '3', 39 | string: 'four', 40 | boolean: 'false', 41 | }) 42 | } 43 | 44 | // testing the stream or the buffer for non-utf8 encoding will not work, 45 | // as we moved the stream decoding from the data.js lib, 46 | // so here we now testing if File.encoding property is correct 47 | 48 | describe('File Base', async () => { 49 | it('cyrillic encoding is working', () => { 50 | const path_ = 'test/fixtures/sample-cyrillic-encoding.csv' 51 | const file = data.open(path_) 52 | //const buffer = await file.buffer 53 | //t.is(buffer.toString().slice(0, 12), 'номер, город') 54 | assert.strictEqual(file.encoding, 'windows-1251') 55 | }) 56 | 57 | it('File class with path', async () => { 58 | // With path 59 | const path_ = 'test/fixtures/sample.csv' 60 | const res = data.open(path_) 61 | await testFile(assert, res) 62 | }) 63 | 64 | it('File class with descriptor', async () => { 65 | const descriptor = { path: 'test/fixtures/sample.csv' } 66 | const obj2 = data.open(descriptor) 67 | await testFile(assert, obj2) 68 | }) 69 | 70 | it('File with path and basePath', async () => { 71 | const obj3 = data.open('sample.csv', { basePath: 'test/fixtures' }) 72 | testFile(assert, obj3) 73 | }) 74 | 75 | it('File name has spaces and dots', async () => { 76 | let path_ = 'test/fixtures/some file.name.ext' 77 | let file = File.load(path_) 78 | assert.strictEqual(file.descriptor.name, 'some-file.name') 79 | }) 80 | }) 81 | 82 | describe('bufferInChunks', () => { 83 | it('File is loaded in chunks', async () => { 84 | const path_ = 'test/fixtures/sample-cyrillic-encoding.csv' 85 | const file = data.open(path_) 86 | 87 | file.bufferInChunks((chunk, percent) => { 88 | assert.strictEqual(chunk.length, 40) 89 | assert.strictEqual(typeof percent, 'number') 90 | }) 91 | }) 92 | }) 93 | 94 | describe('computeHash', () => { 95 | it('compute hash256 of a file stream', async () => { 96 | const path_ = 'test/fixtures/sample-cyrillic-encoding.csv' 97 | const file = data.open(path_) 98 | 99 | let hash256 = await computeHash(file.stream(), file.size, 'sha256') 100 | let hashmd5 = await computeHash(file.stream(), file.size, 'md5') 101 | 102 | assert.strictEqual( 103 | hash256, 104 | '8eff5a7815864615309d48035b461b79aa1bdc4402924e97fc66e123725214fd' 105 | ) 106 | assert.strictEqual(hashmd5, '37d3a5159433f0977afb03d01d4bde6e') 107 | }) 108 | 109 | it('encodes the hash using base64', async () => { 110 | const path_ = 'test/fixtures/sample-cyrillic-encoding.csv' 111 | const file = data.open(path_) 112 | 113 | let hash256 = await computeHash(file.stream(), file.size, 'sha256', null, 'base64') 114 | 115 | assert.strictEqual( 116 | hash256, 117 | 'jv9aeBWGRhUwnUgDW0Ybeaob3EQCkk6X/GbhI3JSFP0=' 118 | ) 119 | }) 120 | }) 121 | 122 | describe('hashSha256', () => { 123 | it('hashSha256 returns right hash', async () => { 124 | const path_ = 'test/fixtures/sample-cyrillic-encoding.csv' 125 | const file = data.open(path_) 126 | 127 | let hash = await file.hashSha256() 128 | assert.strictEqual( 129 | hash, 130 | '8eff5a7815864615309d48035b461b79aa1bdc4402924e97fc66e123725214fd' 131 | ) 132 | }) 133 | }) 134 | 135 | describe('cached hash', () => { 136 | it('hashing the same file twice generates the same result', async () => { 137 | const path_ = 'test/fixtures/sample-cyrillic-encoding.csv' 138 | const file = data.open(path_) 139 | 140 | let hash1 = await file.hash() 141 | let hash2 = await file.hash() 142 | 143 | assert.strictEqual(hash1, hash2) 144 | }) 145 | }) 146 | 147 | 148 | describe('getSample', () => { 149 | it('returns in object for a stream', async () => { 150 | const path_ = 'test/fixtures/sample.xlsx' 151 | const file = data.open(path_) 152 | 153 | let rows = await file.getSample() 154 | assert.strictEqual(rows.length, 2) 155 | }) 156 | }) 157 | -------------------------------------------------------------------------------- /test/file-inline.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import * as data from '../src/data' 3 | import toArray from 'stream-to-array' 4 | import { testFile } from './file-base' 5 | 6 | describe('File-Inline', () => { 7 | it('File with inline JS data', async () => { 8 | const inlineData = { 9 | name: 'abc', 10 | } 11 | const file = data.open({ data: inlineData }) 12 | const stream = await file.stream() 13 | const out = await toArray(stream) 14 | 15 | assert.strictEqual(file.size, 14) 16 | assert.strictEqual(out.toString(), JSON.stringify(inlineData)) 17 | }) 18 | 19 | it('File with inline text (CSV) data', async () => { 20 | const inlineData = `number,string,boolean\n1,two,true\n3,four,false` 21 | // To make it testable with testFile we add the path but it is not needed 22 | const file = data.open({ 23 | path: 'test/fixtures/sample.csv', 24 | format: 'csv', 25 | inlineData, 26 | }) 27 | await testFile(assert, file) 28 | }) 29 | 30 | it('File with inline array data', async () => { 31 | const inlineData = [ 32 | ['number', 'string', 'boolean'], 33 | [1, 'two', true], 34 | [3, 'four', false], 35 | ] 36 | // To make it testable with testFile we add the path but it is not needed 37 | const file = data.open({ 38 | data: inlineData, 39 | }) 40 | 41 | const stream = await file.stream() 42 | const out = await toArray(stream) 43 | const rows = await file.rows() 44 | const out2 = await toArray(rows) 45 | 46 | assert.strictEqual(file.size, 63) 47 | assert.strictEqual(out.toString(), JSON.stringify(inlineData)) 48 | assert.strictEqual(out2.length, 3) 49 | assert.strictEqual(out2[0][0], inlineData[0][0]) 50 | // For some reason this fails with no difference 51 | // assert.strictEqual(out2, data) 52 | // but this works ... 53 | assert.strictEqual(JSON.stringify(out2), JSON.stringify(inlineData)) 54 | }) 55 | 56 | it('addSchema method', async () => { 57 | const inlineData = [ 58 | ['number', 'string', 'boolean'], 59 | [1, 'two', true], 60 | [3, 'four', false], 61 | ] 62 | let file = data.open({ 63 | data: inlineData, 64 | format: 'csv', 65 | }) 66 | assert.strictEqual(file.descriptor.schema, undefined) 67 | 68 | await file.addSchema() 69 | assert.strictEqual(file.descriptor.schema.fields[1].type, 'string') 70 | 71 | let headers = file.descriptor.schema.fields.map((field) => field.name) 72 | assert.deepStrictEqual(headers, ['number', 'string', 'boolean']) 73 | }) 74 | }) 75 | -------------------------------------------------------------------------------- /test/file-local.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import * as data from '../src/data' 3 | import toArray from 'stream-to-array' 4 | 5 | describe('File Local', function () { 6 | it('File class for addSchema method', async () => { 7 | let path_ = 'test/fixtures/sample.csv' 8 | let file = data.open(path_) 9 | assert.strictEqual(file.descriptor.schema, undefined) 10 | 11 | await file.addSchema() 12 | assert.strictEqual(file.descriptor.schema.fields[1].type, 'string') 13 | 14 | let headers = file.descriptor.schema.fields.map((field) => field.name) 15 | assert.deepStrictEqual(headers, ['number', 'string', 'boolean']) 16 | }) 17 | 18 | it("File classes have displayName method'", () => { 19 | const fileLocal = data.open('test/fixtures/sample.csv') 20 | assert.strictEqual(fileLocal.displayName, 'FileLocal') 21 | }) 22 | 23 | it('Calculates the streaming hash (md5) of a csv file', async () => { 24 | const fileLocal = data.open('test/fixtures/sample.csv') 25 | let hash = await fileLocal.hash('md5') 26 | assert.strictEqual(hash, 'b0661d9566498a800fbf95365ce28747') 27 | }) 28 | 29 | it('Calculates the streaming hash (sha256) of a csv file', async () => { 30 | const fileLocal = data.open('test/fixtures/sample.csv') 31 | let hash = await fileLocal.hash('sha256') 32 | assert.strictEqual( 33 | hash, 34 | 'd9d47b90ac9607c5111ff9a83aa37bc10e058ce6206c00b6626ade091784e098' 35 | ) 36 | }) 37 | 38 | it('Test that size of local stream works', async () => { 39 | const file = data.open('test/fixtures/sample.csv') 40 | const stream = await file.stream({ size: 20 }) 41 | assert.strictEqual(stream.end, 20) 42 | }) 43 | 44 | }) 45 | -------------------------------------------------------------------------------- /test/file-remote.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import path from 'path' 3 | import nock from 'nock' 4 | import * as data from '../src/data' 5 | import { testFileStream } from './file-base' 6 | 7 | describe('File Remote', function () { 8 | this.timeout(50000) // all tests in this suite get 10 seconds before timeout 9 | 10 | it('File class stream with url', async () => { 11 | const url = 12 | 'https://raw.githubusercontent.com/datahq/datahub-cli/master/test/fixtures/sample.csv' 13 | nock('https://raw.githubusercontent.com') 14 | .get('/datahq/datahub-cli/master/test/fixtures/sample.csv') 15 | .replyWithFile(200, path.join(__dirname, '/fixtures/sample.csv')) 16 | 17 | nock.cleanAll() 18 | 19 | const file = data.open(url) 20 | await testFileStream(assert, file) 21 | }) 22 | 23 | it('File class for addSchema method', async () => { 24 | let path_ = 25 | 'https://raw.githubusercontent.com/datahq/datahub-cli/master/test/fixtures/sample.csv' 26 | let file = data.open(path_) 27 | assert.strictEqual(file.descriptor.schema, undefined) 28 | 29 | await file.addSchema() 30 | assert.strictEqual(file.descriptor.schema.fields[1].type, 'string') 31 | 32 | let headers = file.descriptor.schema.fields.map((field) => field.name) 33 | assert.deepStrictEqual(headers, ['number', 'string', 'boolean']) 34 | }) 35 | 36 | it('File classes have displayName method', async () => { 37 | const fileRemote = data.open( 38 | 'https://raw.githubusercontent.com/datahq/datahub-cli/master/test/fixtures/sample.csv' 39 | ) 40 | assert.strictEqual(fileRemote.displayName, 'FileRemote') 41 | }) 42 | }) 43 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/README.md: -------------------------------------------------------------------------------- 1 | CO2 PPM - Trends in Atmospheric Carbon Dioxide. Data are sourced from the US Government's Earth System Research Laboratory, Global Monitoring Division. Two main series are provided: the Mauna Loa series (which has the longest continuous series since 1958) and a Global Average series (a global average over marine surface sites). 2 | 3 | ## Data 4 | 5 | ### Description 6 | 7 | > Data are reported as a dry air mole fraction defined as the number of molecules of carbon dioxide divided by the number of all molecules in air, including CO2 itself, after water vapor has been removed. The mole fraction is expressed as parts per million (ppm). Example: 0.000400 is expressed as 400 ppm.[*][ccgg-trends] 8 | 9 | ### Citations 10 | 11 | 1. *Trends in Atmospheric Carbon Dioxide, Mauna Loa, Hawaii.* Dr. Pieter Tans, NOAA/ESRL (www.esrl.noaa.gov/gmd/ccgg/trends/) and Dr. Ralph Keeling, Scripps Institution of Oceanography (scrippsco2.ucsd.edu/). 12 | 1. *Trends in Atmospheric Carbon Dioxide, Global.* Ed Dlugokencky and Pieter Tans, NOAA/ESRL (www.esrl.noaa.gov/gmd/ccgg/trends/). 13 | 14 | ### Sources 15 | 16 | 1. 17 | * Name: Trends in Atmospheric Carbon Dioxide, Mauna Loa, Hawaii 18 | * Web: http://www.esrl.noaa.gov/gmd/ccgg/trends/index.html 19 | 1. 20 | * Name: Trends in Atmospheric Carbon Dioxide, Global 21 | * Web: http://www.esrl.noaa.gov/gmd/ccgg/trends/global.html 22 | 23 | ## Data Preparation 24 | 25 | ### Processing 26 | 27 | Run the following script from this directory to download and process the data: 28 | 29 | ```bash 30 | make data 31 | ``` 32 | 33 | ### Resources 34 | 35 | The raw data are output to `./tmp`. The processed data are output to `./data`. 36 | 37 | ## License 38 | 39 | ### ODC-PDDL-1.0 40 | 41 | This Data Package is made available under the Public Domain Dedication and License v1.0 whose full text can be found at: http://www.opendatacommons.org/licenses/pddl/1.0/ 42 | 43 | ### Notes 44 | 45 | The [terms of use][gmd] of the source dataset list three specific restrictions on public use of these data: 46 | 47 | > The information on government servers are in the public domain, unless specifically annotated otherwise, and may be used freely by the public so long as you do not 1) claim it is your own (e.g. by claiming copyright for NOAA information – see next paragraph), 2) use it in a manner that implies an endorsement or affiliation with NOAA, or 3) modify it in content and then present it as official government material.[*][gmd] 48 | 49 | [ccgg-trends]: http://www.esrl.noaa.gov/gmd/ccgg/trends/index.html 50 | [gmd]: http://www.esrl.noaa.gov/gmd/about/disclaimer.html 51 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/data/co2-annmean-gl.csv: -------------------------------------------------------------------------------- 1 | Year,Mean,Uncertainty 2 | 1980-01-01,338.80,0.10 3 | 1981-01-01,339.99,0.10 4 | 1982-01-01,340.76,0.10 5 | 1983-01-01,342.43,0.10 6 | 1984-01-01,343.98,0.10 7 | 1985-01-01,345.46,0.10 8 | 1986-01-01,346.88,0.10 9 | 1987-01-01,348.62,0.10 10 | 1988-01-01,351.14,0.10 11 | 1989-01-01,352.79,0.10 12 | 1990-01-01,353.96,0.10 13 | 1991-01-01,355.29,0.10 14 | 1992-01-01,355.99,0.10 15 | 1993-01-01,356.71,0.10 16 | 1994-01-01,358.20,0.10 17 | 1995-01-01,360.02,0.10 18 | 1996-01-01,361.79,0.10 19 | 1997-01-01,362.90,0.10 20 | 1998-01-01,365.55,0.10 21 | 1999-01-01,367.63,0.10 22 | 2000-01-01,368.81,0.10 23 | 2001-01-01,370.40,0.10 24 | 2002-01-01,372.42,0.10 25 | 2003-01-01,374.97,0.10 26 | 2004-01-01,376.78,0.10 27 | 2005-01-01,378.81,0.10 28 | 2006-01-01,380.93,0.10 29 | 2007-01-01,382.67,0.10 30 | 2008-01-01,384.78,0.10 31 | 2009-01-01,386.28,0.10 32 | 2010-01-01,388.56,0.10 33 | 2011-01-01,390.44,0.10 34 | 2012-01-01,392.45,0.10 35 | 2013-01-01,395.19,0.10 36 | 2014-01-01,397.11,0.10 37 | 2015-01-01,399.41,0.10 38 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/data/co2-annmean-mlo.csv: -------------------------------------------------------------------------------- 1 | Year,Mean,Uncertainty 2 | 1959-01-01,315.97,0.12 3 | 1960-01-01,316.91,0.12 4 | 1961-01-01,317.64,0.12 5 | 1962-01-01,318.45,0.12 6 | 1963-01-01,318.99,0.12 7 | 1964-01-01,319.62,0.12 8 | 1965-01-01,320.04,0.12 9 | 1966-01-01,321.38,0.12 10 | 1967-01-01,322.16,0.12 11 | 1968-01-01,323.04,0.12 12 | 1969-01-01,324.62,0.12 13 | 1970-01-01,325.68,0.12 14 | 1971-01-01,326.32,0.12 15 | 1972-01-01,327.45,0.12 16 | 1973-01-01,329.68,0.12 17 | 1974-01-01,330.18,0.12 18 | 1975-01-01,331.11,0.12 19 | 1976-01-01,332.04,0.12 20 | 1977-01-01,333.83,0.12 21 | 1978-01-01,335.40,0.12 22 | 1979-01-01,336.84,0.12 23 | 1980-01-01,338.75,0.12 24 | 1981-01-01,340.11,0.12 25 | 1982-01-01,341.45,0.12 26 | 1983-01-01,343.05,0.12 27 | 1984-01-01,344.65,0.12 28 | 1985-01-01,346.12,0.12 29 | 1986-01-01,347.42,0.12 30 | 1987-01-01,349.18,0.12 31 | 1988-01-01,351.57,0.12 32 | 1989-01-01,353.12,0.12 33 | 1990-01-01,354.39,0.12 34 | 1991-01-01,355.61,0.12 35 | 1992-01-01,356.45,0.12 36 | 1993-01-01,357.10,0.12 37 | 1994-01-01,358.83,0.12 38 | 1995-01-01,360.82,0.12 39 | 1996-01-01,362.61,0.12 40 | 1997-01-01,363.73,0.12 41 | 1998-01-01,366.70,0.12 42 | 1999-01-01,368.38,0.12 43 | 2000-01-01,369.55,0.12 44 | 2001-01-01,371.14,0.12 45 | 2002-01-01,373.28,0.12 46 | 2003-01-01,375.80,0.12 47 | 2004-01-01,377.52,0.12 48 | 2005-01-01,379.80,0.12 49 | 2006-01-01,381.90,0.12 50 | 2007-01-01,383.79,0.12 51 | 2008-01-01,385.60,0.12 52 | 2009-01-01,387.43,0.12 53 | 2010-01-01,389.90,0.12 54 | 2011-01-01,391.65,0.12 55 | 2012-01-01,393.85,0.12 56 | 2013-01-01,396.52,0.12 57 | 2014-01-01,398.65,0.12 58 | 2015-01-01,400.83,0.12 59 | 2016-01-01,404.21,0.12 60 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/data/co2-gr-gl.csv: -------------------------------------------------------------------------------- 1 | Year,Annual Increase,Uncertainty 2 | 1959-01-01,0.96,0.31 3 | 1960-01-01,0.71,0.27 4 | 1961-01-01,0.78,0.27 5 | 1962-01-01,0.56,0.27 6 | 1963-01-01,0.57,0.28 7 | 1964-01-01,0.49,0.27 8 | 1965-01-01,1.10,0.26 9 | 1966-01-01,1.10,0.28 10 | 1967-01-01,0.61,0.34 11 | 1968-01-01,0.99,0.32 12 | 1969-01-01,1.32,0.29 13 | 1970-01-01,1.13,0.32 14 | 1971-01-01,0.73,0.30 15 | 1972-01-01,1.47,0.31 16 | 1973-01-01,1.46,0.31 17 | 1974-01-01,0.68,0.31 18 | 1975-01-01,1.23,0.27 19 | 1976-01-01,0.97,0.28 20 | 1977-01-01,1.92,0.29 21 | 1978-01-01,1.29,0.24 22 | 1979-01-01,2.14,0.26 23 | 1980-01-01,1.71,0.17 24 | 1981-01-01,1.15,0.12 25 | 1982-01-01,1.00,0.08 26 | 1983-01-01,1.84,0.09 27 | 1984-01-01,1.24,0.11 28 | 1985-01-01,1.63,0.08 29 | 1986-01-01,1.04,0.14 30 | 1987-01-01,2.69,0.09 31 | 1988-01-01,2.24,0.09 32 | 1989-01-01,1.38,0.09 33 | 1990-01-01,1.18,0.08 34 | 1991-01-01,0.73,0.09 35 | 1992-01-01,0.70,0.10 36 | 1993-01-01,1.22,0.07 37 | 1994-01-01,1.68,0.12 38 | 1995-01-01,1.95,0.11 39 | 1996-01-01,1.07,0.07 40 | 1997-01-01,1.98,0.07 41 | 1998-01-01,2.81,0.10 42 | 1999-01-01,1.34,0.07 43 | 2000-01-01,1.24,0.10 44 | 2001-01-01,1.85,0.10 45 | 2002-01-01,2.38,0.07 46 | 2003-01-01,2.27,0.10 47 | 2004-01-01,1.56,0.05 48 | 2005-01-01,2.43,0.07 49 | 2006-01-01,1.77,0.06 50 | 2007-01-01,2.09,0.07 51 | 2008-01-01,1.78,0.05 52 | 2009-01-01,1.62,0.10 53 | 2010-01-01,2.44,0.06 54 | 2011-01-01,1.68,0.09 55 | 2012-01-01,2.36,0.09 56 | 2013-01-01,2.47,0.09 57 | 2014-01-01,1.99,0.09 58 | 2015-01-01,2.96,0.09 59 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/data/co2-gr-mlo.csv: -------------------------------------------------------------------------------- 1 | Year,Annual Increase,Uncertainty 2 | 1959-01-01,0.94,0.11 3 | 1960-01-01,0.54,0.11 4 | 1961-01-01,0.95,0.11 5 | 1962-01-01,0.64,0.11 6 | 1963-01-01,0.71,0.11 7 | 1964-01-01,0.28,0.11 8 | 1965-01-01,1.02,0.11 9 | 1966-01-01,1.24,0.11 10 | 1967-01-01,0.74,0.11 11 | 1968-01-01,1.03,0.11 12 | 1969-01-01,1.31,0.11 13 | 1970-01-01,1.06,0.11 14 | 1971-01-01,0.84,0.11 15 | 1972-01-01,1.69,0.11 16 | 1973-01-01,1.22,0.11 17 | 1974-01-01,0.81,0.11 18 | 1975-01-01,1.09,0.11 19 | 1976-01-01,0.81,0.11 20 | 1977-01-01,2.16,0.11 21 | 1978-01-01,1.31,0.11 22 | 1979-01-01,1.80,0.11 23 | 1980-01-01,1.68,0.11 24 | 1981-01-01,1.43,0.11 25 | 1982-01-01,0.99,0.11 26 | 1983-01-01,2.10,0.11 27 | 1984-01-01,1.40,0.11 28 | 1985-01-01,1.26,0.11 29 | 1986-01-01,1.48,0.11 30 | 1987-01-01,2.20,0.11 31 | 1988-01-01,2.16,0.11 32 | 1989-01-01,1.36,0.11 33 | 1990-01-01,1.16,0.11 34 | 1991-01-01,1.04,0.11 35 | 1992-01-01,0.46,0.11 36 | 1993-01-01,1.35,0.11 37 | 1994-01-01,1.94,0.11 38 | 1995-01-01,2.00,0.11 39 | 1996-01-01,1.22,0.11 40 | 1997-01-01,1.93,0.11 41 | 1998-01-01,2.93,0.11 42 | 1999-01-01,0.93,0.11 43 | 2000-01-01,1.61,0.11 44 | 2001-01-01,1.61,0.11 45 | 2002-01-01,2.50,0.11 46 | 2003-01-01,2.27,0.11 47 | 2004-01-01,1.60,0.11 48 | 2005-01-01,2.54,0.11 49 | 2006-01-01,1.68,0.11 50 | 2007-01-01,2.27,0.11 51 | 2008-01-01,1.57,0.11 52 | 2009-01-01,2.02,0.11 53 | 2010-01-01,2.32,0.11 54 | 2011-01-01,1.92,0.11 55 | 2012-01-01,2.60,0.11 56 | 2013-01-01,2.06,0.11 57 | 2014-01-01,2.15,0.11 58 | 2015-01-01,3.03,0.11 59 | 2016-01-01,2.77,0.11 60 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/data/co2-mm-gl.csv: -------------------------------------------------------------------------------- 1 | Date,Decimal Date,Average,Trend 2 | 1980-01-01,1980.042,338.45,337.82 3 | 1980-02-01,1980.125,339.14,338.10 4 | 1980-03-01,1980.208,339.46,338.12 5 | 1980-04-01,1980.292,339.86,338.24 6 | 1980-05-01,1980.375,340.30,338.77 7 | 1980-06-01,1980.458,339.86,339.08 8 | 1980-07-01,1980.542,338.34,339.19 9 | 1980-08-01,1980.625,337.13,339.39 10 | 1980-09-01,1980.708,336.95,339.34 11 | 1980-10-01,1980.792,337.71,339.05 12 | 1980-11-01,1980.875,338.83,339.16 13 | 1980-12-01,1980.958,339.54,339.30 14 | 1981-01-01,1981.042,340.09,339.46 15 | 1981-02-01,1981.125,340.64,339.60 16 | 1981-03-01,1981.208,341.27,339.93 17 | 1981-04-01,1981.292,341.56,339.95 18 | 1981-05-01,1981.375,341.29,339.76 19 | 1981-06-01,1981.458,340.48,339.70 20 | 1981-07-01,1981.542,339.10,339.94 21 | 1981-08-01,1981.625,337.95,340.22 22 | 1981-09-01,1981.708,337.85,340.24 23 | 1981-10-01,1981.792,338.96,340.30 24 | 1981-11-01,1981.875,340.06,340.38 25 | 1981-12-01,1981.958,340.63,340.40 26 | 1982-01-01,1982.042,341.27,340.65 27 | 1982-02-01,1982.125,341.85,340.80 28 | 1982-03-01,1982.208,342.13,340.75 29 | 1982-04-01,1982.292,342.42,340.81 30 | 1982-05-01,1982.375,342.27,340.74 31 | 1982-06-01,1982.458,341.39,340.63 32 | 1982-07-01,1982.542,339.66,340.51 33 | 1982-08-01,1982.625,338.02,340.30 34 | 1982-09-01,1982.708,338.08,340.45 35 | 1982-10-01,1982.792,339.53,340.87 36 | 1982-11-01,1982.875,340.86,341.19 37 | 1982-12-01,1982.958,341.68,341.44 38 | 1983-01-01,1983.042,342.27,341.61 39 | 1983-02-01,1983.125,342.64,341.60 40 | 1983-03-01,1983.208,342.93,341.60 41 | 1983-04-01,1983.292,343.41,341.84 42 | 1983-05-01,1983.375,343.74,342.22 43 | 1983-06-01,1983.458,343.40,342.60 44 | 1983-07-01,1983.542,342.05,342.81 45 | 1983-08-01,1983.625,340.53,342.75 46 | 1983-09-01,1983.708,340.45,342.85 47 | 1983-10-01,1983.792,341.66,343.07 48 | 1983-11-01,1983.875,342.73,343.08 49 | 1983-12-01,1983.958,343.39,343.17 50 | 1984-01-01,1984.042,344.22,343.57 51 | 1984-02-01,1984.125,344.74,343.73 52 | 1984-03-01,1984.208,344.86,343.51 53 | 1984-04-01,1984.292,345.09,343.46 54 | 1984-05-01,1984.375,345.24,343.70 55 | 1984-06-01,1984.458,344.47,343.71 56 | 1984-07-01,1984.542,343.11,343.92 57 | 1984-08-01,1984.625,342.13,344.41 58 | 1984-09-01,1984.708,342.04,344.49 59 | 1984-10-01,1984.792,342.91,344.28 60 | 1984-11-01,1984.875,344.08,344.39 61 | 1984-12-01,1984.958,344.92,344.63 62 | 1985-01-01,1985.042,345.27,344.58 63 | 1985-02-01,1985.125,345.71,344.69 64 | 1985-03-01,1985.208,346.55,345.24 65 | 1985-04-01,1985.292,346.84,345.24 66 | 1985-05-01,1985.375,346.68,345.11 67 | 1985-06-01,1985.458,346.18,345.39 68 | 1985-07-01,1985.542,344.86,345.68 69 | 1985-08-01,1985.625,343.37,345.68 70 | 1985-09-01,1985.708,343.30,345.75 71 | 1985-10-01,1985.792,344.56,345.94 72 | 1985-11-01,1985.875,345.69,346.02 73 | 1985-12-01,1985.958,346.45,346.15 74 | 1986-01-01,1986.042,347.01,346.32 75 | 1986-02-01,1986.125,347.23,346.24 76 | 1986-03-01,1986.208,347.61,346.29 77 | 1986-04-01,1986.292,348.13,346.51 78 | 1986-05-01,1986.375,348.23,346.66 79 | 1986-06-01,1986.458,347.67,346.90 80 | 1986-07-01,1986.542,346.21,347.05 81 | 1986-08-01,1986.625,344.83,347.13 82 | 1986-09-01,1986.708,344.76,347.17 83 | 1986-10-01,1986.792,345.95,347.31 84 | 1986-11-01,1986.875,347.22,347.55 85 | 1986-12-01,1986.958,347.68,347.39 86 | 1987-01-01,1987.042,347.94,347.15 87 | 1987-02-01,1987.125,348.49,347.40 88 | 1987-03-01,1987.208,349.21,347.79 89 | 1987-04-01,1987.292,349.91,348.22 90 | 1987-05-01,1987.375,350.20,348.62 91 | 1987-06-01,1987.458,349.41,348.74 92 | 1987-07-01,1987.542,347.81,348.79 93 | 1987-08-01,1987.625,346.57,348.95 94 | 1987-09-01,1987.708,346.65,349.10 95 | 1987-10-01,1987.792,347.93,349.31 96 | 1987-11-01,1987.875,349.20,349.53 97 | 1987-12-01,1987.958,350.16,349.85 98 | 1988-01-01,1988.042,350.87,350.07 99 | 1988-02-01,1988.125,351.43,350.35 100 | 1988-03-01,1988.208,351.81,350.35 101 | 1988-04-01,1988.292,352.26,350.49 102 | 1988-05-01,1988.375,352.43,350.80 103 | 1988-06-01,1988.458,351.75,351.04 104 | 1988-07-01,1988.542,350.27,351.25 105 | 1988-08-01,1988.625,349.07,351.53 106 | 1988-09-01,1988.708,349.27,351.78 107 | 1988-10-01,1988.792,350.45,351.87 108 | 1988-11-01,1988.875,351.63,351.98 109 | 1988-12-01,1988.958,352.48,352.20 110 | 1989-01-01,1989.042,353.03,352.21 111 | 1989-02-01,1989.125,353.49,352.38 112 | 1989-03-01,1989.208,354.04,352.61 113 | 1989-04-01,1989.292,354.42,352.65 114 | 1989-05-01,1989.375,354.23,352.55 115 | 1989-06-01,1989.458,353.29,352.59 116 | 1989-07-01,1989.542,351.58,352.61 117 | 1989-08-01,1989.625,350.19,352.68 118 | 1989-09-01,1989.708,350.52,353.05 119 | 1989-10-01,1989.792,351.81,353.24 120 | 1989-11-01,1989.875,352.98,353.33 121 | 1989-12-01,1989.958,353.84,353.52 122 | 1990-01-01,1990.042,354.39,353.65 123 | 1990-02-01,1990.125,354.78,353.72 124 | 1990-03-01,1990.208,355.08,353.68 125 | 1990-04-01,1990.292,355.40,353.64 126 | 1990-05-01,1990.375,355.31,353.64 127 | 1990-06-01,1990.458,354.24,353.59 128 | 1990-07-01,1990.542,352.63,353.71 129 | 1990-08-01,1990.625,351.50,354.01 130 | 1990-09-01,1990.708,351.71,354.23 131 | 1990-10-01,1990.792,353.10,354.48 132 | 1990-11-01,1990.875,354.34,354.62 133 | 1990-12-01,1990.958,355.09,354.63 134 | 1991-01-01,1991.042,355.66,354.90 135 | 1991-02-01,1991.125,356.08,354.98 136 | 1991-03-01,1991.208,356.57,355.16 137 | 1991-04-01,1991.292,357.08,355.34 138 | 1991-05-01,1991.375,357.00,355.37 139 | 1991-06-01,1991.458,356.08,355.47 140 | 1991-07-01,1991.542,354.44,355.53 141 | 1991-08-01,1991.625,352.95,355.45 142 | 1991-09-01,1991.708,352.82,355.34 143 | 1991-10-01,1991.792,353.90,355.28 144 | 1991-11-01,1991.875,355.06,355.30 145 | 1991-12-01,1991.958,355.85,355.36 146 | 1992-01-01,1992.042,356.42,355.63 147 | 1992-02-01,1992.125,356.80,355.68 148 | 1992-03-01,1992.208,357.16,355.71 149 | 1992-04-01,1992.292,357.65,355.87 150 | 1992-05-01,1992.375,357.73,356.09 151 | 1992-06-01,1992.458,356.78,356.19 152 | 1992-07-01,1992.542,355.09,356.22 153 | 1992-08-01,1992.625,353.58,356.13 154 | 1992-09-01,1992.708,353.49,356.05 155 | 1992-10-01,1992.792,354.70,356.09 156 | 1992-11-01,1992.875,355.87,356.11 157 | 1992-12-01,1992.958,356.63,356.14 158 | 1993-01-01,1993.042,357.08,356.25 159 | 1993-02-01,1993.125,357.42,356.27 160 | 1993-03-01,1993.208,357.83,356.40 161 | 1993-04-01,1993.292,358.29,356.54 162 | 1993-05-01,1993.375,358.24,356.59 163 | 1993-06-01,1993.458,357.22,356.55 164 | 1993-07-01,1993.542,355.62,356.63 165 | 1993-08-01,1993.625,354.34,356.80 166 | 1993-09-01,1993.708,354.35,356.93 167 | 1993-10-01,1993.792,355.59,357.06 168 | 1993-11-01,1993.875,356.83,357.17 169 | 1993-12-01,1993.958,357.72,357.34 170 | 1994-01-01,1994.042,358.34,357.49 171 | 1994-02-01,1994.125,358.87,357.71 172 | 1994-03-01,1994.208,359.22,357.78 173 | 1994-04-01,1994.292,359.57,357.80 174 | 1994-05-01,1994.375,359.61,357.93 175 | 1994-06-01,1994.458,358.68,357.97 176 | 1994-07-01,1994.542,357.16,358.14 177 | 1994-08-01,1994.625,355.94,358.44 178 | 1994-09-01,1994.708,355.90,358.55 179 | 1994-10-01,1994.792,357.13,358.67 180 | 1994-11-01,1994.875,358.57,358.92 181 | 1994-12-01,1994.958,359.44,359.04 182 | 1995-01-01,1995.042,360.00,359.16 183 | 1995-02-01,1995.125,360.46,359.34 184 | 1995-03-01,1995.208,360.89,359.53 185 | 1995-04-01,1995.292,361.36,359.67 186 | 1995-05-01,1995.375,361.32,359.69 187 | 1995-06-01,1995.458,360.48,359.78 188 | 1995-07-01,1995.542,358.84,359.83 189 | 1995-08-01,1995.625,357.56,360.02 190 | 1995-09-01,1995.708,357.90,360.49 191 | 1995-10-01,1995.792,359.29,360.75 192 | 1995-11-01,1995.875,360.62,360.90 193 | 1995-12-01,1995.958,361.50,361.04 194 | 1996-01-01,1996.042,361.99,361.06 195 | 1996-02-01,1996.125,362.35,361.17 196 | 1996-03-01,1996.208,362.67,361.26 197 | 1996-04-01,1996.292,363.00,361.29 198 | 1996-05-01,1996.375,363.14,361.54 199 | 1996-06-01,1996.458,362.73,362.07 200 | 1996-07-01,1996.542,361.43,362.46 201 | 1996-08-01,1996.625,359.92,362.42 202 | 1996-09-01,1996.708,359.55,362.17 203 | 1996-10-01,1996.792,360.52,362.01 204 | 1996-11-01,1996.875,361.63,361.93 205 | 1996-12-01,1996.958,362.49,362.06 206 | 1997-01-01,1997.042,363.13,362.19 207 | 1997-02-01,1997.125,363.51,362.33 208 | 1997-03-01,1997.208,363.87,362.47 209 | 1997-04-01,1997.292,364.34,362.66 210 | 1997-05-01,1997.375,364.38,362.81 211 | 1997-06-01,1997.458,363.49,362.85 212 | 1997-07-01,1997.542,361.81,362.84 213 | 1997-08-01,1997.625,360.28,362.76 214 | 1997-09-01,1997.708,360.27,362.87 215 | 1997-10-01,1997.792,361.79,363.26 216 | 1997-11-01,1997.875,363.46,363.74 217 | 1997-12-01,1997.958,364.51,364.06 218 | 1998-01-01,1998.042,365.07,364.14 219 | 1998-02-01,1998.125,365.43,364.27 220 | 1998-03-01,1998.208,365.81,364.43 221 | 1998-04-01,1998.292,366.41,364.75 222 | 1998-05-01,1998.375,366.71,365.17 223 | 1998-06-01,1998.458,366.11,365.50 224 | 1998-07-01,1998.542,364.68,365.72 225 | 1998-08-01,1998.625,363.63,366.11 226 | 1998-09-01,1998.708,363.86,366.45 227 | 1998-10-01,1998.792,365.16,366.60 228 | 1998-11-01,1998.875,366.42,366.67 229 | 1998-12-01,1998.958,367.28,366.79 230 | 1999-01-01,1999.042,367.95,367.02 231 | 1999-02-01,1999.125,368.34,367.19 232 | 1999-03-01,1999.208,368.73,367.35 233 | 1999-04-01,1999.292,369.12,367.49 234 | 1999-05-01,1999.375,369.03,367.52 235 | 1999-06-01,1999.458,368.19,367.61 236 | 1999-07-01,1999.542,366.53,367.56 237 | 1999-08-01,1999.625,365.16,367.60 238 | 1999-09-01,1999.708,365.27,367.82 239 | 1999-10-01,1999.792,366.59,368.01 240 | 1999-11-01,1999.875,367.89,368.13 241 | 1999-12-01,1999.958,368.72,368.20 242 | 2000-01-01,2000.042,369.21,368.30 243 | 2000-02-01,2000.125,369.46,368.31 244 | 2000-03-01,2000.208,369.78,368.40 245 | 2000-04-01,2000.292,370.18,368.52 246 | 2000-05-01,2000.375,370.08,368.56 247 | 2000-06-01,2000.458,369.17,368.64 248 | 2000-07-01,2000.542,367.79,368.91 249 | 2000-08-01,2000.625,366.62,369.13 250 | 2000-09-01,2000.708,366.56,369.11 251 | 2000-10-01,2000.792,367.77,369.15 252 | 2000-11-01,2000.875,369.12,369.30 253 | 2000-12-01,2000.958,369.92,369.36 254 | 2001-01-01,2001.042,370.51,369.62 255 | 2001-02-01,2001.125,371.00,369.84 256 | 2001-03-01,2001.208,371.40,369.99 257 | 2001-04-01,2001.292,371.72,370.05 258 | 2001-05-01,2001.375,371.62,370.09 259 | 2001-06-01,2001.458,370.67,370.14 260 | 2001-07-01,2001.542,369.29,370.39 261 | 2001-08-01,2001.625,368.16,370.63 262 | 2001-09-01,2001.708,368.20,370.72 263 | 2001-10-01,2001.792,369.56,370.94 264 | 2001-11-01,2001.875,370.88,371.10 265 | 2001-12-01,2001.958,371.80,371.29 266 | 2002-01-01,2002.042,372.34,371.38 267 | 2002-02-01,2002.125,372.73,371.49 268 | 2002-03-01,2002.208,373.21,371.69 269 | 2002-04-01,2002.292,373.56,371.80 270 | 2002-05-01,2002.375,373.53,371.95 271 | 2002-06-01,2002.458,372.66,372.11 272 | 2002-07-01,2002.542,371.25,372.38 273 | 2002-08-01,2002.625,370.20,372.73 274 | 2002-09-01,2002.708,370.52,373.14 275 | 2002-10-01,2002.792,371.79,373.27 276 | 2002-11-01,2002.875,373.12,373.40 277 | 2002-12-01,2002.958,374.09,373.64 278 | 2003-01-01,2003.042,374.78,373.78 279 | 2003-02-01,2003.125,375.30,374.00 280 | 2003-03-01,2003.208,375.73,374.16 281 | 2003-04-01,2003.292,376.22,374.44 282 | 2003-05-01,2003.375,376.36,374.76 283 | 2003-06-01,2003.458,375.53,374.98 284 | 2003-07-01,2003.542,373.99,375.12 285 | 2003-08-01,2003.625,372.74,375.31 286 | 2003-09-01,2003.708,372.91,375.58 287 | 2003-10-01,2003.792,374.20,375.73 288 | 2003-11-01,2003.875,375.49,375.82 289 | 2003-12-01,2003.958,376.34,375.92 290 | 2004-01-01,2004.042,377.02,376.05 291 | 2004-02-01,2004.125,377.53,376.23 292 | 2004-03-01,2004.208,377.96,376.37 293 | 2004-04-01,2004.292,378.30,376.52 294 | 2004-05-01,2004.375,378.23,376.64 295 | 2004-06-01,2004.458,377.35,376.79 296 | 2004-07-01,2004.542,375.81,376.98 297 | 2004-08-01,2004.625,374.33,376.97 298 | 2004-09-01,2004.708,374.22,376.89 299 | 2004-10-01,2004.792,375.56,377.06 300 | 2004-11-01,2004.875,377.03,377.33 301 | 2004-12-01,2004.958,377.99,377.51 302 | 2005-01-01,2005.042,378.56,377.59 303 | 2005-02-01,2005.125,379.09,377.79 304 | 2005-03-01,2005.208,379.70,378.10 305 | 2005-04-01,2005.292,380.16,378.36 306 | 2005-05-01,2005.375,380.27,378.64 307 | 2005-06-01,2005.458,379.45,378.85 308 | 2005-07-01,2005.542,377.78,378.93 309 | 2005-08-01,2005.625,376.55,379.18 310 | 2005-09-01,2005.708,376.58,379.28 311 | 2005-10-01,2005.792,377.89,379.45 312 | 2005-11-01,2005.875,379.36,379.69 313 | 2005-12-01,2005.958,380.34,379.86 314 | 2006-01-01,2006.042,381.09,380.09 315 | 2006-02-01,2006.125,381.72,380.35 316 | 2006-03-01,2006.208,382.12,380.49 317 | 2006-04-01,2006.292,382.46,380.63 318 | 2006-05-01,2006.375,382.41,380.75 319 | 2006-06-01,2006.458,381.55,380.92 320 | 2006-07-01,2006.542,379.88,381.07 321 | 2006-08-01,2006.625,378.30,381.01 322 | 2006-09-01,2006.708,378.42,381.19 323 | 2006-10-01,2006.792,379.84,381.42 324 | 2006-11-01,2006.875,381.21,381.54 325 | 2006-12-01,2006.958,382.17,381.70 326 | 2007-01-01,2007.042,382.80,381.79 327 | 2007-02-01,2007.125,383.31,381.91 328 | 2007-03-01,2007.208,383.78,382.13 329 | 2007-04-01,2007.292,384.04,382.21 330 | 2007-05-01,2007.375,383.91,382.28 331 | 2007-06-01,2007.458,383.07,382.50 332 | 2007-07-01,2007.542,381.36,382.61 333 | 2007-08-01,2007.625,380.06,382.82 334 | 2007-09-01,2007.708,380.45,383.22 335 | 2007-10-01,2007.792,381.86,383.41 336 | 2007-11-01,2007.875,383.20,383.48 337 | 2007-12-01,2007.958,384.21,383.71 338 | 2008-01-01,2008.042,384.98,383.97 339 | 2008-02-01,2008.125,385.49,384.10 340 | 2008-03-01,2008.208,385.89,384.27 341 | 2008-04-01,2008.292,386.29,384.48 342 | 2008-05-01,2008.375,386.26,384.63 343 | 2008-06-01,2008.458,385.34,384.77 344 | 2008-07-01,2008.542,383.86,385.17 345 | 2008-08-01,2008.625,382.53,385.33 346 | 2008-09-01,2008.708,382.30,385.05 347 | 2008-10-01,2008.792,383.43,384.91 348 | 2008-11-01,2008.875,384.92,385.16 349 | 2008-12-01,2008.958,386.01,385.47 350 | 2009-01-01,2009.042,386.80,385.76 351 | 2009-02-01,2009.125,387.26,385.83 352 | 2009-03-01,2009.208,387.49,385.84 353 | 2009-04-01,2009.292,387.77,385.96 354 | 2009-05-01,2009.375,387.74,386.14 355 | 2009-06-01,2009.458,386.74,386.23 356 | 2009-07-01,2009.542,384.80,386.20 357 | 2009-08-01,2009.625,383.42,386.28 358 | 2009-09-01,2009.708,383.72,386.45 359 | 2009-10-01,2009.792,385.28,386.70 360 | 2009-11-01,2009.875,386.74,386.92 361 | 2009-12-01,2009.958,387.63,387.07 362 | 2010-01-01,2010.042,388.42,387.40 363 | 2010-02-01,2010.125,389.14,387.74 364 | 2010-03-01,2010.208,389.48,387.84 365 | 2010-04-01,2010.292,389.77,387.97 366 | 2010-05-01,2010.375,389.74,388.15 367 | 2010-06-01,2010.458,388.80,388.30 368 | 2010-07-01,2010.542,387.16,388.54 369 | 2010-08-01,2010.625,386.04,388.85 370 | 2010-09-01,2010.708,386.50,389.21 371 | 2010-10-01,2010.792,388.06,389.48 372 | 2010-11-01,2010.875,389.43,389.62 373 | 2010-12-01,2010.958,390.19,389.65 374 | 2011-01-01,2011.042,390.74,389.69 375 | 2011-02-01,2011.125,391.16,389.74 376 | 2011-03-01,2011.208,391.46,389.83 377 | 2011-04-01,2011.292,391.84,390.04 378 | 2011-05-01,2011.375,391.88,390.26 379 | 2011-06-01,2011.458,390.95,390.45 380 | 2011-07-01,2011.542,389.01,390.38 381 | 2011-08-01,2011.625,387.68,390.45 382 | 2011-09-01,2011.708,388.08,390.80 383 | 2011-10-01,2011.792,389.66,391.12 384 | 2011-11-01,2011.875,391.02,391.23 385 | 2011-12-01,2011.958,391.83,391.34 386 | 2012-01-01,2012.042,392.41,391.36 387 | 2012-02-01,2012.125,393.00,391.58 388 | 2012-03-01,2012.208,393.52,391.90 389 | 2012-04-01,2012.292,393.80,392.01 390 | 2012-05-01,2012.375,393.67,392.08 391 | 2012-06-01,2012.458,392.63,392.15 392 | 2012-07-01,2012.542,390.80,392.23 393 | 2012-08-01,2012.625,389.69,392.52 394 | 2012-09-01,2012.708,390.36,393.06 395 | 2012-10-01,2012.792,391.97,393.38 396 | 2012-11-01,2012.875,393.37,393.53 397 | 2012-12-01,2012.958,394.17,393.60 398 | 2013-01-01,2013.042,394.86,393.82 399 | 2013-02-01,2013.125,395.49,394.08 400 | 2013-03-01,2013.208,396.07,394.44 401 | 2013-04-01,2013.292,396.52,394.70 402 | 2013-05-01,2013.375,396.53,394.92 403 | 2013-06-01,2013.458,395.74,395.22 404 | 2013-07-01,2013.542,394.27,395.65 405 | 2013-08-01,2013.625,393.05,395.85 406 | 2013-09-01,2013.708,393.01,395.71 407 | 2013-10-01,2013.792,394.31,395.74 408 | 2013-11-01,2013.875,395.76,396.00 409 | 2013-12-01,2013.958,396.65,396.13 410 | 2014-01-01,2014.042,397.27,396.24 411 | 2014-02-01,2014.125,397.73,396.32 412 | 2014-03-01,2014.208,398.03,396.40 413 | 2014-04-01,2014.292,398.39,396.57 414 | 2014-05-01,2014.375,398.46,396.85 415 | 2014-06-01,2014.458,397.50,396.99 416 | 2014-07-01,2014.542,395.91,397.29 417 | 2014-08-01,2014.625,394.80,397.60 418 | 2014-09-01,2014.708,394.89,397.59 419 | 2014-10-01,2014.792,396.15,397.58 420 | 2014-11-01,2014.875,397.63,397.87 421 | 2014-12-01,2014.958,398.60,398.07 422 | 2015-01-01,2015.042,399.30,398.27 423 | 2015-02-01,2015.125,399.86,398.44 424 | 2015-03-01,2015.208,400.30,398.67 425 | 2015-04-01,2015.292,400.69,398.87 426 | 2015-05-01,2015.375,400.64,399.03 427 | 2015-06-01,2015.458,399.79,399.27 428 | 2015-07-01,2015.542,398.12,399.50 429 | 2015-08-01,2015.625,396.84,399.64 430 | 2015-09-01,2015.708,397.15,399.86 431 | 2015-10-01,2015.792,398.60,400.03 432 | 2015-11-01,2015.875,400.15,400.39 433 | 2015-12-01,2015.958,401.42,400.90 434 | 2016-01-01,2016.042,402.39,401.36 435 | 2016-02-01,2016.125,403.01,401.59 436 | 2016-03-01,2016.208,403.56,401.94 437 | 2016-04-01,2016.292,404.08,402.26 438 | 2016-05-01,2016.375,404.16,402.55 439 | 2016-06-01,2016.458,403.35,402.83 440 | 2016-07-01,2016.542,401.85,403.23 441 | 2016-08-01,2016.625,400.55,403.34 442 | 2016-09-01,2016.708,400.68,403.39 443 | 2016-10-01,2016.792,402.31,403.73 444 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/data/co2-mm-mlo.csv: -------------------------------------------------------------------------------- 1 | Date,Decimal Date,Average,Interpolated,Trend,Number of Days 2 | 1958-03-01,1958.208,315.71,315.71,314.62,-1 3 | 1958-04-01,1958.292,317.45,317.45,315.29,-1 4 | 1958-05-01,1958.375,317.50,317.50,314.71,-1 5 | 1958-06-01,1958.458,-99.99,317.10,314.85,-1 6 | 1958-07-01,1958.542,315.86,315.86,314.98,-1 7 | 1958-08-01,1958.625,314.93,314.93,315.94,-1 8 | 1958-09-01,1958.708,313.20,313.20,315.91,-1 9 | 1958-10-01,1958.792,-99.99,312.66,315.61,-1 10 | 1958-11-01,1958.875,313.33,313.33,315.31,-1 11 | 1958-12-01,1958.958,314.67,314.67,315.61,-1 12 | 1959-01-01,1959.042,315.62,315.62,315.70,-1 13 | 1959-02-01,1959.125,316.38,316.38,315.88,-1 14 | 1959-03-01,1959.208,316.71,316.71,315.62,-1 15 | 1959-04-01,1959.292,317.72,317.72,315.56,-1 16 | 1959-05-01,1959.375,318.29,318.29,315.50,-1 17 | 1959-06-01,1959.458,318.15,318.15,315.92,-1 18 | 1959-07-01,1959.542,316.54,316.54,315.66,-1 19 | 1959-08-01,1959.625,314.80,314.80,315.81,-1 20 | 1959-09-01,1959.708,313.84,313.84,316.55,-1 21 | 1959-10-01,1959.792,313.26,313.26,316.19,-1 22 | 1959-11-01,1959.875,314.80,314.80,316.78,-1 23 | 1959-12-01,1959.958,315.58,315.58,316.52,-1 24 | 1960-01-01,1960.042,316.43,316.43,316.51,-1 25 | 1960-02-01,1960.125,316.97,316.97,316.47,-1 26 | 1960-03-01,1960.208,317.58,317.58,316.49,-1 27 | 1960-04-01,1960.292,319.02,319.02,316.86,-1 28 | 1960-05-01,1960.375,320.03,320.03,317.24,-1 29 | 1960-06-01,1960.458,319.59,319.59,317.36,-1 30 | 1960-07-01,1960.542,318.18,318.18,317.30,-1 31 | 1960-08-01,1960.625,315.91,315.91,316.92,-1 32 | 1960-09-01,1960.708,314.16,314.16,316.87,-1 33 | 1960-10-01,1960.792,313.83,313.83,316.76,-1 34 | 1960-11-01,1960.875,315.00,315.00,316.98,-1 35 | 1960-12-01,1960.958,316.19,316.19,317.13,-1 36 | 1961-01-01,1961.042,316.93,316.93,317.03,-1 37 | 1961-02-01,1961.125,317.70,317.70,317.28,-1 38 | 1961-03-01,1961.208,318.54,318.54,317.47,-1 39 | 1961-04-01,1961.292,319.48,319.48,317.27,-1 40 | 1961-05-01,1961.375,320.58,320.58,317.70,-1 41 | 1961-06-01,1961.458,319.77,319.77,317.48,-1 42 | 1961-07-01,1961.542,318.57,318.57,317.70,-1 43 | 1961-08-01,1961.625,316.79,316.79,317.80,-1 44 | 1961-09-01,1961.708,314.80,314.80,317.49,-1 45 | 1961-10-01,1961.792,315.38,315.38,318.35,-1 46 | 1961-11-01,1961.875,316.10,316.10,318.13,-1 47 | 1961-12-01,1961.958,317.01,317.01,317.94,-1 48 | 1962-01-01,1962.042,317.94,317.94,318.06,-1 49 | 1962-02-01,1962.125,318.56,318.56,318.11,-1 50 | 1962-03-01,1962.208,319.68,319.68,318.57,-1 51 | 1962-04-01,1962.292,320.63,320.63,318.45,-1 52 | 1962-05-01,1962.375,321.01,321.01,318.20,-1 53 | 1962-06-01,1962.458,320.55,320.55,318.27,-1 54 | 1962-07-01,1962.542,319.58,319.58,318.67,-1 55 | 1962-08-01,1962.625,317.40,317.40,318.48,-1 56 | 1962-09-01,1962.708,316.26,316.26,319.03,-1 57 | 1962-10-01,1962.792,315.42,315.42,318.33,-1 58 | 1962-11-01,1962.875,316.69,316.69,318.62,-1 59 | 1962-12-01,1962.958,317.69,317.69,318.61,-1 60 | 1963-01-01,1963.042,318.74,318.74,318.91,-1 61 | 1963-02-01,1963.125,319.08,319.08,318.68,-1 62 | 1963-03-01,1963.208,319.86,319.86,318.69,-1 63 | 1963-04-01,1963.292,321.39,321.39,319.09,-1 64 | 1963-05-01,1963.375,322.25,322.25,319.39,-1 65 | 1963-06-01,1963.458,321.47,321.47,319.16,-1 66 | 1963-07-01,1963.542,319.74,319.74,318.77,-1 67 | 1963-08-01,1963.625,317.77,317.77,318.83,-1 68 | 1963-09-01,1963.708,316.21,316.21,319.06,-1 69 | 1963-10-01,1963.792,315.99,315.99,319.00,-1 70 | 1963-11-01,1963.875,317.12,317.12,319.10,-1 71 | 1963-12-01,1963.958,318.31,318.31,319.25,-1 72 | 1964-01-01,1964.042,319.57,319.57,319.67,-1 73 | 1964-02-01,1964.125,-99.99,320.07,319.61,-1 74 | 1964-03-01,1964.208,-99.99,320.73,319.55,-1 75 | 1964-04-01,1964.292,-99.99,321.77,319.48,-1 76 | 1964-05-01,1964.375,322.25,322.25,319.42,-1 77 | 1964-06-01,1964.458,321.89,321.89,319.69,-1 78 | 1964-07-01,1964.542,320.44,320.44,319.58,-1 79 | 1964-08-01,1964.625,318.70,318.70,319.81,-1 80 | 1964-09-01,1964.708,316.70,316.70,319.56,-1 81 | 1964-10-01,1964.792,316.79,316.79,319.78,-1 82 | 1964-11-01,1964.875,317.79,317.79,319.72,-1 83 | 1964-12-01,1964.958,318.71,318.71,319.59,-1 84 | 1965-01-01,1965.042,319.44,319.44,319.48,-1 85 | 1965-02-01,1965.125,320.44,320.44,319.97,-1 86 | 1965-03-01,1965.208,320.89,320.89,319.65,-1 87 | 1965-04-01,1965.292,322.13,322.13,319.80,-1 88 | 1965-05-01,1965.375,322.16,322.16,319.36,-1 89 | 1965-06-01,1965.458,321.87,321.87,319.65,-1 90 | 1965-07-01,1965.542,321.39,321.39,320.51,-1 91 | 1965-08-01,1965.625,318.81,318.81,319.93,-1 92 | 1965-09-01,1965.708,317.81,317.81,320.68,-1 93 | 1965-10-01,1965.792,317.30,317.30,320.36,-1 94 | 1965-11-01,1965.875,318.87,318.87,320.87,-1 95 | 1965-12-01,1965.958,319.42,319.42,320.26,-1 96 | 1966-01-01,1966.042,320.62,320.62,320.63,-1 97 | 1966-02-01,1966.125,321.59,321.59,321.10,-1 98 | 1966-03-01,1966.208,322.39,322.39,321.16,-1 99 | 1966-04-01,1966.292,323.87,323.87,321.51,-1 100 | 1966-05-01,1966.375,324.01,324.01,321.18,-1 101 | 1966-06-01,1966.458,323.75,323.75,321.52,-1 102 | 1966-07-01,1966.542,322.39,322.39,321.49,-1 103 | 1966-08-01,1966.625,320.37,320.37,321.50,-1 104 | 1966-09-01,1966.708,318.64,318.64,321.54,-1 105 | 1966-10-01,1966.792,318.10,318.10,321.18,-1 106 | 1966-11-01,1966.875,319.79,319.79,321.84,-1 107 | 1966-12-01,1966.958,321.08,321.08,321.95,-1 108 | 1967-01-01,1967.042,322.07,322.07,322.07,-1 109 | 1967-02-01,1967.125,322.50,322.50,321.94,-1 110 | 1967-03-01,1967.208,323.04,323.04,321.72,-1 111 | 1967-04-01,1967.292,324.42,324.42,322.05,-1 112 | 1967-05-01,1967.375,325.00,325.00,322.27,-1 113 | 1967-06-01,1967.458,324.09,324.09,321.94,-1 114 | 1967-07-01,1967.542,322.55,322.55,321.66,-1 115 | 1967-08-01,1967.625,320.92,320.92,322.04,-1 116 | 1967-09-01,1967.708,319.31,319.31,322.19,-1 117 | 1967-10-01,1967.792,319.31,319.31,322.36,-1 118 | 1967-11-01,1967.875,320.72,320.72,322.78,-1 119 | 1967-12-01,1967.958,321.96,321.96,322.86,-1 120 | 1968-01-01,1968.042,322.57,322.57,322.55,-1 121 | 1968-02-01,1968.125,323.15,323.15,322.56,-1 122 | 1968-03-01,1968.208,323.89,323.89,322.59,-1 123 | 1968-04-01,1968.292,325.02,325.02,322.73,-1 124 | 1968-05-01,1968.375,325.57,325.57,322.87,-1 125 | 1968-06-01,1968.458,325.36,325.36,323.20,-1 126 | 1968-07-01,1968.542,324.14,324.14,323.25,-1 127 | 1968-08-01,1968.625,322.03,322.03,323.15,-1 128 | 1968-09-01,1968.708,320.41,320.41,323.31,-1 129 | 1968-10-01,1968.792,320.25,320.25,323.32,-1 130 | 1968-11-01,1968.875,321.31,321.31,323.32,-1 131 | 1968-12-01,1968.958,322.84,322.84,323.69,-1 132 | 1969-01-01,1969.042,324.00,324.00,323.98,-1 133 | 1969-02-01,1969.125,324.42,324.42,323.89,-1 134 | 1969-03-01,1969.208,325.64,325.64,324.41,-1 135 | 1969-04-01,1969.292,326.66,326.66,324.35,-1 136 | 1969-05-01,1969.375,327.34,327.34,324.57,-1 137 | 1969-06-01,1969.458,326.76,326.76,324.63,-1 138 | 1969-07-01,1969.542,325.88,325.88,325.08,-1 139 | 1969-08-01,1969.625,323.67,323.67,324.80,-1 140 | 1969-09-01,1969.708,322.38,322.38,325.28,-1 141 | 1969-10-01,1969.792,321.78,321.78,324.84,-1 142 | 1969-11-01,1969.875,322.85,322.85,324.78,-1 143 | 1969-12-01,1969.958,324.11,324.11,324.88,-1 144 | 1970-01-01,1970.042,325.03,325.03,325.04,-1 145 | 1970-02-01,1970.125,325.99,325.99,325.42,-1 146 | 1970-03-01,1970.208,326.87,326.87,325.69,-1 147 | 1970-04-01,1970.292,328.13,328.13,325.86,-1 148 | 1970-05-01,1970.375,328.07,328.07,325.27,-1 149 | 1970-06-01,1970.458,327.66,327.66,325.52,-1 150 | 1970-07-01,1970.542,326.35,326.35,325.51,-1 151 | 1970-08-01,1970.625,324.69,324.69,325.76,-1 152 | 1970-09-01,1970.708,323.10,323.10,325.93,-1 153 | 1970-10-01,1970.792,323.16,323.16,326.15,-1 154 | 1970-11-01,1970.875,323.98,323.98,325.96,-1 155 | 1970-12-01,1970.958,325.13,325.13,326.06,-1 156 | 1971-01-01,1971.042,326.17,326.17,326.25,-1 157 | 1971-02-01,1971.125,326.68,326.68,326.10,-1 158 | 1971-03-01,1971.208,327.18,327.18,325.94,-1 159 | 1971-04-01,1971.292,327.78,327.78,325.47,-1 160 | 1971-05-01,1971.375,328.92,328.92,326.11,-1 161 | 1971-06-01,1971.458,328.57,328.57,326.40,-1 162 | 1971-07-01,1971.542,327.34,327.34,326.45,-1 163 | 1971-08-01,1971.625,325.46,325.46,326.49,-1 164 | 1971-09-01,1971.708,323.36,323.36,326.19,-1 165 | 1971-10-01,1971.792,323.57,323.57,326.58,-1 166 | 1971-11-01,1971.875,324.80,324.80,326.82,-1 167 | 1971-12-01,1971.958,326.01,326.01,327.02,-1 168 | 1972-01-01,1972.042,326.77,326.77,326.85,-1 169 | 1972-02-01,1972.125,327.63,327.63,327.04,-1 170 | 1972-03-01,1972.208,327.75,327.75,326.53,-1 171 | 1972-04-01,1972.292,329.72,329.72,327.42,-1 172 | 1972-05-01,1972.375,330.07,330.07,327.23,-1 173 | 1972-06-01,1972.458,329.09,329.09,326.92,-1 174 | 1972-07-01,1972.542,328.05,328.05,327.20,-1 175 | 1972-08-01,1972.625,326.32,326.32,327.37,-1 176 | 1972-09-01,1972.708,324.93,324.93,327.76,-1 177 | 1972-10-01,1972.792,325.06,325.06,328.06,-1 178 | 1972-11-01,1972.875,326.50,326.50,328.50,-1 179 | 1972-12-01,1972.958,327.55,327.55,328.55,-1 180 | 1973-01-01,1973.042,328.54,328.54,328.58,-1 181 | 1973-02-01,1973.125,329.56,329.56,328.86,-1 182 | 1973-03-01,1973.208,330.30,330.30,328.99,-1 183 | 1973-04-01,1973.292,331.50,331.50,329.14,-1 184 | 1973-05-01,1973.375,332.48,332.48,329.62,-1 185 | 1973-06-01,1973.458,332.07,332.07,329.94,-1 186 | 1973-07-01,1973.542,330.87,330.87,330.05,-1 187 | 1973-08-01,1973.625,329.31,329.31,330.42,-1 188 | 1973-09-01,1973.708,327.51,327.51,330.45,-1 189 | 1973-10-01,1973.792,327.18,327.18,330.24,-1 190 | 1973-11-01,1973.875,328.16,328.16,330.16,-1 191 | 1973-12-01,1973.958,328.64,328.64,329.66,-1 192 | 1974-01-01,1974.042,329.35,329.35,329.45,-1 193 | 1974-02-01,1974.125,330.71,330.71,330.12,-1 194 | 1974-03-01,1974.208,331.48,331.48,330.20,-1 195 | 1974-04-01,1974.292,332.65,332.65,330.26,-1 196 | 1974-05-01,1974.375,333.20,333.20,330.27,14 197 | 1974-06-01,1974.458,332.16,332.16,329.94,26 198 | 1974-07-01,1974.542,331.07,331.07,330.23,24 199 | 1974-08-01,1974.625,329.12,329.12,330.26,27 200 | 1974-09-01,1974.708,327.32,327.32,330.28,24 201 | 1974-10-01,1974.792,327.28,327.28,330.36,24 202 | 1974-11-01,1974.875,328.30,328.30,330.28,27 203 | 1974-12-01,1974.958,329.58,329.58,330.55,28 204 | 1975-01-01,1975.042,330.73,330.73,330.89,29 205 | 1975-02-01,1975.125,331.46,331.46,330.93,26 206 | 1975-03-01,1975.208,331.90,331.90,330.54,18 207 | 1975-04-01,1975.292,333.17,333.17,330.67,25 208 | 1975-05-01,1975.375,333.94,333.94,330.98,28 209 | 1975-06-01,1975.458,333.45,333.45,331.20,26 210 | 1975-07-01,1975.542,331.98,331.98,331.12,24 211 | 1975-08-01,1975.625,329.95,329.95,331.11,24 212 | 1975-09-01,1975.708,328.50,328.50,331.48,23 213 | 1975-10-01,1975.792,328.35,328.35,331.46,12 214 | 1975-11-01,1975.875,329.37,329.37,331.41,19 215 | 1975-12-01,1975.958,-99.99,330.59,331.60,0 216 | 1976-01-01,1976.042,331.59,331.59,331.79,20 217 | 1976-02-01,1976.125,332.75,332.75,332.20,22 218 | 1976-03-01,1976.208,333.52,333.52,332.04,20 219 | 1976-04-01,1976.292,334.64,334.64,332.13,19 220 | 1976-05-01,1976.375,334.77,334.77,331.84,22 221 | 1976-06-01,1976.458,334.00,334.00,331.65,17 222 | 1976-07-01,1976.542,333.06,333.06,332.14,16 223 | 1976-08-01,1976.625,330.68,330.68,331.88,23 224 | 1976-09-01,1976.708,328.95,328.95,331.94,13 225 | 1976-10-01,1976.792,328.75,328.75,331.92,20 226 | 1976-11-01,1976.875,330.15,330.15,332.29,25 227 | 1976-12-01,1976.958,331.63,331.63,332.66,20 228 | 1977-01-01,1977.042,332.66,332.66,332.76,24 229 | 1977-02-01,1977.125,333.13,333.13,332.51,19 230 | 1977-03-01,1977.208,334.95,334.95,333.35,23 231 | 1977-04-01,1977.292,336.13,336.13,333.51,21 232 | 1977-05-01,1977.375,336.93,336.93,333.98,20 233 | 1977-06-01,1977.458,336.16,336.16,333.80,22 234 | 1977-07-01,1977.542,334.88,334.88,334.02,21 235 | 1977-08-01,1977.625,332.56,332.56,333.91,18 236 | 1977-09-01,1977.708,331.29,331.29,334.36,19 237 | 1977-10-01,1977.792,331.27,331.27,334.52,23 238 | 1977-11-01,1977.875,332.41,332.41,334.64,21 239 | 1977-12-01,1977.958,333.60,333.60,334.61,26 240 | 1978-01-01,1978.042,334.95,334.95,335.01,22 241 | 1978-02-01,1978.125,335.25,335.25,334.58,25 242 | 1978-03-01,1978.208,336.66,336.66,335.00,28 243 | 1978-04-01,1978.292,337.69,337.69,335.06,18 244 | 1978-05-01,1978.375,338.03,338.03,335.06,26 245 | 1978-06-01,1978.458,338.01,338.01,335.59,17 246 | 1978-07-01,1978.542,336.41,336.41,335.57,22 247 | 1978-08-01,1978.625,334.41,334.41,335.87,19 248 | 1978-09-01,1978.708,332.37,332.37,335.51,17 249 | 1978-10-01,1978.792,332.41,332.41,335.68,23 250 | 1978-11-01,1978.875,333.75,333.75,335.99,24 251 | 1978-12-01,1978.958,334.90,334.90,335.88,27 252 | 1979-01-01,1979.042,336.14,336.14,336.22,27 253 | 1979-02-01,1979.125,336.69,336.69,336.01,26 254 | 1979-03-01,1979.208,338.27,338.27,336.54,21 255 | 1979-04-01,1979.292,338.96,338.96,336.24,21 256 | 1979-05-01,1979.375,339.21,339.21,336.21,12 257 | 1979-06-01,1979.458,339.26,339.26,336.84,19 258 | 1979-07-01,1979.542,337.54,337.54,336.72,26 259 | 1979-08-01,1979.625,335.75,335.75,337.24,23 260 | 1979-09-01,1979.708,333.98,333.98,337.20,19 261 | 1979-10-01,1979.792,334.19,334.19,337.53,24 262 | 1979-11-01,1979.875,335.31,335.31,337.57,27 263 | 1979-12-01,1979.958,336.81,336.81,337.79,22 264 | 1980-01-01,1980.042,337.90,337.90,338.09,29 265 | 1980-02-01,1980.125,338.34,338.34,337.82,26 266 | 1980-03-01,1980.208,340.01,340.01,338.43,25 267 | 1980-04-01,1980.292,340.93,340.93,338.30,24 268 | 1980-05-01,1980.375,341.48,341.48,338.43,25 269 | 1980-06-01,1980.458,341.33,341.33,338.84,22 270 | 1980-07-01,1980.542,339.40,339.40,338.54,21 271 | 1980-08-01,1980.625,337.70,337.70,339.12,17 272 | 1980-09-01,1980.708,336.19,336.19,339.33,17 273 | 1980-10-01,1980.792,336.15,336.15,339.42,25 274 | 1980-11-01,1980.875,337.27,337.27,339.42,24 275 | 1980-12-01,1980.958,338.32,338.32,339.26,19 276 | 1981-01-01,1981.042,339.29,339.29,339.38,28 277 | 1981-02-01,1981.125,340.55,340.55,339.93,25 278 | 1981-03-01,1981.208,341.61,341.61,340.06,25 279 | 1981-04-01,1981.292,342.53,342.53,339.94,24 280 | 1981-05-01,1981.375,343.03,343.03,339.98,30 281 | 1981-06-01,1981.458,342.54,342.54,340.07,25 282 | 1981-07-01,1981.542,340.78,340.78,339.92,24 283 | 1981-08-01,1981.625,338.44,338.44,339.86,26 284 | 1981-09-01,1981.708,336.95,336.95,340.17,27 285 | 1981-10-01,1981.792,337.08,337.08,340.43,28 286 | 1981-11-01,1981.875,338.58,338.58,340.74,25 287 | 1981-12-01,1981.958,339.88,339.88,340.79,19 288 | 1982-01-01,1982.042,340.96,340.96,341.10,27 289 | 1982-02-01,1982.125,341.73,341.73,341.10,23 290 | 1982-03-01,1982.208,342.82,342.82,341.21,18 291 | 1982-04-01,1982.292,343.97,343.97,341.37,8 292 | 1982-05-01,1982.375,344.63,344.63,341.56,26 293 | 1982-06-01,1982.458,343.79,343.79,341.35,26 294 | 1982-07-01,1982.542,342.32,342.32,341.55,28 295 | 1982-08-01,1982.625,340.09,340.09,341.51,24 296 | 1982-09-01,1982.708,338.28,338.28,341.47,21 297 | 1982-10-01,1982.792,338.29,338.29,341.65,26 298 | 1982-11-01,1982.875,339.60,339.60,341.73,25 299 | 1982-12-01,1982.958,340.90,340.90,341.79,26 300 | 1983-01-01,1983.042,341.68,341.68,341.84,28 301 | 1983-02-01,1983.125,342.90,342.90,342.32,24 302 | 1983-03-01,1983.208,343.33,343.33,341.82,26 303 | 1983-04-01,1983.292,345.25,345.25,342.67,24 304 | 1983-05-01,1983.375,346.03,346.03,342.87,28 305 | 1983-06-01,1983.458,345.63,345.63,343.15,20 306 | 1983-07-01,1983.542,344.19,344.19,343.44,20 307 | 1983-08-01,1983.625,342.27,342.27,343.66,16 308 | 1983-09-01,1983.708,340.35,340.35,343.49,15 309 | 1983-10-01,1983.792,340.38,340.38,343.72,20 310 | 1983-11-01,1983.875,341.59,341.59,343.71,26 311 | 1983-12-01,1983.958,343.05,343.05,343.96,19 312 | 1984-01-01,1984.042,344.10,344.10,344.20,23 313 | 1984-02-01,1984.125,344.79,344.79,344.22,23 314 | 1984-03-01,1984.208,345.52,345.52,344.09,19 315 | 1984-04-01,1984.292,-99.99,346.84,344.27,2 316 | 1984-05-01,1984.375,347.63,347.63,344.45,21 317 | 1984-06-01,1984.458,346.97,346.97,344.51,21 318 | 1984-07-01,1984.542,345.53,345.53,344.77,21 319 | 1984-08-01,1984.625,343.55,343.55,344.95,12 320 | 1984-09-01,1984.708,341.40,341.40,344.58,14 321 | 1984-10-01,1984.792,341.67,341.67,345.00,12 322 | 1984-11-01,1984.875,343.10,343.10,345.20,18 323 | 1984-12-01,1984.958,344.70,344.70,345.57,12 324 | 1985-01-01,1985.042,345.21,345.21,345.31,23 325 | 1985-02-01,1985.125,346.16,346.16,345.61,17 326 | 1985-03-01,1985.208,347.74,347.74,346.37,16 327 | 1985-04-01,1985.292,348.33,348.33,345.79,19 328 | 1985-05-01,1985.375,349.06,349.06,345.91,24 329 | 1985-06-01,1985.458,348.38,348.38,345.93,23 330 | 1985-07-01,1985.542,346.72,346.72,345.90,18 331 | 1985-08-01,1985.625,345.02,345.02,346.35,18 332 | 1985-09-01,1985.708,343.27,343.27,346.40,25 333 | 1985-10-01,1985.792,343.13,343.13,346.42,20 334 | 1985-11-01,1985.875,344.49,344.49,346.61,22 335 | 1985-12-01,1985.958,345.88,345.88,346.81,25 336 | 1986-01-01,1986.042,346.56,346.56,346.59,23 337 | 1986-02-01,1986.125,347.28,347.28,346.74,25 338 | 1986-03-01,1986.208,348.01,348.01,346.68,17 339 | 1986-04-01,1986.292,349.77,349.77,347.22,22 340 | 1986-05-01,1986.375,350.38,350.38,347.26,18 341 | 1986-06-01,1986.458,349.93,349.93,347.52,17 342 | 1986-07-01,1986.542,348.16,348.16,347.33,20 343 | 1986-08-01,1986.625,346.08,346.08,347.42,18 344 | 1986-09-01,1986.708,345.22,345.22,348.36,17 345 | 1986-10-01,1986.792,344.51,344.51,347.77,26 346 | 1986-11-01,1986.875,345.93,345.93,348.04,23 347 | 1986-12-01,1986.958,347.21,347.21,348.12,24 348 | 1987-01-01,1987.042,348.52,348.52,348.46,26 349 | 1987-02-01,1987.125,348.73,348.73,348.02,25 350 | 1987-03-01,1987.208,349.73,349.73,348.29,22 351 | 1987-04-01,1987.292,351.31,351.31,348.76,26 352 | 1987-05-01,1987.375,352.09,352.09,349.01,27 353 | 1987-06-01,1987.458,351.53,351.53,349.19,21 354 | 1987-07-01,1987.542,350.11,350.11,349.39,16 355 | 1987-08-01,1987.625,347.92,347.92,349.36,19 356 | 1987-09-01,1987.708,346.52,346.52,349.71,23 357 | 1987-10-01,1987.792,346.59,346.59,349.86,22 358 | 1987-11-01,1987.875,347.96,347.96,350.07,22 359 | 1987-12-01,1987.958,349.16,349.16,350.05,27 360 | 1988-01-01,1988.042,350.39,350.39,350.38,24 361 | 1988-02-01,1988.125,351.64,351.64,350.94,24 362 | 1988-03-01,1988.208,352.41,352.41,350.87,25 363 | 1988-04-01,1988.292,353.69,353.69,351.01,27 364 | 1988-05-01,1988.375,354.21,354.21,351.06,28 365 | 1988-06-01,1988.458,353.72,353.72,351.36,26 366 | 1988-07-01,1988.542,352.69,352.69,352.02,27 367 | 1988-08-01,1988.625,350.40,350.40,351.92,26 368 | 1988-09-01,1988.708,348.92,348.92,352.14,27 369 | 1988-10-01,1988.792,349.13,349.13,352.41,26 370 | 1988-11-01,1988.875,350.20,350.20,352.34,25 371 | 1988-12-01,1988.958,351.41,351.41,352.34,28 372 | 1989-01-01,1989.042,352.91,352.91,352.85,27 373 | 1989-02-01,1989.125,353.27,353.27,352.54,25 374 | 1989-03-01,1989.208,353.96,353.96,352.47,29 375 | 1989-04-01,1989.292,355.64,355.64,352.97,28 376 | 1989-05-01,1989.375,355.86,355.86,352.67,28 377 | 1989-06-01,1989.458,355.37,355.37,352.97,26 378 | 1989-07-01,1989.542,353.99,353.99,353.31,25 379 | 1989-08-01,1989.625,351.81,351.81,353.39,24 380 | 1989-09-01,1989.708,350.05,350.05,353.33,23 381 | 1989-10-01,1989.792,350.25,350.25,353.52,25 382 | 1989-11-01,1989.875,351.49,351.49,353.65,27 383 | 1989-12-01,1989.958,352.85,352.85,353.80,27 384 | 1990-01-01,1990.042,353.80,353.80,353.74,25 385 | 1990-02-01,1990.125,355.04,355.04,354.33,28 386 | 1990-03-01,1990.208,355.73,355.73,354.23,28 387 | 1990-04-01,1990.292,356.32,356.32,353.68,28 388 | 1990-05-01,1990.375,357.32,357.32,354.16,29 389 | 1990-06-01,1990.458,356.34,356.34,353.96,29 390 | 1990-07-01,1990.542,354.84,354.84,354.20,30 391 | 1990-08-01,1990.625,353.01,353.01,354.62,22 392 | 1990-09-01,1990.708,351.31,351.31,354.61,27 393 | 1990-10-01,1990.792,351.62,351.62,354.88,28 394 | 1990-11-01,1990.875,353.07,353.07,355.13,24 395 | 1990-12-01,1990.958,354.33,354.33,355.19,28 396 | 1991-01-01,1991.042,354.84,354.84,354.82,28 397 | 1991-02-01,1991.125,355.73,355.73,355.02,27 398 | 1991-03-01,1991.208,357.23,357.23,355.67,30 399 | 1991-04-01,1991.292,358.66,358.66,356.02,30 400 | 1991-05-01,1991.375,359.13,359.13,356.00,29 401 | 1991-06-01,1991.458,358.13,358.13,355.80,29 402 | 1991-07-01,1991.542,356.19,356.19,355.59,24 403 | 1991-08-01,1991.625,353.85,353.85,355.46,25 404 | 1991-09-01,1991.708,352.25,352.25,355.56,27 405 | 1991-10-01,1991.792,352.35,352.35,355.62,27 406 | 1991-11-01,1991.875,353.81,353.81,355.80,28 407 | 1991-12-01,1991.958,355.12,355.12,355.93,30 408 | 1992-01-01,1992.042,356.25,356.25,356.20,31 409 | 1992-02-01,1992.125,357.11,357.11,356.38,27 410 | 1992-03-01,1992.208,357.86,357.86,356.27,24 411 | 1992-04-01,1992.292,359.09,359.09,356.39,27 412 | 1992-05-01,1992.375,359.59,359.59,356.41,26 413 | 1992-06-01,1992.458,359.33,359.33,356.97,30 414 | 1992-07-01,1992.542,357.01,357.01,356.44,26 415 | 1992-08-01,1992.625,354.94,354.94,356.62,23 416 | 1992-09-01,1992.708,352.95,352.95,356.29,26 417 | 1992-10-01,1992.792,353.32,353.32,356.63,29 418 | 1992-11-01,1992.875,354.32,354.32,356.38,29 419 | 1992-12-01,1992.958,355.57,355.57,356.39,31 420 | 1993-01-01,1993.042,357.00,357.00,356.96,28 421 | 1993-02-01,1993.125,357.31,357.31,356.44,28 422 | 1993-03-01,1993.208,358.47,358.47,356.76,30 423 | 1993-04-01,1993.292,359.27,359.27,356.59,25 424 | 1993-05-01,1993.375,360.19,360.19,357.03,30 425 | 1993-06-01,1993.458,359.52,359.52,357.12,28 426 | 1993-07-01,1993.542,357.33,357.33,356.76,25 427 | 1993-08-01,1993.625,355.64,355.64,357.32,27 428 | 1993-09-01,1993.708,354.03,354.03,357.39,23 429 | 1993-10-01,1993.792,354.12,354.12,357.49,28 430 | 1993-11-01,1993.875,355.41,355.41,357.54,29 431 | 1993-12-01,1993.958,356.91,356.91,357.80,30 432 | 1994-01-01,1994.042,358.24,358.24,358.13,27 433 | 1994-02-01,1994.125,358.92,358.92,358.09,25 434 | 1994-03-01,1994.208,359.99,359.99,358.29,29 435 | 1994-04-01,1994.292,361.23,361.23,358.46,28 436 | 1994-05-01,1994.375,361.65,361.65,358.46,30 437 | 1994-06-01,1994.458,360.81,360.81,358.44,27 438 | 1994-07-01,1994.542,359.38,359.38,358.79,31 439 | 1994-08-01,1994.625,357.46,357.46,359.16,24 440 | 1994-09-01,1994.708,355.73,355.73,359.17,24 441 | 1994-10-01,1994.792,356.07,356.07,359.49,28 442 | 1994-11-01,1994.875,357.53,357.53,359.68,28 443 | 1994-12-01,1994.958,358.98,358.98,359.83,28 444 | 1995-01-01,1995.042,359.92,359.92,359.79,30 445 | 1995-02-01,1995.125,360.86,360.86,360.05,28 446 | 1995-03-01,1995.208,361.83,361.83,360.22,29 447 | 1995-04-01,1995.292,363.30,363.30,360.62,29 448 | 1995-05-01,1995.375,363.69,363.69,360.58,29 449 | 1995-06-01,1995.458,363.19,363.19,360.84,27 450 | 1995-07-01,1995.542,361.64,361.64,360.97,28 451 | 1995-08-01,1995.625,359.12,359.12,360.73,25 452 | 1995-09-01,1995.708,358.17,358.17,361.55,24 453 | 1995-10-01,1995.792,357.99,357.99,361.37,29 454 | 1995-11-01,1995.875,359.45,359.45,361.59,27 455 | 1995-12-01,1995.958,360.68,360.68,361.53,30 456 | 1996-01-01,1996.042,362.07,362.07,361.85,29 457 | 1996-02-01,1996.125,363.24,363.24,362.35,27 458 | 1996-03-01,1996.208,364.17,364.17,362.53,27 459 | 1996-04-01,1996.292,364.57,364.57,361.86,29 460 | 1996-05-01,1996.375,365.13,365.13,362.10,30 461 | 1996-06-01,1996.458,364.92,364.92,362.69,30 462 | 1996-07-01,1996.542,363.55,363.55,362.85,31 463 | 1996-08-01,1996.625,361.38,361.38,362.98,28 464 | 1996-09-01,1996.708,359.54,359.54,362.99,25 465 | 1996-10-01,1996.792,359.58,359.58,362.97,29 466 | 1996-11-01,1996.875,360.89,360.89,363.03,29 467 | 1996-12-01,1996.958,362.25,362.25,363.08,29 468 | 1997-01-01,1997.042,363.09,363.09,362.88,31 469 | 1997-02-01,1997.125,364.03,364.03,363.22,27 470 | 1997-03-01,1997.208,364.51,364.51,362.88,31 471 | 1997-04-01,1997.292,366.35,366.35,363.68,21 472 | 1997-05-01,1997.375,366.64,366.64,363.74,29 473 | 1997-06-01,1997.458,365.59,365.59,363.41,27 474 | 1997-07-01,1997.542,364.31,364.31,363.60,24 475 | 1997-08-01,1997.625,362.25,362.25,363.84,25 476 | 1997-09-01,1997.708,360.29,360.29,363.68,26 477 | 1997-10-01,1997.792,360.82,360.82,364.12,27 478 | 1997-11-01,1997.875,362.49,362.49,364.56,30 479 | 1997-12-01,1997.958,364.38,364.38,365.15,30 480 | 1998-01-01,1998.042,365.26,365.26,365.07,30 481 | 1998-02-01,1998.125,365.98,365.98,365.17,28 482 | 1998-03-01,1998.208,367.24,367.24,365.60,31 483 | 1998-04-01,1998.292,368.66,368.66,366.03,29 484 | 1998-05-01,1998.375,369.42,369.42,366.55,30 485 | 1998-06-01,1998.458,368.99,368.99,366.80,28 486 | 1998-07-01,1998.542,367.82,367.82,367.14,23 487 | 1998-08-01,1998.625,365.95,365.95,367.55,30 488 | 1998-09-01,1998.708,364.02,364.02,367.37,28 489 | 1998-10-01,1998.792,364.40,364.40,367.67,30 490 | 1998-11-01,1998.875,365.52,365.52,367.56,23 491 | 1998-12-01,1998.958,367.13,367.13,367.88,26 492 | 1999-01-01,1999.042,368.18,368.18,367.96,27 493 | 1999-02-01,1999.125,369.07,369.07,368.26,22 494 | 1999-03-01,1999.208,369.68,369.68,368.08,25 495 | 1999-04-01,1999.292,370.99,370.99,368.45,29 496 | 1999-05-01,1999.375,370.96,370.96,368.15,26 497 | 1999-06-01,1999.458,370.30,370.30,368.13,26 498 | 1999-07-01,1999.542,369.45,369.45,368.77,27 499 | 1999-08-01,1999.625,366.90,366.90,368.48,25 500 | 1999-09-01,1999.708,364.81,364.81,368.13,28 501 | 1999-10-01,1999.792,365.37,365.37,368.64,31 502 | 1999-11-01,1999.875,366.72,366.72,368.71,28 503 | 1999-12-01,1999.958,368.10,368.10,368.77,26 504 | 2000-01-01,2000.042,369.29,369.29,369.08,26 505 | 2000-02-01,2000.125,369.54,369.54,368.83,19 506 | 2000-03-01,2000.208,370.60,370.60,369.09,30 507 | 2000-04-01,2000.292,371.81,371.81,369.28,27 508 | 2000-05-01,2000.375,371.58,371.58,368.71,28 509 | 2000-06-01,2000.458,371.70,371.70,369.50,28 510 | 2000-07-01,2000.542,369.86,369.86,369.20,25 511 | 2000-08-01,2000.625,368.13,368.13,369.72,27 512 | 2000-09-01,2000.708,367.00,367.00,370.30,26 513 | 2000-10-01,2000.792,367.03,367.03,370.26,30 514 | 2000-11-01,2000.875,368.37,368.37,370.32,25 515 | 2000-12-01,2000.958,369.67,369.67,370.30,30 516 | 2001-01-01,2001.042,370.59,370.59,370.43,30 517 | 2001-02-01,2001.125,371.51,371.51,370.78,26 518 | 2001-03-01,2001.208,372.43,372.43,370.87,26 519 | 2001-04-01,2001.292,373.37,373.37,370.81,29 520 | 2001-05-01,2001.375,373.85,373.85,370.94,24 521 | 2001-06-01,2001.458,373.21,373.21,370.99,26 522 | 2001-07-01,2001.542,371.51,371.51,370.90,25 523 | 2001-08-01,2001.625,369.61,369.61,371.22,27 524 | 2001-09-01,2001.708,368.18,368.18,371.44,28 525 | 2001-10-01,2001.792,368.45,368.45,371.69,31 526 | 2001-11-01,2001.875,369.76,369.76,371.74,24 527 | 2001-12-01,2001.958,371.24,371.24,371.92,29 528 | 2002-01-01,2002.042,372.53,372.53,372.30,28 529 | 2002-02-01,2002.125,373.20,373.20,372.33,28 530 | 2002-03-01,2002.208,374.12,374.12,372.44,24 531 | 2002-04-01,2002.292,375.02,375.02,372.37,29 532 | 2002-05-01,2002.375,375.76,375.76,372.81,29 533 | 2002-06-01,2002.458,375.52,375.52,373.30,28 534 | 2002-07-01,2002.542,374.01,374.01,373.42,26 535 | 2002-08-01,2002.625,371.85,371.85,373.52,28 536 | 2002-09-01,2002.708,370.75,370.75,374.11,23 537 | 2002-10-01,2002.792,370.55,370.55,373.88,31 538 | 2002-11-01,2002.875,372.25,372.25,374.34,29 539 | 2002-12-01,2002.958,373.79,373.79,374.54,31 540 | 2003-01-01,2003.042,374.88,374.88,374.63,30 541 | 2003-02-01,2003.125,375.64,375.64,374.77,27 542 | 2003-03-01,2003.208,376.46,376.46,374.80,28 543 | 2003-04-01,2003.292,377.73,377.73,375.06,27 544 | 2003-05-01,2003.375,378.60,378.60,375.55,30 545 | 2003-06-01,2003.458,378.28,378.28,376.03,25 546 | 2003-07-01,2003.542,376.70,376.70,376.19,29 547 | 2003-08-01,2003.625,374.38,374.38,376.08,23 548 | 2003-09-01,2003.708,373.17,373.17,376.48,25 549 | 2003-10-01,2003.792,373.14,373.14,376.47,30 550 | 2003-11-01,2003.875,374.66,374.66,376.81,26 551 | 2003-12-01,2003.958,375.99,375.99,376.75,27 552 | 2004-01-01,2004.042,377.00,377.00,376.78,30 553 | 2004-02-01,2004.125,377.87,377.87,377.02,29 554 | 2004-03-01,2004.208,378.88,378.88,377.23,28 555 | 2004-04-01,2004.292,380.35,380.35,377.62,26 556 | 2004-05-01,2004.375,380.62,380.62,377.48,28 557 | 2004-06-01,2004.458,379.69,379.69,377.39,21 558 | 2004-07-01,2004.542,377.47,377.47,376.94,25 559 | 2004-08-01,2004.625,376.01,376.01,377.74,16 560 | 2004-09-01,2004.708,374.25,374.25,377.62,15 561 | 2004-10-01,2004.792,374.46,374.46,377.82,29 562 | 2004-11-01,2004.875,376.16,376.16,378.31,29 563 | 2004-12-01,2004.958,377.51,377.51,378.31,30 564 | 2005-01-01,2005.042,378.46,378.46,378.21,31 565 | 2005-02-01,2005.125,379.73,379.73,378.93,24 566 | 2005-03-01,2005.208,380.77,380.77,379.27,26 567 | 2005-04-01,2005.292,382.29,382.29,379.65,26 568 | 2005-05-01,2005.375,382.45,382.45,379.31,31 569 | 2005-06-01,2005.458,382.22,382.22,379.88,28 570 | 2005-07-01,2005.542,380.74,380.74,380.18,29 571 | 2005-08-01,2005.625,378.74,378.74,380.42,26 572 | 2005-09-01,2005.708,376.70,376.70,380.01,26 573 | 2005-10-01,2005.792,377.00,377.00,380.30,14 574 | 2005-11-01,2005.875,378.35,378.35,380.50,23 575 | 2005-12-01,2005.958,380.11,380.11,380.90,26 576 | 2006-01-01,2006.042,381.38,381.38,381.14,24 577 | 2006-02-01,2006.125,382.19,382.19,381.38,25 578 | 2006-03-01,2006.208,382.67,382.67,381.14,30 579 | 2006-04-01,2006.292,384.61,384.61,381.91,25 580 | 2006-05-01,2006.375,385.03,385.03,381.87,24 581 | 2006-06-01,2006.458,384.05,384.05,381.75,28 582 | 2006-07-01,2006.542,382.46,382.46,381.91,24 583 | 2006-08-01,2006.625,380.41,380.41,382.08,27 584 | 2006-09-01,2006.708,378.85,378.85,382.16,27 585 | 2006-10-01,2006.792,379.13,379.13,382.46,23 586 | 2006-11-01,2006.875,380.15,380.15,382.33,29 587 | 2006-12-01,2006.958,381.82,381.82,382.64,27 588 | 2007-01-01,2007.042,382.89,382.89,382.67,24 589 | 2007-02-01,2007.125,383.90,383.90,383.01,21 590 | 2007-03-01,2007.208,384.58,384.58,382.94,26 591 | 2007-04-01,2007.292,386.50,386.50,383.71,26 592 | 2007-05-01,2007.375,386.56,386.56,383.34,29 593 | 2007-06-01,2007.458,386.10,386.10,383.84,26 594 | 2007-07-01,2007.542,384.50,384.50,384.02,27 595 | 2007-08-01,2007.625,381.99,381.99,383.70,22 596 | 2007-09-01,2007.708,380.96,380.96,384.32,21 597 | 2007-10-01,2007.792,381.12,381.12,384.47,29 598 | 2007-11-01,2007.875,382.45,382.45,384.65,30 599 | 2007-12-01,2007.958,383.95,383.95,384.83,21 600 | 2008-01-01,2008.042,385.52,385.52,385.28,31 601 | 2008-02-01,2008.125,385.82,385.82,384.96,26 602 | 2008-03-01,2008.208,386.03,386.03,384.48,30 603 | 2008-04-01,2008.292,387.21,387.21,384.58,24 604 | 2008-05-01,2008.375,388.54,388.54,385.45,25 605 | 2008-06-01,2008.458,387.76,387.76,385.46,23 606 | 2008-07-01,2008.542,386.36,386.36,385.80,10 607 | 2008-08-01,2008.625,384.09,384.09,385.75,25 608 | 2008-09-01,2008.708,383.18,383.18,386.46,27 609 | 2008-10-01,2008.792,382.99,382.99,386.27,23 610 | 2008-11-01,2008.875,384.19,384.19,386.37,28 611 | 2008-12-01,2008.958,385.56,385.56,386.41,29 612 | 2009-01-01,2009.042,386.94,386.94,386.63,30 613 | 2009-02-01,2009.125,387.47,387.47,386.59,26 614 | 2009-03-01,2009.208,388.82,388.82,387.32,28 615 | 2009-04-01,2009.292,389.55,389.55,386.92,29 616 | 2009-05-01,2009.375,390.14,390.14,387.02,30 617 | 2009-06-01,2009.458,389.48,389.48,387.24,29 618 | 2009-07-01,2009.542,388.03,388.03,387.55,22 619 | 2009-08-01,2009.625,386.11,386.11,387.80,27 620 | 2009-09-01,2009.708,384.74,384.74,388.01,28 621 | 2009-10-01,2009.792,384.43,384.43,387.68,30 622 | 2009-11-01,2009.875,386.02,386.02,388.16,30 623 | 2009-12-01,2009.958,387.42,387.42,388.23,20 624 | 2010-01-01,2010.042,388.71,388.71,388.41,30 625 | 2010-02-01,2010.125,390.20,390.20,389.26,20 626 | 2010-03-01,2010.208,391.17,391.17,389.65,25 627 | 2010-04-01,2010.292,392.46,392.46,389.89,26 628 | 2010-05-01,2010.375,393.00,393.00,389.88,28 629 | 2010-06-01,2010.458,392.15,392.15,389.89,28 630 | 2010-07-01,2010.542,390.20,390.20,389.72,29 631 | 2010-08-01,2010.625,388.35,388.35,390.01,26 632 | 2010-09-01,2010.708,386.85,386.85,390.14,29 633 | 2010-10-01,2010.792,387.24,387.24,390.53,31 634 | 2010-11-01,2010.875,388.67,388.67,390.79,28 635 | 2010-12-01,2010.958,389.79,389.79,390.60,29 636 | 2011-01-01,2011.042,391.33,391.33,391.03,29 637 | 2011-02-01,2011.125,391.86,391.86,390.94,28 638 | 2011-03-01,2011.208,392.60,392.60,391.07,29 639 | 2011-04-01,2011.292,393.25,393.25,390.63,28 640 | 2011-05-01,2011.375,394.19,394.19,391.02,30 641 | 2011-06-01,2011.458,393.73,393.73,391.44,28 642 | 2011-07-01,2011.542,392.51,392.51,392.04,26 643 | 2011-08-01,2011.625,390.13,390.13,391.83,27 644 | 2011-09-01,2011.708,389.08,389.08,392.40,26 645 | 2011-10-01,2011.792,389.00,389.00,392.33,31 646 | 2011-11-01,2011.875,390.28,390.28,392.44,28 647 | 2011-12-01,2011.958,391.86,391.86,392.66,28 648 | 2012-01-01,2012.042,393.12,393.12,392.89,30 649 | 2012-02-01,2012.125,393.86,393.86,393.04,26 650 | 2012-03-01,2012.208,394.40,394.40,392.80,30 651 | 2012-04-01,2012.292,396.18,396.18,393.43,29 652 | 2012-05-01,2012.375,396.74,396.74,393.54,30 653 | 2012-06-01,2012.458,395.71,395.71,393.45,28 654 | 2012-07-01,2012.542,394.36,394.36,393.92,26 655 | 2012-08-01,2012.625,392.39,392.39,394.17,30 656 | 2012-09-01,2012.708,391.11,391.11,394.54,27 657 | 2012-10-01,2012.792,391.05,391.05,394.41,28 658 | 2012-11-01,2012.875,392.98,392.98,395.02,29 659 | 2012-12-01,2012.958,394.34,394.34,395.04,29 660 | 2013-01-01,2013.042,395.55,395.55,395.38,28 661 | 2013-02-01,2013.125,396.80,396.80,395.98,25 662 | 2013-03-01,2013.208,397.43,397.43,395.81,30 663 | 2013-04-01,2013.292,398.41,398.41,395.56,22 664 | 2013-05-01,2013.375,399.78,399.78,396.46,28 665 | 2013-06-01,2013.458,398.61,398.61,396.30,26 666 | 2013-07-01,2013.542,397.32,397.32,396.91,21 667 | 2013-08-01,2013.625,395.20,395.20,397.04,27 668 | 2013-09-01,2013.708,393.45,393.45,396.94,27 669 | 2013-10-01,2013.792,393.70,393.70,397.05,28 670 | 2013-11-01,2013.875,395.16,395.16,397.20,30 671 | 2013-12-01,2013.958,396.84,396.84,397.61,30 672 | 2014-01-01,2014.042,397.85,397.85,397.69,31 673 | 2014-02-01,2014.125,398.01,398.01,397.20,26 674 | 2014-03-01,2014.208,399.77,399.77,398.15,24 675 | 2014-04-01,2014.292,401.38,401.38,398.53,28 676 | 2014-05-01,2014.375,401.78,401.78,398.45,22 677 | 2014-06-01,2014.458,401.25,401.25,398.95,28 678 | 2014-07-01,2014.542,399.10,399.10,398.69,25 679 | 2014-08-01,2014.625,397.03,397.03,398.87,21 680 | 2014-09-01,2014.708,395.38,395.38,398.87,21 681 | 2014-10-01,2014.792,396.03,396.03,399.38,24 682 | 2014-11-01,2014.875,397.28,397.28,399.31,27 683 | 2014-12-01,2014.958,398.91,398.91,399.68,29 684 | 2015-01-01,2015.042,399.98,399.98,399.81,30 685 | 2015-02-01,2015.125,400.28,400.28,399.47,27 686 | 2015-03-01,2015.208,401.54,401.54,399.92,24 687 | 2015-04-01,2015.292,403.28,403.28,400.43,27 688 | 2015-05-01,2015.375,403.96,403.96,400.63,30 689 | 2015-06-01,2015.458,402.80,402.80,400.50,28 690 | 2015-07-01,2015.542,401.31,401.31,400.90,23 691 | 2015-08-01,2015.625,398.93,398.93,400.77,28 692 | 2015-09-01,2015.708,397.63,397.63,401.12,25 693 | 2015-10-01,2015.792,398.29,398.29,401.65,28 694 | 2015-11-01,2015.875,400.16,400.16,402.19,27 695 | 2015-12-01,2015.958,401.85,401.85,402.62,30 696 | 2016-01-01,2016.042,402.52,402.52,402.36,27 697 | 2016-02-01,2016.125,404.04,404.04,403.22,26 698 | 2016-03-01,2016.208,404.83,404.83,403.21,29 699 | 2016-04-01,2016.292,407.42,407.42,404.57,25 700 | 2016-05-01,2016.375,407.70,407.70,404.38,29 701 | 2016-06-01,2016.458,406.81,406.81,404.51,26 702 | 2016-07-01,2016.542,404.39,404.39,403.98,28 703 | 2016-08-01,2016.625,402.25,402.25,404.09,23 704 | 2016-09-01,2016.708,401.03,401.03,404.52,24 705 | 2016-10-01,2016.792,401.57,401.57,404.93,29 706 | 2016-11-01,2016.875,403.53,403.53,405.57,27 707 | 2016-12-01,2016.958,404.48,404.48,405.25,29 708 | -------------------------------------------------------------------------------- /test/fixtures/co2-ppm/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "co2-ppm", 3 | "title": "CO2 PPM - Trends in Atmospheric Carbon Dioxide", 4 | "description": "Data are sourced from the US Government's Earth System Research Laboratory, Global Monitoring Division. Two main series are provided: the Mauna Loa series (which has the longest continuous series since 1958) and a Global Average series (a global average over marine surface sites).", 5 | "homepage": "http://www.esrl.noaa.gov/gmd/ccgg/trends", 6 | "version": "0.1.0", 7 | "license": "ODC-PDDL-1.0", 8 | "sources": [ 9 | { 10 | "name": "Trends in Atmospheric Carbon Dioxide, Mauna Loa, Hawaii", 11 | "web": "http://www.esrl.noaa.gov/gmd/ccgg/trends/index.html" 12 | }, 13 | { 14 | "name": "Trends in Atmospheric Carbon Dioxide, Global", 15 | "web": "http://www.esrl.noaa.gov/gmd/ccgg/trends/global.html" 16 | } 17 | ], 18 | "resources": [ 19 | { 20 | "name": "co2-mm-mlo", 21 | "path": "data/co2-mm-mlo.csv", 22 | "format": "csv", 23 | "mediatype": "text/csv", 24 | "schema": { 25 | "fields": [ 26 | { 27 | "name": "Date", 28 | "type": "date", 29 | "description": "YYYY-MM-DD" 30 | }, 31 | { 32 | "name": "Decimal Date", 33 | "type": "number", 34 | "description": "" 35 | }, 36 | { 37 | "name": "Average", 38 | "type": "number", 39 | "description": "The monthly mean CO2 mole fraction determined from daily averages. If there are missing days concentrated either early or late in the month, the monthly mean is corrected to the middle of the month using the average seasonal cycle. Missing months are denoted by -99.99." 40 | }, 41 | { 42 | "name": "Interpolated", 43 | "type": "number", 44 | "description": "Values from the average column and interpolated values where data are missing. Interpolated values are computed in two steps. First, we compute for each month the average seasonal cycle in a 7-year window around each monthly value. In this way the seasonal cycle is allowed to change slowly over time. We then determine the trend value for each month by removing the seasonal cycle; this result is shown in the trend column. Trend values are linearly interpolated for missing months. The interpolated monthly mean is then the sum of the average seasonal cycle value and the trend value for the missing month." 45 | }, 46 | { 47 | "name": "Trend", 48 | "type": "number", 49 | "description": "Seasonally corrected." 50 | }, 51 | { 52 | "name": "Number of Days", 53 | "type": "number", 54 | "description": "-1 denotes no data for number of daily averages in the month." 55 | } 56 | ] 57 | } 58 | }, 59 | { 60 | "name": "co2-annmean-mlo", 61 | "path": "data/co2-annmean-mlo.csv", 62 | "format": "csv", 63 | "mediatype": "text/csv", 64 | "schema": { 65 | "fields": [ 66 | { 67 | "name": "Year", 68 | "type": "date", 69 | "description": "" 70 | }, 71 | { 72 | "name": "Mean", 73 | "type": "number", 74 | "description": "" 75 | }, 76 | { 77 | "name": "Uncertainty", 78 | "type": "number", 79 | "description": "The estimated uncertainty in the annual mean is the standard deviation of the differences of annual mean values determined independently by NOAA/ESRL and the Scripps Institution of Oceanography." 80 | } 81 | ] 82 | } 83 | }, 84 | { 85 | "name": "co2-gr-mlo", 86 | "path": "data/co2-gr-mlo.csv", 87 | "format": "csv", 88 | "mediatype": "text/csv", 89 | "schema": { 90 | "fields": [ 91 | { 92 | "name": "Year", 93 | "type": "date", 94 | "description": "" 95 | }, 96 | { 97 | "name": "Annual Increase", 98 | "type": "number", 99 | "description": "Annual CO2 mole fraction increase (ppm) from Jan 1 through Dec 31." 100 | }, 101 | { 102 | "name": "Uncertainty", 103 | "type": "number", 104 | "description": "Estimated from the standard deviation of the differences between monthly mean values determined independently by the Scripps Institution of Oceanography and by NOAA/ESRL." 105 | } 106 | ] 107 | } 108 | }, 109 | { 110 | "name": "co2-mm-gl", 111 | "path": "data/co2-mm-gl.csv", 112 | "format": "csv", 113 | "mediatype": "text/csv", 114 | "schema": { 115 | "fields": [ 116 | { 117 | "name": "Date", 118 | "type": "date", 119 | "description": "YYYY-MM-DD" 120 | }, 121 | { 122 | "name": "Decimal Date", 123 | "type": "number", 124 | "description": "" 125 | }, 126 | { 127 | "name": "Average", 128 | "type": "number", 129 | "description": "" 130 | }, 131 | { 132 | "name": "Trend", 133 | "type": "number", 134 | "description": "" 135 | } 136 | ] 137 | } 138 | }, 139 | { 140 | "name": "co2-annmean-gl", 141 | "path": "data/co2-annmean-gl.csv", 142 | "format": "csv", 143 | "mediatype": "text/csv", 144 | "schema": { 145 | "fields": [ 146 | { 147 | "name": "Year", 148 | "type": "date", 149 | "description": "" 150 | }, 151 | { 152 | "name": "Mean", 153 | "type": "number", 154 | "description": "" 155 | }, 156 | { 157 | "name": "Uncertainty", 158 | "type": "number", 159 | "description": "The uncertainty in the global annual mean is estimated using a monte carlo technique that computes 100 global annual averages, each time using a slightly different set of measurement records from the NOAA ESRL cooperative air sampling network. The reported uncertainty is the mean of the standard deviations for each annual average using this technique. Please see Conway et al., 1994, JGR, vol. 99, no. D11. for a complete discussion." 160 | } 161 | ] 162 | } 163 | }, 164 | { 165 | "name": "co2-gr-gl", 166 | "path": "data/co2-gr-gl.csv", 167 | "format": "csv", 168 | "mediatype": "text/csv", 169 | "schema": { 170 | "fields": [ 171 | { 172 | "name": "Year", 173 | "type": "date", 174 | "description": "" 175 | }, 176 | { 177 | "name": "Annual Increase", 178 | "type": "number", 179 | "description": "Annual CO2 mole fraction increase (ppm) from Jan 1 through Dec 31." 180 | }, 181 | { 182 | "name": "Uncertainty", 183 | "type": "number", 184 | "description": "The uncertainty in the global annual mean growth rate is estimated using a monte carlo technique that computes 100 time series of global annual growth rates, each time using measurement records from a different sampling of sites from the NOAA ESRL cooperative air sampling network. Each year has a different uncertainty. Please see Conway et al., 1994, JGR, vol. 99, no. D11. for a complete discussion. The last one or two years listed could have preliminary uncertainties set equal to the average uncertainty since 1980. Before 1980 the global growth rate has been approximated by taking the average of Mauna Loa and the South Pole, correcting for the offset between (MLO+SPO)/2 and the global Marine Boundary Layer, as described in Ballantyne et al, 2012." 185 | } 186 | ] 187 | } 188 | } 189 | ], 190 | "views": [ 191 | { 192 | "id": "graph", 193 | "label": "Graph", 194 | "type": "Graph", 195 | "state": { 196 | "group": "Date", 197 | "series": ["Interpolated", "Trend"], 198 | "graphType": "lines-and-points" 199 | } 200 | } 201 | ] 202 | } 203 | -------------------------------------------------------------------------------- /test/fixtures/dp-with-inline-readme/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "abc", 3 | "readme": "This is the README", 4 | "resources": [] 5 | } 6 | -------------------------------------------------------------------------------- /test/fixtures/encodings/iso8859.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/frictionless-js/fa7b8f4f36880159dada651309a81a2068e7d0c2/test/fixtures/encodings/iso8859.csv -------------------------------------------------------------------------------- /test/fixtures/encodings/western-macos-roman.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/frictionless-js/fa7b8f4f36880159dada651309a81a2068e7d0c2/test/fixtures/encodings/western-macos-roman.csv -------------------------------------------------------------------------------- /test/fixtures/sample-cyrillic-encoding.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/frictionless-js/fa7b8f4f36880159dada651309a81a2068e7d0c2/test/fixtures/sample-cyrillic-encoding.csv -------------------------------------------------------------------------------- /test/fixtures/sample-multi-sheet.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/frictionless-js/fa7b8f4f36880159dada651309a81a2068e7d0c2/test/fixtures/sample-multi-sheet.xlsx -------------------------------------------------------------------------------- /test/fixtures/sample.csv: -------------------------------------------------------------------------------- 1 | number,string,boolean 2 | 1,two,true 3 | 3,four,false 4 | -------------------------------------------------------------------------------- /test/fixtures/sample.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/frictionless-js/fa7b8f4f36880159dada651309a81a2068e7d0c2/test/fixtures/sample.xls -------------------------------------------------------------------------------- /test/fixtures/sample.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/frictionless-js/fa7b8f4f36880159dada651309a81a2068e7d0c2/test/fixtures/sample.xlsx -------------------------------------------------------------------------------- /test/fixtures/semicolon-delimited.csv: -------------------------------------------------------------------------------- 1 | id;name 2 | 1;John 3 | 2;David 4 | -------------------------------------------------------------------------------- /test/fixtures/some file.name.ext: -------------------------------------------------------------------------------- 1 | Content 2 | -------------------------------------------------------------------------------- /test/parser/csv.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import toArray from 'stream-to-array' 3 | import { csvParser, guessParseOptions } from '../../src/parser/csv' 4 | import * as data from '../../src/index' 5 | 6 | describe('csvParser', function () { 7 | this.timeout(30000) // all tests in this suite get 10 seconds before timeout 8 | 9 | it('csvParser iso8859 file encoding', async () => { 10 | const path_ = 'test/fixtures/encodings/iso8859.csv' 11 | const file = data.open(path_) 12 | const parsed_file = await csvParser(file) 13 | const rows = await toArray(parsed_file) 14 | assert.deepStrictEqual(rows[1], ['Réunion', 'ECS', '1989', '838462813']) 15 | }) 16 | 17 | it('csvParser iso8859 remote file encoding', async () => { 18 | const url = 19 | 'https://raw.githubusercontent.com/frictionlessdata/test-data/master/files/csv/encodings/iso8859.csv' 20 | const file = data.open(url) 21 | const parsed_file = await csvParser(file) 22 | const rows = await toArray(parsed_file) 23 | assert.deepStrictEqual(rows[1], ['R�union', 'ECS', '1989', '838462813']) 24 | }) 25 | 26 | it('csvParser western-macos-roman file encoding', async () => { 27 | const path_ = 'test/fixtures/encodings/western-macos-roman.csv' 28 | const file = data.open(path_) 29 | const parsed_file = await csvParser(file) 30 | const rows = await toArray(parsed_file) 31 | assert.deepStrictEqual(rows[1], ['RŽunion', 'ECS', '1989', '838462813']) 32 | }) 33 | 34 | it('csvParser western-macos-roman remote file encoding', async () => { 35 | const url = 36 | 'https://raw.githubusercontent.com/frictionlessdata/test-data/master/files/csv/encodings/western-macos-roman.csv' 37 | const file = data.open(url) 38 | const parsed_file = await csvParser(file) 39 | const rows = await toArray(parsed_file) 40 | assert.notDeepStrictEqual(rows[1], ['Réunion', 'ECS', '1989', '838462813']) 41 | }) 42 | 43 | it('guessParseOptions for local data', async () => { 44 | const path_ = 'test/fixtures/semicolon-delimited.csv' 45 | let file = data.open(path_) 46 | let parseOptions = await guessParseOptions(file) 47 | assert.deepStrictEqual(parseOptions.delimiter, ';') 48 | }) 49 | 50 | it('guessParseOptions for local data', async () => { 51 | const url_ = 52 | 'https://raw.githubusercontent.com/frictionlessdata/test-data/master/files/csv/separators/semicolon.csv' 53 | let file = await data.open(url_) 54 | let parseOptions = await guessParseOptions(file) 55 | assert.deepStrictEqual(parseOptions.delimiter, ';') 56 | }) 57 | }) 58 | -------------------------------------------------------------------------------- /test/parser/xlsx.js: -------------------------------------------------------------------------------- 1 | import assert from 'assert' 2 | import toArray from 'stream-to-array' 3 | import { xlsxParser } from '../../src/parser/xlsx' 4 | import * as data from '../../src/index' 5 | 6 | describe('xlsxParser', function () { 7 | it('xlsxParser works with local XLSX files', async () => { 8 | const path_ = 'test/fixtures/sample.xlsx' 9 | const file = await data.open(path_) 10 | const rows = await toArray(await xlsxParser(file)) 11 | assert.deepStrictEqual(rows[0], ['number', 'string', 'boolean']) 12 | }) 13 | 14 | it('xlsxParser works with keyed option', async () => { 15 | const path_ = 'test/fixtures/sample.xls' 16 | const file = await data.open(path_) 17 | const keyed = true 18 | const rows = await toArray(await xlsxParser(file, keyed)) 19 | assert.deepStrictEqual(rows[0], { 20 | number: '1', 21 | string: 'two', 22 | boolean: 'TRUE', 23 | }) 24 | }) 25 | 26 | it('xlsxParser works with specified sheet index', async () => { 27 | const path_ = 'test/fixtures/sample-multi-sheet.xlsx' 28 | const file = await data.open(path_) 29 | let sheetIdx = 0 30 | let rows = await toArray(await xlsxParser(file, false, sheetIdx)) 31 | assert.deepStrictEqual(rows[0], ['a', 'b', 'c']) 32 | 33 | sheetIdx = 1 34 | rows = await toArray(await xlsxParser(file, false, sheetIdx)) 35 | assert.deepStrictEqual(rows[0], ['d', 'e', 'f']) 36 | }) 37 | 38 | it('xlsxParser works with specified sheet name', async () => { 39 | const path_ = 'test/fixtures/sample-multi-sheet.xlsx' 40 | const file = await data.open(path_) 41 | let sheetName = 'Sheet1' 42 | let rows = await toArray(await xlsxParser(file, false, sheetName)) 43 | assert.deepStrictEqual(rows[0], ['a', 'b', 'c']) 44 | 45 | sheetName = 'Sheet2' 46 | rows = await toArray(await xlsxParser(file, false, sheetName)) 47 | assert.deepStrictEqual(rows[0], ['d', 'e', 'f']) 48 | }) 49 | }) 50 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | 3 | const createConfig = (target) => { 4 | return { 5 | mode: 'production', 6 | devtool: 'source-map', 7 | context: path.resolve(__dirname), 8 | entry: { 9 | index: `./src/index.js`, 10 | }, 11 | target: target, 12 | output: { 13 | path: path.resolve(__dirname, 'dist/browser'), 14 | filename: `bundle.js`, 15 | library: 'data', 16 | }, 17 | module: { 18 | rules: [ 19 | { 20 | use: { 21 | loader: 'babel-loader', 22 | options: { presets: ['@babel/preset-env'] }, 23 | }, 24 | test: /\.(js|jsx)$/, 25 | exclude: /node_modules/, 26 | }, 27 | ], 28 | }, 29 | node: { fs: 'empty' }, 30 | } 31 | } 32 | 33 | module.exports = [ 34 | createConfig('web'), 35 | ] 36 | --------------------------------------------------------------------------------