├── .coveralls.yml ├── .github ├── issue_template.md ├── pull_request_template.md ├── stale.yml └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .php_cs.dist ├── CONTRIBUTING.md ├── LEAD.md ├── LICENSE ├── Makefile ├── README.md ├── composer.json ├── src ├── DataStreams │ ├── BaseDataStream.php │ ├── DefaultDataStream.php │ ├── TabularDataStream.php │ └── TabularInlineDataStream.php ├── Datapackages │ ├── BaseDatapackage.php │ ├── CustomDatapackage.php │ ├── DefaultDatapackage.php │ └── TabularDatapackage.php ├── Exceptions │ ├── DataStreamOpenException.php │ ├── DataStreamValidationException.php │ ├── DatapackageInvalidSourceException.php │ ├── DatapackageValidationFailedException.php │ └── ResourceValidationFailedException.php ├── Factory.php ├── Package.php ├── Registry.php ├── Resource.php ├── Resources │ ├── BaseResource.php │ ├── CustomResource.php │ ├── DefaultResource.php │ └── TabularResource.php ├── Utils.php └── Validators │ ├── BaseValidationError.php │ ├── BaseValidator.php │ ├── DatapackageValidationError.php │ ├── DatapackageValidator.php │ ├── ResourceValidationError.php │ ├── ResourceValidator.php │ └── schemas │ ├── CHANGELOG │ ├── data-package.json │ ├── data-resource.json │ ├── fiscal-data-package.json │ ├── registry.json │ ├── table-schema.json │ ├── tabular-data-package.json │ └── tabular-data-resource.json ├── tests ├── DatapackageTest.php ├── FactoryTest.php ├── Mocks │ ├── MockDatapackageValidator.php │ ├── MockDefaultDatapackage.php │ ├── MockDefaultResource.php │ ├── MockFactory.php │ ├── MockResource.php │ ├── MockResourceValidator.php │ ├── MyCustomDatapackage.php │ └── MyCustomResource.php ├── RegistryTest.php ├── ResourceTest.php ├── autoload.php └── fixtures │ ├── bar.txt │ ├── baz.txt │ ├── committees │ ├── datapackage-lolsv.json │ ├── datapackage.json │ ├── kns_committee.csv │ └── kns_committee.lolsv │ ├── datahub-country-list │ ├── data.csv │ ├── data_csv.csv │ ├── data_json.json │ ├── datapackage.json │ └── datapackage_zip.zip │ ├── datapackage_zip.zip │ ├── default_resource_invalid_data.json │ ├── fiscal-datapackage │ ├── budget.csv │ ├── datapackage.json │ └── entities.csv │ ├── foo.txt │ ├── invalid_tabular_data.csv │ ├── invalid_tabular_resource.json │ ├── multi_data_datapackage.json │ ├── simple_invalid_datapackage.json │ ├── simple_tabular_data.csv │ ├── simple_valid_datapackage.json │ ├── simple_valid_datapackage_mock_http_data.json │ ├── tabular_resource_invalid_data.json │ ├── test-custom-profile.schema.json │ ├── test-custom-resource-profile.schema.json │ └── valid_emails_tabular_data.csv └── update_registry.php /.coveralls.yml: -------------------------------------------------------------------------------- 1 | coverage_clover: coverage-clover.xml 2 | json_path: coveralls.json 3 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | Please replace this line with full information about your idea or problem. If it's a bug share as much as possible to reproduce it 4 | 5 | --- 6 | 7 | Please preserve this line to notify @DiegoPino (lead of this repository) 8 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | Please replace this line with full information about your pull request. Make sure that tests pass before publishing it 4 | 5 | --- 6 | 7 | Please preserve this line to notify @DiegoPino (lead of this repository) 8 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 90 3 | 4 | # Number of days of inactivity before a stale issue is closed 5 | daysUntilClose: 30 6 | 7 | # Issues with these labels will never be considered stale 8 | exemptLabels: 9 | - feature 10 | - enhancement 11 | - bug 12 | 13 | # Label to use when marking an issue as stale 14 | staleLabel: wontfix 15 | 16 | # Comment to post when marking an issue as stale. Set to `false` to disable 17 | markComment: > 18 | This issue has been automatically marked as stale because it has not had 19 | recent activity. It will be closed if no further activity occurs. Thank you 20 | for your contributions. 21 | 22 | # Comment to post when closing a stale issue. Set to `false` to disable 23 | closeComment: false 24 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | jobs: 8 | build: 9 | runs-on: ${{ matrix.operating-system }} 10 | strategy: 11 | matrix: 12 | operating-system: [ubuntu-latest] 13 | php-versions: ['7.1', '7.4'] 14 | 15 | name: frictionlessdata/datapackage-php PHP ${{ matrix.php-versions }} test on ${{ matrix.operating-system }} 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | 20 | - name: Cache Composer dependencies 21 | uses: actions/cache@v2 22 | with: 23 | path: /tmp/composer-cache 24 | key: ${{ runner.os }}-${{ hashFiles('**/composer.lock') }} 25 | 26 | - uses: php-actions/composer@v6 27 | with: 28 | php_version: ${{ matrix.php-versions }} 29 | php_extensions: zip 30 | version: 2 31 | 32 | - name: Validate composer.json & composer.lock for PHP ${{ matrix.php-versions }} 33 | uses: php-actions/composer@v6 34 | with: 35 | php_version: ${{ matrix.php-versions }} 36 | php_extensions: zip 37 | version: 2 38 | command: validate --strict 39 | 40 | - name: Run Code Style Check for PHP ${{ matrix.php-versions }} 41 | uses: php-actions/composer@v6 42 | with: 43 | php_version: ${{ matrix.php-versions }} 44 | php_extensions: zip 45 | version: 2 46 | dev: yes 47 | command: style-check 48 | 49 | - name: Run Tests for PHP ${{ matrix.php-versions }} 50 | uses: php-actions/composer@v6 51 | with: 52 | php_version: ${{ matrix.php-versions }} 53 | php_extensions: zip 54 | dev: yes 55 | command: test 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*.*.*' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v1 14 | - name: Release 15 | uses: softprops/action-gh-release@v1 16 | env: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/ 2 | /.idea/ 3 | /coverage-clover.xml 4 | /composer.lock 5 | /.php_cs 6 | /.php_cs.cache 7 | /php-cs-fixer 8 | -------------------------------------------------------------------------------- /.php_cs.dist: -------------------------------------------------------------------------------- 1 | exclude("tests/fixtures") 12 | ->exclude("src/Validators/schemas") 13 | ->in(__DIR__) 14 | ; 15 | $config 16 | ->setRules([ 17 | '@PSR2' => true, 18 | '@Symfony' => true 19 | ]) 20 | ->setFinder($finder) 21 | ; 22 | return $config; 23 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | The project follows the [Open Knowledge International coding standards](https://github.com/okfn/coding-standards). 4 | 5 | All PHP Code MUST conform to [PHP-FIG](http://www.php-fig.org/psr/) accepted PSRs. 6 | 7 | Flow Framework has a nice guide regarding coding standards: 8 | * [Printable summary of most important coding guidelines on one page **(.pdf)**](http://flowframework.readthedocs.io/en/stable/_downloads/Flow_Coding_Guidelines_on_one_page.pdf) 9 | * [The full guide **(.html)**](http://flowframework.readthedocs.io/en/stable/TheDefinitiveGuide/PartV/CodingGuideLines/PHP.html) 10 | 11 | 12 | ## Getting Started 13 | 14 | 1. Clone the repo 15 | 2. Run the tests 16 | ``` 17 | $ composer install 18 | $ composer test 19 | ``` 20 | 21 | 22 | ## Phpunit - for unit tests 23 | 24 | [![Travis](https://travis-ci.org/frictionlessdata/datapackage-php.svg?branch=master)](https://travis-ci.org/frictionlessdata/datapackage-php) 25 | 26 | Phpunit is used for unit tests, you can find the tests under tests directory 27 | 28 | Running Phpunit directly: `vendor/bin/phpunit --bootstrap tests/autoload.php tests/` 29 | 30 | 31 | ## Coveralls - for coverage 32 | 33 | [![Coveralls](http://img.shields.io/coveralls/frictionlessdata/datapackage-php.svg?branch=master)](https://coveralls.io/r/frictionlessdata/datapackage-php?branch=master) 34 | 35 | when running `composer test` phpunit generates coverage report in coverage-clover.xml - this is then sent to Coveralls via Travis. 36 | 37 | 38 | ## Scrutinizer-ci - for code analysis 39 | 40 | [![Scrutinizer-ci](https://scrutinizer-ci.com/g/OriHoch/datapackage-php/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/OriHoch/datapackage-php/) 41 | 42 | Scrutinizer-ci integrates with GitHub and runs on commits. 43 | 44 | It does static code analysis and ensure confirmation to the coding stnadards. 45 | 46 | At the moment, the integration with frictionlessdata repo is not working, you can setup a Scrutinizer-ci account for your fork and run against that. 47 | 48 | ## php-cs-fixer - code style check & autofix 49 | 50 | [php-cs-fixer](https://github.com/FriendsOfPHP/PHP-CS-Fixer) can be used to check and fix code style 51 | 52 | you need to manually install it, then you can run : `composer style-check` or `composer style-fix` 53 | 54 | ## Publishing a release and updating Packagist 55 | 56 | [![Packagist](https://img.shields.io/packagist/dm/frictionlessdata/datapackage.svg)](https://packagist.org/packages/frictionlessdata/datapackage) 57 | [![SemVer](https://img.shields.io/badge/versions-SemVer-brightgreen.svg)](http://semver.org/) 58 | 59 | * Publish a release (using SemVer) on GitHub 60 | * go to https://packagist.org/packages/frictionlessdata/datapackage 61 | * Login with GitHub which has permissions 62 | * click "Update" 63 | * all releases from GitHub appear as releases on Packagist 64 | 65 | ## updating frictionlessdata schemas 66 | 67 | The json schemas for the frictionlessdata specs are stored locally as part of the package. 68 | 69 | They might change from time to time, to donwnload the latest specs run `composer update_registry` 70 | 71 | ## Testing with a local copy of tableschema-php 72 | 73 | Add the following to `composer.json` 74 | 75 | ``` 76 | "repositories": [ 77 | { 78 | "type": "path", 79 | "url": "../tableschema-php" 80 | } 81 | ] 82 | ``` 83 | 84 | Run `composer require "frictionlessdata/tableschema @dev"` 85 | 86 | -------------------------------------------------------------------------------- /LEAD.md: -------------------------------------------------------------------------------- 1 | DiegoPino 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Open Knowledge Foundation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all list release templates version 2 | 3 | 4 | VERSION := '0.1.9' 5 | LEAD := $(shell head -n 1 LEAD.md) 6 | 7 | 8 | all: list 9 | 10 | list: 11 | @grep '^\.PHONY' Makefile | cut -d' ' -f2- | tr ' ' '\n' 12 | 13 | release: 14 | git checkout master && git pull origin && git fetch -p && git diff 15 | @echo "\nContinuing in 10 seconds. Press to abort\n" && sleep 10 16 | @git log --pretty=format:"%C(yellow)%h%Creset %s%Cgreen%d" --reverse -20 17 | @echo "\nReleasing v$(VERSION) in 10 seconds. Press to abort\n" && sleep 10 18 | git commit -a -m 'v$(VERSION)' && git tag -a v$(VERSION) -m 'v$(VERSION)' 19 | git push --follow-tags 20 | 21 | templates: 22 | sed -i -E "s/@(\w*)/@$(LEAD)/" .github/issue_template.md 23 | sed -i -E "s/@(\w*)/@$(LEAD)/" .github/pull_request_template.md 24 | 25 | version: 26 | @echo $(VERSION) 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Data Package 2 | 3 | [![Build](https://github.com/frictionlessdata/datapackage-php/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/frictionlessdata/datapackage-php/actions/workflows/ci.yml) 4 | [![Coveralls](http://img.shields.io/coveralls/frictionlessdata/datapackage-php.svg?branch=master)](https://coveralls.io/r/frictionlessdata/datapackage-php?branch=master) 5 | [![Scrutinizer-ci](https://scrutinizer-ci.com/g/frictionlessdata/datapackage-php/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/frictionlessdata/datapackage-php/) 6 | [![Packagist](https://img.shields.io/packagist/dm/frictionlessdata/datapackage.svg)](https://packagist.org/packages/frictionlessdata/datapackage) 7 | [![SemVer](https://img.shields.io/badge/versions-SemVer-brightgreen.svg)](http://semver.org/) 8 | [![Codebase](https://img.shields.io/badge/codebase-github-brightgreen)](https://github.com/frictionlessdata/datapackage-php) 9 | [![Support](https://img.shields.io/badge/support-discord-brightgreen)](https://discordapp.com/invite/Sewv6av) 10 | 11 | A utility library for working with [Data Package](https://specs.frictionlessdata.io/data-package/) in PHP. 12 | 13 | ## Features summary and Usage guide 14 | 15 | ### Installation 16 | 17 | ```bash 18 | composer require frictionlessdata/datapackage 19 | ``` 20 | 21 | Optionally, to create zip files you will need the PHP zip extension. On Ubuntu it can be enabled with `sudo apt-get install php-zip` 22 | 23 | ### Package 24 | 25 | Load a data package conforming to the specs 26 | 27 | ```php 28 | use frictionlessdata\datapackage\Package; 29 | $package = Package::load("tests/fixtures/multi_data_datapackage.json"); 30 | ``` 31 | 32 | Iterate over the resources and the data 33 | 34 | ```php 35 | foreach ($package as $resource) { 36 | echo $resource->name(); 37 | foreach ($resource as $row) { 38 | echo $row; 39 | } 40 | } 41 | ``` 42 | 43 | Get all the data as an array (loads all the data into memory, not recommended for large data sets) 44 | 45 | ```php 46 | foreach ($package as $resource) { 47 | var_dump($resource->read()); 48 | } 49 | ``` 50 | 51 | All data and schemas are validated and throws exceptions in case of any problems. 52 | 53 | Validate the data explicitly and get a list of errors 54 | 55 | ```php 56 | Package::validate("tests/fixtures/simple_invalid_datapackage.json"); // array of validation errors 57 | ``` 58 | 59 | Load a zip file 60 | 61 | ```php 62 | $package = Package::load('http://datahub.io/opendatafortaxjustice/eucountrydatawb/r/datapackage_zip.zip'); 63 | ``` 64 | 65 | Provide read options which are passed through to [tableschema-php](https://github.com/frictionlessdata/tableschema-php) Table::read method 66 | 67 | ```php 68 | $package = Package::load('http://datahub.io/opendatafortaxjustice/eucountrydatawb/r/datapackage_zip.zip'); 69 | foreach ($package as $resource) { 70 | $resource->read(["cast" => false]); 71 | } 72 | ``` 73 | 74 | The package object has some useful methods to access and manipulate the resources 75 | 76 | ```php 77 | $package = Package::load("tests/fixtures/multi_data_datapackage.json"); 78 | $package->resources(); // array of resource name => Resource object (see below for Resource class reference) 79 | $package->getResource("first-resource"); // Resource object matching the given name 80 | $package->removeResource("first-resource"); 81 | // add a tabular resource 82 | $package->addResource("tabular-resource-name", [ 83 | "profile" => "tabular-data-resource", 84 | "schema" => [ 85 | "fields" => [ 86 | ["name" => "id", "type" => "integer"], 87 | ["name" => "name", "type" => "string"] 88 | ] 89 | ], 90 | "path" => [ 91 | "tests/fixtures/simple_tabular_data.csv", 92 | ] 93 | ]); 94 | ``` 95 | 96 | Create a new package from scratch 97 | 98 | ```php 99 | $package = Package::create([ 100 | "name" => "datapackage-name", 101 | "profile" => "tabular-data-package" 102 | ]); 103 | // add a resource 104 | $package->addResource("resource-name", [ 105 | "profile" => "tabular-data-resource", 106 | "schema" => [ 107 | "fields" => [ 108 | ["name" => "id", "type" => "integer"], 109 | ["name" => "name", "type" => "string"] 110 | ] 111 | ], 112 | "path" => "tests/fixtures/simple_tabular_data.csv" 113 | ]); 114 | // save the package descriptor to a file 115 | $package->saveDescriptor("datapackage.json"); 116 | ``` 117 | 118 | Save the entire datapackage including any local data to a zip file 119 | 120 | ```php 121 | $package->save("datapackage.zip"); 122 | ``` 123 | 124 | ### Resource 125 | 126 | Resource objects can be accessed from a Package as described above 127 | 128 | ```php 129 | $resource = $package->getResource("resource-name") 130 | ``` 131 | 132 | or instantiated directly 133 | 134 | ```php 135 | use frictionlessdata\datapackage\Resource; 136 | $resource = Resource::create([ 137 | "name" => "my-resource", 138 | "profile" => "tabular-data-resource", 139 | "path" => "tests/fixtures/simple_tabular_data.csv", 140 | "schema" => ["fields" => [["name" => "id", "type" => "integer"], ["name" => "name", "type" => "string"]]] 141 | ]); 142 | ``` 143 | 144 | Iterating or reading over the resource produces combined rows from all the path or data elements 145 | 146 | ```php 147 | foreach ($resource as $row) {}; // iterating 148 | $resource->read(); // get all the data as an array 149 | ``` 150 | 151 | 152 | ## Contributing 153 | 154 | Please read the contribution guidelines: [How to Contribute](CONTRIBUTING.md) 155 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frictionlessdata/datapackage", 3 | "description": "A utility library for working with Data Packages", 4 | "license": "MIT", 5 | "require": { 6 | "php": ">=7.1", 7 | "ext-zip": "*", 8 | "justinrainbow/json-schema": "^5.2", 9 | "frictionlessdata/tableschema": "^v1.0.0" 10 | }, 11 | "require-dev": { 12 | "phpunit/phpunit": "^7.5.20", 13 | "satooshi/php-coveralls": "^1.0", 14 | "psy/psysh": "@stable", 15 | "squizlabs/php_codesniffer": "^3.5" 16 | }, 17 | "autoload": { 18 | "psr-4": { 19 | "frictionlessdata\\datapackage\\": "src/" 20 | } 21 | }, 22 | "scripts": { 23 | "test": "phpunit --coverage-clover coverage-clover.xml --bootstrap tests/autoload.php tests/", 24 | "update_registry": "php update_registry.php", 25 | "style-check": "php vendor/bin/phpcs --standard=psr2 src/ -n", 26 | "style-fix": "php vendor/bin/phpcbf --standard=psr2 src/ -n" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/DataStreams/BaseDataStream.php: -------------------------------------------------------------------------------- 1 | dataSource = $dataSource; 23 | $this->dataSourceOptions = $dataSourceOptions; 24 | } 25 | 26 | abstract public function save($filename); 27 | 28 | /** 29 | * @return mixed 30 | * 31 | * @throws \frictionlessdata\datapackage\Exceptions\DataStreamValidationException 32 | */ 33 | abstract public function current(); 34 | } 35 | -------------------------------------------------------------------------------- /src/DataStreams/DefaultDataStream.php: -------------------------------------------------------------------------------- 1 | fopenResource = fopen($this->dataSource, 'r'); 26 | } catch (\Exception $e) { 27 | throw new DataStreamOpenException('Failed to open data source '.json_encode($this->dataSource).': '.json_encode($e->getMessage())); 28 | } 29 | } 30 | 31 | public function __destruct() 32 | { 33 | fclose($this->fopenResource); 34 | } 35 | 36 | public function rewind() 37 | { 38 | if ($this->currentLineNumber == 0) { 39 | // starting iterations 40 | $this->currentLineNumber = 1; 41 | } else { 42 | throw new \Exception('DataStream does not support rewinding a stream, sorry'); 43 | } 44 | } 45 | 46 | public function save($filename) 47 | { 48 | $target = fopen($filename, 'w'); 49 | stream_copy_to_stream($this->fopenResource, $target); 50 | fclose($target); 51 | } 52 | 53 | public function current() 54 | { 55 | return fgets($this->fopenResource); 56 | } 57 | 58 | public function key() 59 | { 60 | return $this->currentLineNumber; 61 | } 62 | 63 | public function next() 64 | { 65 | ++$this->currentLineNumber; 66 | } 67 | 68 | public function valid() 69 | { 70 | return !feof($this->fopenResource); 71 | } 72 | 73 | protected $currentLineNumber = 0; 74 | } 75 | -------------------------------------------------------------------------------- /src/DataStreams/TabularDataStream.php: -------------------------------------------------------------------------------- 1 | dataSourceOptions = (object) $this->dataSourceOptions; 22 | $schema = $this->dataSourceOptions->schema; 23 | $dialect = $this->dataSourceOptions->dialect; 24 | if (empty($schema)) { 25 | throw new \Exception('schema is required for tabular data stream'); 26 | } else { 27 | try { 28 | $this->schema = new Schema($schema); 29 | $this->table = new Table($this->getDataSourceObject(), $this->schema, $dialect); 30 | } catch (\Exception $e) { 31 | throw new DataStreamOpenException('Failed to open tabular data source '.json_encode($dataSource).': '.json_encode($e->getMessage())); 32 | } 33 | } 34 | } 35 | 36 | protected function getDataSourceObject() 37 | { 38 | return new CsvDataSource($this->dataSource); 39 | } 40 | 41 | public function rewind() 42 | { 43 | $this->table->rewind(); 44 | } 45 | 46 | public function save($filename) 47 | { 48 | $this->table->save($filename); 49 | } 50 | 51 | /** 52 | * @return array 53 | * 54 | * @throws DataStreamValidationException 55 | */ 56 | public function current() 57 | { 58 | try { 59 | return $this->table->current(); 60 | } catch (DataSourceException $e) { 61 | throw new DataStreamValidationException($e->getMessage()); 62 | } catch (FieldValidationException $e) { 63 | throw new DataStreamValidationException($e->getMessage()); 64 | } 65 | } 66 | 67 | public function key() 68 | { 69 | return $this->table->key(); 70 | } 71 | 72 | public function next() 73 | { 74 | $this->table->next(); 75 | } 76 | 77 | public function valid() 78 | { 79 | return $this->table->valid(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/DataStreams/TabularInlineDataStream.php: -------------------------------------------------------------------------------- 1 | dataSource), true); 13 | if (is_array($data)) { 14 | $numFields = count($this->schema->fields()); 15 | $objRows = []; 16 | if (array_sum(array_keys($data[0])) == array_sum(range(0, $numFields - 1))) { 17 | // Row Arrays - convert to Row Objects 18 | $header = array_shift($data); 19 | foreach ($data as $row) { 20 | $objRow = []; 21 | foreach ($header as $fieldOrder => $fieldName) { 22 | $objRow[$fieldName] = $row[$fieldOrder]; 23 | } 24 | $objRows[] = $objRow; 25 | } 26 | } else { 27 | // Row Objects - no processing needed 28 | $objRows = $data; 29 | } 30 | 31 | return new NativeDataSource($objRows); 32 | } else { 33 | throw new DataStreamOpenException('inline tabular data must be an array'); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/Datapackages/BaseDatapackage.php: -------------------------------------------------------------------------------- 1 | descriptor = $descriptor; 30 | $this->basePath = $basePath; 31 | $this->skipValidations = $skipValidations; 32 | if (!$this->skipValidations) { 33 | $this->revalidate(); 34 | } 35 | } 36 | 37 | public static function create($name, $resources, $basePath = null) 38 | { 39 | $datapackage = new static((object) [ 40 | 'name' => $name, 41 | 'resources' => [], 42 | ], $basePath, true); 43 | foreach ($resources as $resource) { 44 | $datapackage->addResource($resource); 45 | } 46 | 47 | return $datapackage; 48 | } 49 | 50 | public function revalidate() 51 | { 52 | $this->rewind(); 53 | $validationErrors = $this->datapackageValidate(); 54 | if (count($validationErrors) > 0) { 55 | throw new DatapackageValidationFailedException($validationErrors); 56 | } 57 | } 58 | 59 | public static function handlesDescriptor($descriptor) 60 | { 61 | return static::handlesProfile(Registry::getDatapackageValidationProfile($descriptor)); 62 | } 63 | 64 | /** 65 | * returns the descriptor as-is, without adding default values or normalizing. 66 | * 67 | * @return object 68 | */ 69 | public function descriptor() 70 | { 71 | return $this->descriptor; 72 | } 73 | 74 | public function resources() 75 | { 76 | $resources = []; 77 | foreach ($this->descriptor->resources as $resourceDescriptor) { 78 | $resources[$resourceDescriptor->name] = $this->initResource($resourceDescriptor); 79 | } 80 | 81 | return $resources; 82 | } 83 | 84 | public function getResource($name) 85 | { 86 | foreach ($this->descriptor->resources as $resourceDescriptor) { 87 | if ($resourceDescriptor->name == $name) { 88 | return $this->initResource($resourceDescriptor); 89 | } 90 | } 91 | throw new \Exception("couldn't find matching resource with name = '{$name}'"); 92 | } 93 | 94 | public function addResource($name, $resource) 95 | { 96 | if (is_a($resource, 'frictionlessdata\\datapackage\\Resources\\BaseResource')) { 97 | $resource = $resource->descriptor(); 98 | } else { 99 | $resource = Utils::objectify($resource); 100 | } 101 | $resource->name = $name; 102 | $resourceDescriptors = []; 103 | $gotMatch = false; 104 | foreach ($this->descriptor->resources as $resourceDescriptor) { 105 | if ($resourceDescriptor->name == $resource->name) { 106 | $resourceDescriptors[] = $resource; 107 | $gotMatch = true; 108 | } else { 109 | $resourceDescriptors[] = $resourceDescriptor; 110 | } 111 | } 112 | if (!$gotMatch) { 113 | $resourceDescriptors[] = $resource; 114 | } 115 | $this->descriptor->resources = $resourceDescriptors; 116 | if (!$this->skipValidations) { 117 | $this->revalidate(); 118 | } 119 | } 120 | 121 | // TODO: remove this function and use the getResource / addResource directly (will need to modify a lot of tests code) 122 | public function resource($name, $resource = null) 123 | { 124 | if ($resource) { 125 | $this->addResource($name, $resource); 126 | } else { 127 | return $this->getResource($name); 128 | } 129 | } 130 | 131 | public function removeResource($name) 132 | { 133 | $resourceDescriptors = []; 134 | foreach ($this->descriptor->resources as $resourceDescriptor) { 135 | if ($resourceDescriptor->name != $name) { 136 | $resourceDescriptors[] = $resourceDescriptor; 137 | } 138 | } 139 | $this->descriptor->resources = $resourceDescriptors; 140 | if (!$this->skipValidations) { 141 | $this->revalidate(); 142 | } 143 | } 144 | 145 | public function saveDescriptor($filename) 146 | { 147 | return file_put_contents($filename, json_encode($this->descriptor())); 148 | } 149 | 150 | // standard iterator functions - to iterate over the resources 151 | public function rewind() 152 | { 153 | $this->currentResourcePosition = 0; 154 | } 155 | 156 | public function current() 157 | { 158 | return $this->initResource($this->descriptor()->resources[$this->currentResourcePosition]); 159 | } 160 | 161 | public function key() 162 | { 163 | return $this->currentResourcePosition; 164 | } 165 | 166 | public function next() 167 | { 168 | ++$this->currentResourcePosition; 169 | } 170 | 171 | public function valid() 172 | { 173 | return isset($this->descriptor()->resources[$this->currentResourcePosition]); 174 | } 175 | 176 | /** 177 | * @param $zip_filename 178 | * 179 | * @throws \frictionlessdata\datapackage\Exceptions\DatapackageInvalidSourceException 180 | * @throws \Exception 181 | */ 182 | public function save($zip_filename) 183 | { 184 | Package::isZipPresent(); 185 | $zip = new ZipArchive(); 186 | 187 | $packageCopy = $this->copy(); 188 | 189 | $base = tempnam(sys_get_temp_dir(), 'datapackage-zip-'); 190 | $files = [ 191 | 'datapackage.json' => $base.'datapackage.json', 192 | ]; 193 | $ri = 0; 194 | foreach ($packageCopy as $resource) { 195 | if ($resource->isRemote()) { 196 | continue; 197 | } 198 | 199 | $resourceFiles = []; 200 | $fileNames = $resource->save($base.'resource-'.$ri); 201 | foreach ($fileNames as $fileName) { 202 | $relname = str_replace($base.'resource-'.$ri, '', $fileName); 203 | $files['resource-'.$ri.$relname] = $fileName; 204 | $resourceFiles[] = 'resource-'.$ri.$relname; 205 | } 206 | $resource->descriptor()->path = count($resourceFiles) == 1 ? $resourceFiles[0] : $resourceFiles; 207 | ++$ri; 208 | } 209 | $packageCopy->saveDescriptor($files['datapackage.json']); 210 | 211 | register_shutdown_function(function () use ($base) { 212 | Utils::removeDir($base); 213 | }); 214 | if ($zip->open($zip_filename, ZipArchive::CREATE | ZipArchive::OVERWRITE) === true) { 215 | foreach ($files as $filename => $resource) { 216 | $zip->addFile($resource, $filename); 217 | } 218 | $zip->close(); 219 | } else { 220 | throw new DatapackageInvalidSourceException('zip file could not be saved.'); 221 | } 222 | } 223 | 224 | /** 225 | * Make a new Datapackage object that is a copy of the current Datapackage. Bypasses validation. 226 | * @return $this 227 | * @throws DatapackageValidationFailedException 228 | */ 229 | protected function copy() 230 | { 231 | return new static($this->descriptor, $this->basePath, true); 232 | } 233 | 234 | protected $descriptor; 235 | protected $currentResourcePosition = 0; 236 | protected $basePath; 237 | protected $skipValidations = false; 238 | 239 | /** 240 | * called by the resources iterator for each iteration. 241 | * 242 | * @param object $descriptor 243 | * 244 | * @return \frictionlessdata\datapackage\Resources\BaseResource 245 | * @throws \frictionlessdata\datapackage\Exceptions\ResourceValidationFailedException 246 | */ 247 | protected function initResource($descriptor) 248 | { 249 | return Factory::resource($descriptor, $this->basePath, $this->skipValidations); 250 | } 251 | 252 | protected function datapackageValidate() 253 | { 254 | return DatapackageValidator::validate($this->descriptor(), $this->basePath); 255 | } 256 | 257 | protected static function handlesProfile($profile) 258 | { 259 | return false; 260 | } 261 | } 262 | -------------------------------------------------------------------------------- /src/Datapackages/CustomDatapackage.php: -------------------------------------------------------------------------------- 1 | validationErrors = $validationErrors; 14 | parent::__construct('Datapackage validation failed: '.DatapackageValidationError::getErrorMessages($validationErrors)); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Exceptions/ResourceValidationFailedException.php: -------------------------------------------------------------------------------- 1 | validationErrors = $validationErrors; 14 | parent::__construct('resource validation failed: '.ResourceValidationError::getErrorMessages($validationErrors)); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/Factory.php: -------------------------------------------------------------------------------- 1 | descriptor; 40 | $basePath = $source->basePath; 41 | $datapackageClass = static::getDatapackageClass($descriptor); 42 | $datapackage = new $datapackageClass($descriptor, $basePath); 43 | 44 | return $datapackage; 45 | } 46 | 47 | /** 48 | * create a resource object. 49 | * 50 | * @param object $descriptor 51 | * @param null|string $basePath 52 | * @param bool $skipValidations 53 | * 54 | * @return Resources\BaseResource 55 | * 56 | */ 57 | public static function resource($descriptor, $basePath = null, $skipValidations = false) 58 | { 59 | $resourceClass = static::getResourceClass($descriptor); 60 | $resource = new $resourceClass($descriptor, $basePath, $skipValidations); 61 | 62 | return $resource; 63 | } 64 | 65 | /** 66 | * validates a given datapackage descriptor 67 | * will load all resources, and sample 10 lines of data from each data source. 68 | * 69 | * @param mixed $source datapackage source - same as in datapackage function 70 | * @param null|string $basePath same as in datapackage function 71 | * 72 | * @return Validators\DatapackageValidationError[] 73 | */ 74 | public static function validate($source, $basePath = null) 75 | { 76 | $curResource = 1; 77 | $curLine = null; 78 | try { 79 | $datapackage = static::datapackage($source, $basePath); 80 | foreach ($datapackage as $resource) { 81 | $curLine = 1; 82 | foreach ($resource as $line) { 83 | if ($curLine == self::VALIDATE_PEEK_LINES) { 84 | break; 85 | } 86 | ++$curLine; 87 | } 88 | ++$curResource; 89 | } 90 | // no validation errors 91 | return []; 92 | } catch (Exceptions\DatapackageInvalidSourceException $e) { 93 | // failed to load the datapackage descriptor 94 | // return a list containing a single LOAD_FAILED validation error 95 | return [ 96 | new Validators\DatapackageValidationError( 97 | Validators\DatapackageValidationError::LOAD_FAILED, 98 | $e->getMessage() 99 | ), 100 | ]; 101 | } catch (Exceptions\DatapackageValidationFailedException $e) { 102 | // datapackage descriptor failed validation - return the validation errors 103 | return $e->validationErrors; 104 | } catch (Exceptions\ResourceValidationFailedException $e) { 105 | // resource descriptor failed validation - return the validation errors 106 | return [ 107 | new Validators\DatapackageValidationError( 108 | Validators\DatapackageValidationError::RESOURCE_FAILED_VALIDATION, 109 | [ 110 | 'resource' => $curResource, 111 | 'validationErrors' => $e->validationErrors, 112 | ] 113 | ), 114 | ]; 115 | } catch (Exceptions\DataStreamOpenException $e) { 116 | // failed to open data stream 117 | return [ 118 | new Validators\DatapackageValidationError( 119 | Validators\DatapackageValidationError::DATA_STREAM_FAILURE, 120 | [ 121 | 'resource' => $curResource, 122 | 'line' => 0, 123 | 'error' => $e->getMessage(), 124 | ] 125 | ), 126 | ]; 127 | } catch (Exceptions\DataStreamValidationException $e) { 128 | // failed to validate the data stream 129 | return [ 130 | new Validators\DatapackageValidationError( 131 | Validators\DatapackageValidationError::DATA_STREAM_FAILURE, 132 | [ 133 | 'resource' => $curResource, 134 | 'line' => $curLine, 135 | 'error' => $e->getMessage(), 136 | ] 137 | ), 138 | ]; 139 | } 140 | } 141 | 142 | public static function registerDatapackageClass($datapackageClass) 143 | { 144 | static::$registeredDatapackageClasses[] = $datapackageClass; 145 | } 146 | 147 | public static function clearRegisteredDatapackageClasses() 148 | { 149 | static::$registeredDatapackageClasses = []; 150 | } 151 | 152 | /** 153 | * @param $descriptor 154 | * 155 | * @return BaseDatapackage::class 156 | */ 157 | public static function getDatapackageClass($descriptor) 158 | { 159 | $datapackageClasses = array_merge( 160 | // custom classes 161 | static::$registeredDatapackageClasses, 162 | // core classes 163 | [ 164 | "frictionlessdata\\datapackage\\Datapackages\TabularDatapackage", 165 | "frictionlessdata\\datapackage\\Datapackages\DefaultDatapackage", 166 | ] 167 | ); 168 | $res = null; 169 | foreach ($datapackageClasses as $datapackageClass) { 170 | if (call_user_func([$datapackageClass, 'handlesDescriptor'], $descriptor)) { 171 | $res = $datapackageClass; 172 | break; 173 | } 174 | } 175 | if (!$res) { 176 | // not matched by any known classes 177 | $res = "frictionlessdata\\datapackage\\Datapackages\CustomDatapackage"; 178 | } 179 | 180 | return $res; 181 | } 182 | 183 | public static function registerResourceClass($resourceClass) 184 | { 185 | static::$registeredResourceClasses[] = $resourceClass; 186 | } 187 | 188 | public static function clearRegisteredResourceClasses() 189 | { 190 | static::$registeredResourceClasses = []; 191 | } 192 | 193 | /** 194 | * @param $descriptor 195 | * 196 | * @return BaseResource::class 197 | */ 198 | public static function getResourceClass($descriptor) 199 | { 200 | $descriptor = Utils::objectify($descriptor); 201 | $resourceClasses = array_merge( 202 | // custom classes 203 | static::$registeredResourceClasses, 204 | // core classes 205 | [ 206 | 'frictionlessdata\\datapackage\\Resources\\TabularResource', 207 | 'frictionlessdata\\datapackage\\Resources\\DefaultResource', 208 | ] 209 | ); 210 | $res = null; 211 | foreach ($resourceClasses as $resourceClass) { 212 | if (call_user_func([$resourceClass, 'handlesDescriptor'], $descriptor)) { 213 | $res = $resourceClass; 214 | break; 215 | } 216 | } 217 | if (!$res) { 218 | // not matched by any known classes 219 | $res = 'frictionlessdata\\datapackage\\Resources\\CustomResource'; 220 | } 221 | 222 | return $res; 223 | } 224 | 225 | protected static $registeredDatapackageClasses = []; 226 | protected static $registeredResourceClasses = []; 227 | 228 | /** 229 | * allows extending classes to add custom sources 230 | * used by unit tests to add a mock http source. 231 | * 232 | * @param $source 233 | * 234 | * @return mixed 235 | */ 236 | protected static function normalizeHttpSource($source) 237 | { 238 | return $source; 239 | } 240 | 241 | /** 242 | * allows extending classes to add custom sources 243 | * used by unit tests to add a mock http source. 244 | * @param $source 245 | * @return bool 246 | */ 247 | protected static function isHttpSource($source) 248 | { 249 | return Utils::isHttpSource($source); 250 | } 251 | 252 | /** 253 | * loads the datapackage descriptor from different sources 254 | * returns an object containing: 255 | * - the datapackage descriptor as native php object 256 | * - normalized basePath. 257 | * 258 | * @param $source 259 | * @param $basePath 260 | * 261 | * @return object 262 | * 263 | * @throws Exceptions\DatapackageInvalidSourceException 264 | * @throws \Exception 265 | */ 266 | protected static function loadSource($source, $basePath) 267 | { 268 | if (is_object($source)) { 269 | $descriptor = $source; 270 | } elseif (is_string($source)) { 271 | if (Utils::isJsonString($source)) { 272 | try { 273 | $descriptor = json_decode($source); 274 | } catch (\Exception $e) { 275 | throw new Exceptions\DatapackageInvalidSourceException( 276 | 'Failed to load source: '.json_encode($source).': '.$e->getMessage() 277 | ); 278 | } 279 | } elseif (static::isHttpSource($source)) { 280 | if (static::isHttpZipSource($source)) { 281 | return static::loadHttpZipSource($source); 282 | } else { 283 | try { 284 | $descriptor = json_decode(file_get_contents(static::normalizeHttpSource($source))); 285 | } catch (\Exception $e) { 286 | throw new Exceptions\DatapackageInvalidSourceException( 287 | 'Failed to load source: '.json_encode($source).': '.$e->getMessage() 288 | ); 289 | } 290 | // http sources don't allow relative paths, hence basePath should remain null 291 | $basePath = null; 292 | } 293 | } else { 294 | // not a json string and not a url - assume it's a file path 295 | if (static::isFileZipSource($source)) { 296 | return static::loadFileZipSource($source); 297 | } else { 298 | if (empty($basePath)) { 299 | // no basePath 300 | // - assume source is the absolute path of the file 301 | // - set it's directory as the basePath 302 | $basePath = dirname($source); 303 | } else { 304 | // got a basePath 305 | // - try to prepend it to the source and see if such a file exists 306 | // - if not - assume it's an absolute path 307 | $absPath = $basePath.DIRECTORY_SEPARATOR.$source; 308 | if (file_exists($absPath)) { 309 | $source = $absPath; 310 | } 311 | } 312 | try { 313 | $descriptor = json_decode(file_get_contents($source)); 314 | } catch (\Exception $e) { 315 | throw new Exceptions\DatapackageInvalidSourceException( 316 | 'Failed to load source: '.json_encode($source).': '.$e->getMessage() 317 | ); 318 | } 319 | } 320 | } 321 | if (json_last_error()) { 322 | throw new Exceptions\DatapackageInvalidSourceException( 323 | json_last_error_msg().' when loading source: '.json_encode($source) 324 | ); 325 | } 326 | } else { 327 | throw new Exceptions\DatapackageInvalidSourceException( 328 | 'Invalid source: '.json_encode($source) 329 | ); 330 | } 331 | 332 | return (object) ['descriptor' => $descriptor, 'basePath' => $basePath]; 333 | } 334 | 335 | protected static function isHttpZipSource($source) 336 | { 337 | return strtolower(substr($source, -4)) == '.zip'; 338 | } 339 | 340 | protected static function isFileZipSource($source) 341 | { 342 | return strtolower(substr($source, -4)) == '.zip'; 343 | } 344 | 345 | /** 346 | * @param $source 347 | * 348 | * @return object 349 | * @throws \frictionlessdata\datapackage\Exceptions\DatapackageInvalidSourceException 350 | * @throws \Exception 351 | */ 352 | protected static function loadHttpZipSource($source) 353 | { 354 | Package::isZipPresent(); 355 | $tempfile = tempnam(sys_get_temp_dir(), 'datapackage-php'); 356 | unlink($tempfile); 357 | $tempfile .= '.zip'; 358 | stream_copy_to_stream(fopen($source, 'r'), fopen($tempfile, 'w')); 359 | register_shutdown_function(function () use ($tempfile) { 360 | unlink($tempfile); 361 | }); 362 | 363 | return self::loadFileZipSource($tempfile); 364 | } 365 | 366 | /** 367 | * @param $source 368 | * 369 | * @return object 370 | * @throws \frictionlessdata\datapackage\Exceptions\DatapackageInvalidSourceException 371 | * @throws \Exception 372 | */ 373 | protected static function loadFileZipSource($source) 374 | { 375 | Package::isZipPresent(); 376 | $zip = new ZipArchive(); 377 | $tempdir = tempnam(sys_get_temp_dir(), 'datapackage-php'); 378 | unlink($tempdir); 379 | mkdir($tempdir); 380 | register_shutdown_function(function () use ($tempdir) { 381 | Utils::removeDir($tempdir); 382 | }); 383 | /* @noinspection PhpUnhandledExceptionInspection File existence is checked afterwards anyway */ 384 | if (($zip->open($source) === true) && ($zip->extractTo($tempdir) === true)) { 385 | $zip->close(); 386 | } else { 387 | throw new Exceptions\DatapackageInvalidSourceException('zip file could not be opened from source.'); 388 | } 389 | 390 | if (!file_exists($tempdir.'/datapackage.json')) { 391 | throw new Exceptions\DatapackageInvalidSourceException('zip file must contain a datapackage.json file'); 392 | } 393 | return static::loadSource('datapackage.json', $tempdir); 394 | } 395 | } 396 | -------------------------------------------------------------------------------- /src/Package.php: -------------------------------------------------------------------------------- 1 | resources)) { 47 | $descriptor->resources = []; 48 | } 49 | $packageClass = Factory::getDatapackageClass($descriptor); 50 | 51 | return new $packageClass($descriptor, $basePath, true); 52 | } 53 | 54 | /** 55 | * @throws \Exception 56 | */ 57 | public static function isZipPresent() 58 | { 59 | //If ZipArchive is not available throw Exception. 60 | if (!class_exists('ZipArchive')) { 61 | throw new Exception('Error: Your PHP version is not compiled with zip support'); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Registry.php: -------------------------------------------------------------------------------- 1 | profile) && $descriptor->profile != 'default') { 21 | return $descriptor->profile; 22 | } else { 23 | return 'data-resource'; 24 | } 25 | } 26 | 27 | /** 28 | * Get the profile which should be used for validation from the given datapackage descriptor 29 | * 30 | * Corresponds to the id from the registry. 31 | * 32 | * @param $descriptor 33 | * 34 | * @return string 35 | */ 36 | public static function getDatapackageValidationProfile($descriptor) 37 | { 38 | if (isset($descriptor->profile) && $descriptor->profile != 'default') { 39 | return $descriptor->profile; 40 | } else { 41 | return 'data-package'; 42 | } 43 | } 44 | 45 | /** 46 | * Given a normalized profile - get the corresponding schema file for known schema in the registry. 47 | * 48 | * Returns false in case of unknown schema 49 | * works the same for both datapackage schema and resource schemas. 50 | * 51 | * @param $profile 52 | * @return false|string 53 | */ 54 | public static function getJsonSchemaFile($profile) 55 | { 56 | foreach (static::getAllSchemas() as $schema) { 57 | if ($schema->id != 'registry' && $schema->id == $profile) { 58 | if (isset($schema->schema_path)) { 59 | return realpath(dirname(__FILE__)).'/Validators/schemas/'.$schema->schema_path; 60 | } else { 61 | return $schema->schema_filename; 62 | } 63 | } 64 | } 65 | 66 | return false; 67 | } 68 | 69 | public static function registerSchema($profile, $filename) 70 | { 71 | static::$registeredSchemas[$profile] = ['filename' => $filename]; 72 | } 73 | 74 | public static function clearRegisteredSchemas() 75 | { 76 | static::$registeredSchemas = []; 77 | } 78 | 79 | /** 80 | * Returns array of all known schemas in the registry. 81 | */ 82 | public static function getAllSchemas() 83 | { 84 | // registry schema 85 | $registrySchemaFilename = dirname(__FILE__).'/Validators/schemas/registry.json'; 86 | $registry = [ 87 | (object) [ 88 | 'id' => 'registry', 89 | 'schema' => 'https://specs.frictionlessdata.io/schemas/registry.json', 90 | 'schema_path' => 'registry.json', 91 | ], 92 | ]; 93 | // schemas from the registry (currently contains only the datapackage scheams) 94 | $schemaIds = []; 95 | if (file_exists($registrySchemaFilename)) { 96 | foreach (json_decode(file_get_contents($registrySchemaFilename)) as $schema) { 97 | $schemaIds[] = $schema->id; 98 | $registry[] = $schema; 99 | } 100 | // resource schemas - currently not in the registry 101 | foreach (['data-resource', 'tabular-data-resource'] as $id) { 102 | if (!in_array($id, $schemaIds)) { 103 | $registry[] = (object) [ 104 | 'id' => $id, 105 | 'schema' => "https://specs.frictionlessdata.io/schemas/{$id}.json", 106 | 'schema_path' => "{$id}.json", 107 | ]; 108 | } 109 | } 110 | } 111 | // custom registered schemas 112 | foreach (static::$registeredSchemas as $profile => $schema) { 113 | $registry[] = (object) [ 114 | 'id' => $profile, 115 | 'schema_filename' => $schema['filename'], 116 | ]; 117 | } 118 | 119 | return $registry; 120 | } 121 | 122 | protected static $registeredSchemas = []; 123 | } 124 | -------------------------------------------------------------------------------- /src/Resource.php: -------------------------------------------------------------------------------- 1 | basePath = $basePath; 28 | $this->descriptor = Utils::objectify($descriptor); 29 | $this->skipValidations = $skipValidations; 30 | if (!$this->skipValidations) { 31 | $validationErrors = $this->validateResource(); 32 | if (count($validationErrors) > 0) { 33 | throw new ResourceValidationFailedException($validationErrors); 34 | } 35 | } 36 | } 37 | 38 | public static function handlesDescriptor($descriptor) 39 | { 40 | return static::handlesProfile(Registry::getResourceValidationProfile($descriptor)); 41 | } 42 | 43 | public function read($readOptions = null) 44 | { 45 | $limit = ($readOptions && isset($readOptions['limit'])) ? $readOptions['limit'] : null; 46 | $rows = []; 47 | foreach ($this->dataStreams() as $dataStream) { 48 | if (isset($dataStream->table)) { 49 | $readOptions['limit'] = $limit; 50 | foreach ($dataStream->table->read($readOptions) as $row) { 51 | $rows[] = $row; 52 | if ($limit !== null) { 53 | --$limit; 54 | if ($limit < 0) { 55 | break; 56 | } 57 | } 58 | } 59 | } else { 60 | foreach ($dataStream as $row) { 61 | $rows[] = $row; 62 | if ($limit !== null) { 63 | --$limit; 64 | if ($limit < 0) { 65 | break; 66 | } 67 | } 68 | } 69 | } 70 | if ($limit !== null && $limit < 0) { 71 | break; 72 | } 73 | } 74 | 75 | return $rows; 76 | } 77 | 78 | /** 79 | * Loads $this->dataStreams based on $this->path() and $this->data 80 | * @return BaseDataStream[]|null 81 | */ 82 | public function dataStreams() 83 | { 84 | if (is_null($this->dataStreams)) { 85 | $this->dataStreams = []; 86 | foreach ($this->path() as $path) { 87 | $this->dataStreams[] = $this->getDataStream($path); 88 | } 89 | $data = $this->data(); 90 | if ($data) { 91 | $this->dataStreams[] = $this->getInlineDataStream($data); 92 | } 93 | } 94 | 95 | return $this->dataStreams; 96 | } 97 | 98 | /** 99 | * @return object 100 | */ 101 | public function descriptor() 102 | { 103 | return $this->descriptor; 104 | } 105 | 106 | /** 107 | * @return string 108 | */ 109 | public function name() 110 | { 111 | return $this->descriptor()->name; 112 | } 113 | 114 | public function path() 115 | { 116 | if (isset($this->descriptor()->path)) { 117 | $path = $this->descriptor()->path; 118 | if (!is_array($path)) { 119 | $path = [$path]; 120 | } 121 | 122 | return $path; 123 | } else { 124 | return []; 125 | } 126 | } 127 | 128 | /** 129 | * If the resource's $path is local (non-http) 130 | * @return bool 131 | */ 132 | public function isLocal() 133 | { 134 | return !$this->isRemote(); 135 | } 136 | 137 | /** 138 | * If the resource's $path is remote (http) 139 | * @return bool 140 | */ 141 | public function isRemote() 142 | { 143 | $path = $this->path(); 144 | return Utils::isHttpSource(is_array($path) && count($path) > 0 ? $path[0] : $path); 145 | } 146 | 147 | public function data() 148 | { 149 | return isset($this->descriptor()->data) ? $this->descriptor()->data : null; 150 | } 151 | 152 | // standard iterator functions - to iterate over the data sources 153 | public function rewind() 154 | { 155 | $this->dataStreams = null; 156 | $this->currentDataStream = 0; 157 | foreach ($this->dataStreams() as $dataStream) { 158 | $dataStream->rewind(); 159 | } 160 | } 161 | 162 | public function current() 163 | { 164 | return $this->dataStreams()[$this->currentDataStream]->current(); 165 | } 166 | 167 | public function key() 168 | { 169 | return $this->dataStreams()[$this->currentDataStream]->key(); 170 | } 171 | 172 | public function next() 173 | { 174 | return $this->dataStreams()[$this->currentDataStream]->next(); 175 | } 176 | 177 | public function valid() 178 | { 179 | $dataStreams = $this->dataStreams(); 180 | if ($dataStreams[$this->currentDataStream]->valid()) { 181 | // current data stream is still valid 182 | return true; 183 | } else { 184 | ++$this->currentDataStream; 185 | if (isset($dataStreams[$this->currentDataStream])) { 186 | // current data stream is done, but we have another data stream 187 | return true; 188 | } else { 189 | // no more data and no more data streams 190 | return false; 191 | } 192 | } 193 | } 194 | 195 | public function getFileExtension() 196 | { 197 | return ''; 198 | } 199 | 200 | public function save($baseFilename) 201 | { 202 | $dataStreams = $this->dataStreams(); 203 | $numDataStreams = count($dataStreams); 204 | $fileNames = []; 205 | $i = 0; 206 | foreach ($dataStreams as $dataStream) { 207 | if ($numDataStreams == 1) { 208 | $filename = $baseFilename.$this->getFileExtension(); 209 | } else { 210 | $filename = $baseFilename.'-data-'.$i.$this->getFileExtension(); 211 | } 212 | $fileNames[] = $filename; 213 | $dataStream->save($filename); 214 | ++$i; 215 | } 216 | 217 | return $fileNames; 218 | } 219 | 220 | public static function validateDataSource($dataSource, $basePath = null) 221 | { 222 | $errors = []; 223 | $dataSource = static::normalizeDataSource($dataSource, $basePath); 224 | if (!Utils::isHttpSource($dataSource) && !file_exists($dataSource)) { 225 | $errors[] = new ResourceValidationError( 226 | ResourceValidationError::SCHEMA_VIOLATION, 227 | "data source file does not exist or is not readable: {$dataSource}" 228 | ); 229 | } 230 | 231 | return $errors; 232 | } 233 | 234 | /** 235 | * allows extending classes to add custom sources 236 | * used by unit tests to add a mock http source. 237 | * 238 | * @param string $dataSource 239 | * @param string|null $basePath 240 | * 241 | * @return string 242 | */ 243 | public static function normalizeDataSource($dataSource, string $basePath = null) 244 | { 245 | if (!empty($basePath) && !Utils::isHttpSource($dataSource)) { 246 | // TODO: support JSON pointers 247 | $absPath = $basePath.DIRECTORY_SEPARATOR.$dataSource; 248 | if (file_exists($absPath)) { 249 | $dataSource = $absPath; 250 | } 251 | } 252 | 253 | return $dataSource; 254 | } 255 | 256 | protected $descriptor; 257 | protected $basePath; 258 | protected $skipValidations = false; 259 | protected $currentDataPosition = 0; 260 | protected $currentDataStream = 0; 261 | protected $dataStreams = null; 262 | 263 | protected function validateResource() 264 | { 265 | return ResourceValidator::validate($this->descriptor(), $this->basePath); 266 | } 267 | 268 | /** 269 | * @param string $dataSource 270 | * 271 | * @return BaseDataStream 272 | */ 273 | abstract protected function getDataStream($dataSource); 274 | 275 | abstract protected function getInlineDataStream($data); 276 | 277 | protected static function handlesProfile($profile) 278 | { 279 | return false; 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /src/Resources/CustomResource.php: -------------------------------------------------------------------------------- 1 | normalizeDataSource($dataSource, $this->basePath), $dataSourceOptions); 21 | } 22 | 23 | protected function getInlineDataStream($data) 24 | { 25 | return $data; 26 | } 27 | 28 | protected static function handlesProfile($profile) 29 | { 30 | return $profile == 'data-resource'; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Resources/TabularResource.php: -------------------------------------------------------------------------------- 1 | descriptor()->schema; 15 | } 16 | 17 | public function getFileExtension() 18 | { 19 | return '.csv'; 20 | } 21 | 22 | /** 23 | * @param string $dataSource 24 | * 25 | * @param null $dataSourceOptions 26 | * 27 | * @return TabularDataStream 28 | * @throws \frictionlessdata\datapackage\Exceptions\DataStreamOpenException 29 | */ 30 | protected function getDataStream($dataSource, $dataSourceOptions = null) 31 | { 32 | $dataSourceOptions = array_merge([ 33 | 'schema' => $this->schema(), 34 | 'dialect' => isset($this->descriptor()->dialect) ? $this->descriptor()->dialect : null, 35 | ], (array) $dataSourceOptions); 36 | 37 | return new TabularDataStream($this->normalizeDataSource($dataSource, $this->basePath), $dataSourceOptions); 38 | } 39 | 40 | protected function getInlineDataStream($data) 41 | { 42 | return new TabularInlineDataStream($data, [ 43 | 'schema' => $this->schema(), 44 | 'dialect' => isset($this->descriptor()->dialect) ? $this->descriptor()->dialect : null, 45 | ]); 46 | } 47 | 48 | public static function handlesDescriptor($descriptor) 49 | { 50 | return 51 | Registry::getResourceValidationProfile($descriptor) == 'tabular-data-resource' 52 | || (isset($descriptor->format) && in_array($descriptor->format, ['csv', 'tsv'])) 53 | ; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/Utils.php: -------------------------------------------------------------------------------- 1 | basePath = $basePath; 15 | parent::__construct($descriptor); 16 | } 17 | 18 | /** 19 | * validate a descriptor. 20 | * 21 | * @param object $descriptor 22 | * @param null $basePath 23 | * 24 | * @return \frictionlessdata\tableschema\SchemaValidationError[] 25 | */ 26 | public static function validate($descriptor, $basePath = null) 27 | { 28 | $validator = new static($descriptor, $basePath); 29 | 30 | return $validator->getValidationErrors(); 31 | } 32 | 33 | protected $basePath; 34 | 35 | /** 36 | * should be implemented properly by extending classes 37 | * should return the profile used for validation 38 | * if using the default getValidationSchemaUrl function - this value should correspond to a file in schemas/ directory. 39 | * 40 | * @return string 41 | */ 42 | protected function getValidationProfile() 43 | { 44 | return $this->descriptor->profile; 45 | } 46 | 47 | protected function convertValidationSchemaFilenameToUrl($filename) 48 | { 49 | $filename = realpath($filename); 50 | if (file_exists($filename)) { 51 | return 'file://'.$filename; 52 | } else { 53 | throw new \Exception("failed to find schema file: '{$filename}' for descriptor ".json_encode($this->descriptor)); 54 | } 55 | } 56 | 57 | protected function getJsonSchemaFileFromRegistry($profile) 58 | { 59 | return false; 60 | } 61 | 62 | /** 63 | * Get the url which the schema for validation can be fetched from. 64 | * 65 | * @return string 66 | * @throws \Exception 67 | * @throws \Exception 68 | */ 69 | protected function getValidationSchemaUrl() 70 | { 71 | $profile = $this->getValidationProfile(); 72 | if ($filename = $this->getJsonSchemaFileFromRegistry($profile)) { 73 | // known profile id in the registry 74 | return $this->convertValidationSchemaFilenameToUrl($filename); 75 | } elseif (Utils::isHttpSource($profile)) { 76 | // url 77 | return $profile; 78 | } elseif (file_exists($filename = $this->basePath.DIRECTORY_SEPARATOR.$profile)) { 79 | // relative path - prefixed with basePath 80 | return $this->convertValidationSchemaFilenameToUrl($filename); 81 | } else { 82 | // absolute path (or relative to current working directory) 83 | return $this->convertValidationSchemaFilenameToUrl($profile); 84 | } 85 | } 86 | 87 | /** 88 | * Allows to specify different error classes for different validators. 89 | * 90 | * @return string 91 | */ 92 | protected function getSchemaValidationErrorClass() 93 | { 94 | return 'frictionlessdata\\tableschema\\SchemaValidationError'; 95 | } 96 | 97 | /** 98 | * Allows extending classes to modify the descriptor before passing to the validator. 99 | * 100 | * @return object 101 | */ 102 | protected function getDescriptorForValidation() 103 | { 104 | return $this->descriptor; 105 | } 106 | 107 | /** 108 | * Convert the validation error message received from JsonSchema to human readable string. 109 | * 110 | * @param array $error 111 | * 112 | * @return string 113 | */ 114 | protected function getValidationErrorMessage(array $error) 115 | { 116 | return sprintf('[%s] %s', $error['property'], $error['message']); 117 | } 118 | 119 | /** 120 | * Does the validation, adds errors to the validator object using _addError method. 121 | */ 122 | protected function validateSchema() 123 | { 124 | $this->validateSchemaUrl($this->getValidationSchemaUrl()); 125 | } 126 | 127 | protected function validateSchemaUrl($url) 128 | { 129 | $validator = new \JsonSchema\Validator(); 130 | $descriptor = $this->getDescriptorForValidation(); 131 | $validator->validate($descriptor, (object) ['$ref' => $url]); 132 | if (!$validator->isValid()) { 133 | foreach ($validator->getErrors() as $error) { 134 | $this->addError( 135 | SchemaValidationError::SCHEMA_VIOLATION, 136 | $this->getValidationErrorMessage($error) 137 | ); 138 | } 139 | } 140 | } 141 | 142 | /** 143 | * Add an error to the validator object - errors are aggregated and returned by validate function. 144 | * 145 | * @param int $code 146 | * @param null|mixed $extraDetails 147 | */ 148 | protected function addError($code, $extraDetails = null) 149 | { 150 | // modified from parent function to support changing the error class 151 | $errorClass = $this->getSchemaValidationErrorClass(); 152 | $this->errors[] = new $errorClass($code, $extraDetails); 153 | } 154 | 155 | protected function validateKeys() 156 | { 157 | // this can be used to do further validations on $this->descriptor 158 | // it will run only in case validateSchema succeeded 159 | // so no need to check if attribute exists or in correct type 160 | // the parent SchemaValidator does some checks specific to table schema - so don't call the parent function 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/Validators/DatapackageValidationError.php: -------------------------------------------------------------------------------- 1 | code) { 16 | case static::RESOURCE_FAILED_VALIDATION: 17 | return "resource {$this->extraDetails['resource']} failed validation: " 18 | .ResourceValidationError::getErrorMessages($this->extraDetails['validationErrors']); 19 | case static::DATA_STREAM_FAILURE: 20 | return "resource {$this->extraDetails['resource']}" 21 | //."data stream {$this->extraDetails['dataStream']}" 22 | .($this->extraDetails['line'] ? ", line number {$this->extraDetails['line']}" : '') 23 | .': '.$this->extraDetails['error']; 24 | default: 25 | return parent::getMessage(); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Validators/DatapackageValidator.php: -------------------------------------------------------------------------------- 1 | descriptor); 21 | } 22 | 23 | protected function getDescriptorForValidation() 24 | { 25 | // add base path to uri fields - it runs before validations, so need to validate the attributes we need 26 | // TODO: find a more elegant way to do it with support for registring custom url fields 27 | $descriptor = clone $this->descriptor; 28 | if (isset($descriptor->resources) && is_array($descriptor->resources)) { 29 | foreach ($descriptor->resources as &$resource) { 30 | if (is_object($resource)) { 31 | $resource = clone $resource; 32 | if (isset($resource->path)) { 33 | if (is_array($resource->path)) { 34 | foreach ($resource->path as &$url) { 35 | if (is_string($url)) { 36 | $url = 'file://'.$url; 37 | } 38 | } 39 | } elseif (is_string($resource->path)) { 40 | $resource->path .= 'file://'.$resource->path; 41 | } 42 | } 43 | } 44 | } 45 | } 46 | 47 | return $descriptor; 48 | } 49 | 50 | protected function validateSchema() 51 | { 52 | parent::validateSchema(); 53 | if ($this->getValidationProfile() != 'data-package') { 54 | // all schemas must be an extension of datapackage spec 55 | $this->validateSchemaUrl( 56 | $this->convertValidationSchemaFilenameToUrl( 57 | $this->getJsonSchemaFileFromRegistry('data-package') 58 | ) 59 | ); 60 | } 61 | } 62 | 63 | protected function validateKeys() 64 | { 65 | foreach ($this->descriptor->resources as $resourceDescriptor) { 66 | foreach ($this->resourceValidate($resourceDescriptor) as $error) { 67 | $this->errors[] = $error; 68 | } 69 | } 70 | } 71 | 72 | protected function resourceValidate($resourceDescriptor) 73 | { 74 | return ResourceValidator::validate($resourceDescriptor, $this->basePath); 75 | } 76 | 77 | protected function getJsonSchemaFileFromRegistry($profile) 78 | { 79 | if ($filename = Registry::getJsonSchemaFile($profile)) { 80 | return $filename; 81 | } else { 82 | return parent::getJsonSchemaFileFromRegistry($profile); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/Validators/ResourceValidationError.php: -------------------------------------------------------------------------------- 1 | descriptor); 22 | } 23 | 24 | protected function getDescriptorForValidation() 25 | { 26 | $descriptor = clone $this->descriptor; 27 | 28 | return $descriptor; 29 | } 30 | 31 | protected function getValidationErrorMessage(array $error) 32 | { 33 | $property = $error['property']; 34 | // silly hack to only show properties within the resource of the fake datapackage 35 | // $property = str_replace("resources[0].", "", $property); 36 | return sprintf('[%s] %s', $property, $error['message']); 37 | } 38 | 39 | protected function getResourceClass() 40 | { 41 | return Factory::getResourceClass($this->descriptor); 42 | } 43 | 44 | protected function validateKeys() 45 | { 46 | $resourceClass = $this->getResourceClass(); 47 | // DS: path can be a string or an array 48 | $sources = is_array($this->descriptor->path) ? $this->descriptor->path : array($this->descriptor->path); 49 | foreach ($sources as $dataSource) { 50 | foreach ($resourceClass::validateDataSource($dataSource, $this->basePath) as $error) { 51 | $this->errors[] = $error; 52 | } 53 | } 54 | } 55 | 56 | protected function getJsonSchemaFileFromRegistry($profile) 57 | { 58 | if ($filename = Registry::getJsonSchemaFile($profile)) { 59 | return $filename; 60 | } else { 61 | return parent::getJsonSchemaFileFromRegistry($profile); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Validators/schemas/CHANGELOG: -------------------------------------------------------------------------------- 1 | 2017-07-13T16:25:21+03:00 2 | 3 | * added table-schema.json to registry (no effect on datapackage library) 4 | * uri format was changed to simpler string format without uri restrictions (to support paths) 5 | * added organisation 6 | * changed role enum 7 | * data attribute changed to path which can be either a single string or array of strings 8 | * changes to table-schema (not affecting datapackage directly) 9 | 10 | 2017-11-21T20:55:19+02:00 11 | 12 | * added schemas from registry: data-resource, tabular-data-resource 13 | * default profile changed from 'default' to 'data-package' 14 | * homepage type changed from 'object' to 'string' with format 'uri' 15 | * changed contributor 'name' attribute to 'title' 16 | * added default contributor role - 'contributor' 17 | * added image attribute - an image to represent the data package (string) 18 | * license name - must be an Open Definition license identifier - "^([-a-zA-Z0-9._])+$" 19 | * resource must have either one of "data" or "path" attribute (but not both) 20 | * default resource profile changed from 'default' to 'data-resource' 21 | * added 'data' - Inline data for this resource 22 | * changed source 'name' attribute to 'title' 23 | -------------------------------------------------------------------------------- /src/Validators/schemas/data-package.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "title": "Data Package", 4 | "description": "Data Package is a simple specification for data access and delivery.", 5 | "type": "object", 6 | "required": [ 7 | "resources" 8 | ], 9 | "properties": { 10 | "profile": { 11 | "default": "data-package", 12 | "propertyOrder": 10, 13 | "title": "Profile", 14 | "description": "The profile of this descriptor.", 15 | "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", 16 | "type": "string", 17 | "examples": [ 18 | "{\n \"profile\": \"tabular-data-package\"\n}\n", 19 | "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" 20 | ] 21 | }, 22 | "name": { 23 | "propertyOrder": 20, 24 | "title": "Name", 25 | "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", 26 | "type": "string", 27 | "pattern": "^([-a-z0-9._/])+$", 28 | "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", 29 | "examples": [ 30 | "{\n \"name\": \"my-nice-name\"\n}\n" 31 | ] 32 | }, 33 | "id": { 34 | "propertyOrder": 30, 35 | "title": "ID", 36 | "description": "A property reserved for globally unique identifiers. Examples of identifiers that are unique include UUIDs and DOIs.", 37 | "context": "A common usage pattern for Data Packages is as a packaging format within the bounds of a system or platform. In these cases, a unique identifier for a package is desired for common data handling workflows, such as updating an existing package. While at the level of the specification, global uniqueness cannot be validated, consumers using the `id` property `MUST` ensure identifiers are globally unique.", 38 | "type": "string", 39 | "examples": [ 40 | "{\n \"id\": \"b03ec84-77fd-4270-813b-0c698943f7ce\"\n}\n", 41 | "{\n \"id\": \"http://dx.doi.org/10.1594/PANGAEA.726855\"\n}\n" 42 | ] 43 | }, 44 | "title": { 45 | "propertyOrder": 40, 46 | "title": "Title", 47 | "description": "A human-readable title.", 48 | "type": "string", 49 | "examples": [ 50 | "{\n \"title\": \"My Package Title\"\n}\n" 51 | ] 52 | }, 53 | "description": { 54 | "propertyOrder": 50, 55 | "format": "textarea", 56 | "title": "Description", 57 | "description": "A text description. Markdown is encouraged.", 58 | "type": "string", 59 | "examples": [ 60 | "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" 61 | ] 62 | }, 63 | "homepage": { 64 | "propertyOrder": 60, 65 | "title": "Home Page", 66 | "description": "The home on the web that is related to this data package.", 67 | "type": "string", 68 | "format": "uri", 69 | "examples": [ 70 | "{\n \"homepage\": \"http://example.com/\"\n}\n" 71 | ] 72 | }, 73 | "created": { 74 | "propertyOrder": 70, 75 | "title": "Created", 76 | "description": "The datetime on which this descriptor was created.", 77 | "context": "The datetime must conform to the string formats for datetime as described in [RFC3339](https://tools.ietf.org/html/rfc3339#section-5.6)", 78 | "type": "string", 79 | "format": "date-time", 80 | "examples": [ 81 | "{\n \"created\": \"1985-04-12T23:20:50.52Z\"\n}\n" 82 | ] 83 | }, 84 | "contributors": { 85 | "propertyOrder": 80, 86 | "title": "Contributors", 87 | "description": "The contributors to this descriptor.", 88 | "type": "array", 89 | "minItems": 1, 90 | "items": { 91 | "title": "Contributor", 92 | "description": "A contributor to this descriptor.", 93 | "properties": { 94 | "title": { 95 | "title": "Title", 96 | "description": "A human-readable title.", 97 | "type": "string", 98 | "examples": [ 99 | "{\n \"title\": \"My Package Title\"\n}\n" 100 | ] 101 | }, 102 | "path": { 103 | "title": "Path", 104 | "description": "A fully qualified URL, or a POSIX file path..", 105 | "type": "string", 106 | "examples": [ 107 | "{\n \"path\": \"file.csv\"\n}\n", 108 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 109 | ], 110 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 111 | }, 112 | "email": { 113 | "title": "Email", 114 | "description": "An email address.", 115 | "type": "string", 116 | "format": "email", 117 | "examples": [ 118 | "{\n \"email\": \"example@example.com\"\n}\n" 119 | ] 120 | }, 121 | "organisation": { 122 | "title": "Organization", 123 | "description": "An organizational affiliation for this contributor.", 124 | "type": "string" 125 | }, 126 | "role": { 127 | "type": "string", 128 | "enum": [ 129 | "publisher", 130 | "author", 131 | "maintainer", 132 | "wrangler", 133 | "contributor" 134 | ], 135 | "default": "contributor" 136 | } 137 | }, 138 | "required": [ 139 | "title" 140 | ], 141 | "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." 142 | }, 143 | "examples": [ 144 | "{\n \"contributors\": [\n {\n \"title\": \"Joe Bloggs\"\n }\n ]\n}\n", 145 | "{\n \"contributors\": [\n {\n \"title\": \"Joe Bloggs\",\n \"email\": \"joe@example.com\",\n \"role\": \"author\"\n }\n ]\n}\n" 146 | ] 147 | }, 148 | "keywords": { 149 | "propertyOrder": 90, 150 | "title": "Keywords", 151 | "description": "A list of keywords that describe this package.", 152 | "type": "array", 153 | "minItems": 1, 154 | "items": { 155 | "type": "string" 156 | }, 157 | "examples": [ 158 | "{\n \"keywords\": [\n \"data\",\n \"fiscal\",\n \"transparency\"\n ]\n}\n" 159 | ] 160 | }, 161 | "image": { 162 | "propertyOrder": 100, 163 | "title": "Image", 164 | "description": "A image to represent this package.", 165 | "type": "string", 166 | "examples": [ 167 | "{\n \"image\": \"http://example.com/image.jpg\"\n}\n", 168 | "{\n \"image\": \"relative/to/image.jpg\"\n}\n" 169 | ] 170 | }, 171 | "licenses": { 172 | "propertyOrder": 110, 173 | "title": "Licenses", 174 | "description": "The license(s) under which this package is published.", 175 | "type": "array", 176 | "minItems": 1, 177 | "items": { 178 | "title": "License", 179 | "description": "A license for this descriptor.", 180 | "type": "object", 181 | "properties": { 182 | "name": { 183 | "title": "Open Definition license identifier", 184 | "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", 185 | "type": "string", 186 | "pattern": "^([-a-zA-Z0-9._])+$" 187 | }, 188 | "path": { 189 | "title": "Path", 190 | "description": "A fully qualified URL, or a POSIX file path..", 191 | "type": "string", 192 | "examples": [ 193 | "{\n \"path\": \"file.csv\"\n}\n", 194 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 195 | ], 196 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 197 | }, 198 | "title": { 199 | "title": "Title", 200 | "description": "A human-readable title.", 201 | "type": "string", 202 | "examples": [ 203 | "{\n \"title\": \"My Package Title\"\n}\n" 204 | ] 205 | } 206 | }, 207 | "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." 208 | }, 209 | "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", 210 | "examples": [ 211 | "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"uri\": \"http://opendatacommons.org/licenses/pddl/\"\n }\n ]\n}\n" 212 | ] 213 | }, 214 | "resources": { 215 | "propertyOrder": 120, 216 | "title": "Data Resources", 217 | "description": "An `array` of Data Resource objects, each compliant with the [Data Resource](/data-resource/) specification.", 218 | "type": "array", 219 | "minItems": 1, 220 | "items": { 221 | "title": "Data Resource", 222 | "description": "Data Resource.", 223 | "type": "object", 224 | "oneOf": [ 225 | { 226 | "required": [ 227 | "name", 228 | "data" 229 | ] 230 | }, 231 | { 232 | "required": [ 233 | "name", 234 | "path" 235 | ] 236 | } 237 | ], 238 | "properties": { 239 | "profile": { 240 | "propertyOrder": 10, 241 | "default": "data-resource", 242 | "title": "Profile", 243 | "description": "The profile of this descriptor.", 244 | "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", 245 | "type": "string", 246 | "examples": [ 247 | "{\n \"profile\": \"tabular-data-package\"\n}\n", 248 | "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" 249 | ] 250 | }, 251 | "name": { 252 | "propertyOrder": 20, 253 | "title": "Name", 254 | "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", 255 | "type": "string", 256 | "pattern": "^([-a-z0-9._/])+$", 257 | "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", 258 | "examples": [ 259 | "{\n \"name\": \"my-nice-name\"\n}\n" 260 | ] 261 | }, 262 | "path": { 263 | "propertyOrder": 30, 264 | "title": "Path", 265 | "description": "A reference to the data for this resource, as either a path as a string, or an array of paths as strings. of valid URIs.", 266 | "oneOf": [ 267 | { 268 | "title": "Path", 269 | "description": "A fully qualified URL, or a POSIX file path..", 270 | "type": "string", 271 | "examples": [ 272 | "{\n \"path\": \"file.csv\"\n}\n", 273 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 274 | ], 275 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 276 | }, 277 | { 278 | "type": "array", 279 | "minItems": 1, 280 | "items": { 281 | "title": "Path", 282 | "description": "A fully qualified URL, or a POSIX file path..", 283 | "type": "string", 284 | "examples": [ 285 | "{\n \"path\": \"file.csv\"\n}\n", 286 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 287 | ], 288 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 289 | }, 290 | "examples": [ 291 | "[ \"file.csv\" ]\n", 292 | "[ \"http://example.com/file.csv\" ]\n" 293 | ] 294 | } 295 | ], 296 | "context": "The dereferenced value of each referenced data source in `path` `MUST` be commensurate with a native, dereferenced representation of the data the resource describes. For example, in a *Tabular* Data Resource, this means that the dereferenced value of `path` `MUST` be an array.", 297 | "examples": [ 298 | "{\n \"path\": [\n \"file.csv\",\n \"file2.csv\"\n ]\n}\n", 299 | "{\n \"path\": [\n \"http://example.com/file.csv\",\n \"http://example.com/file2.csv\"\n ]\n}\n", 300 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 301 | ] 302 | }, 303 | "data": { 304 | "propertyOrder": 230, 305 | "title": "Data", 306 | "description": "Inline data for this resource." 307 | }, 308 | "schema": { 309 | "propertyOrder": 40, 310 | "title": "Schema", 311 | "description": "A schema for this resource.", 312 | "type": "object" 313 | }, 314 | "title": { 315 | "propertyOrder": 50, 316 | "title": "Title", 317 | "description": "A human-readable title.", 318 | "type": "string", 319 | "examples": [ 320 | "{\n \"title\": \"My Package Title\"\n}\n" 321 | ] 322 | }, 323 | "description": { 324 | "propertyOrder": 60, 325 | "format": "textarea", 326 | "title": "Description", 327 | "description": "A text description. Markdown is encouraged.", 328 | "type": "string", 329 | "examples": [ 330 | "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" 331 | ] 332 | }, 333 | "homepage": { 334 | "propertyOrder": 70, 335 | "title": "Home Page", 336 | "description": "The home on the web that is related to this data package.", 337 | "type": "string", 338 | "format": "uri", 339 | "examples": [ 340 | "{\n \"homepage\": \"http://example.com/\"\n}\n" 341 | ] 342 | }, 343 | "sources": { 344 | "propertyOrder": 140, 345 | "options": { 346 | "hidden": true 347 | }, 348 | "title": "Sources", 349 | "description": "The raw sources for this resource.", 350 | "type": "array", 351 | "minItems": 1, 352 | "items": { 353 | "title": "Source", 354 | "description": "A source file.", 355 | "type": "object", 356 | "required": [ 357 | "title" 358 | ], 359 | "properties": { 360 | "title": { 361 | "title": "Title", 362 | "description": "A human-readable title.", 363 | "type": "string", 364 | "examples": [ 365 | "{\n \"title\": \"My Package Title\"\n}\n" 366 | ] 367 | }, 368 | "path": { 369 | "title": "Path", 370 | "description": "A fully qualified URL, or a POSIX file path..", 371 | "type": "string", 372 | "examples": [ 373 | "{\n \"path\": \"file.csv\"\n}\n", 374 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 375 | ], 376 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 377 | }, 378 | "email": { 379 | "title": "Email", 380 | "description": "An email address.", 381 | "type": "string", 382 | "format": "email", 383 | "examples": [ 384 | "{\n \"email\": \"example@example.com\"\n}\n" 385 | ] 386 | } 387 | } 388 | }, 389 | "examples": [ 390 | "{\n \"sources\": [\n {\n \"name\": \"World Bank and OECD\",\n \"uri\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" 391 | ] 392 | }, 393 | "licenses": { 394 | "description": "The license(s) under which the resource is published.", 395 | "propertyOrder": 150, 396 | "options": { 397 | "hidden": true 398 | }, 399 | "title": "Licenses", 400 | "type": "array", 401 | "minItems": 1, 402 | "items": { 403 | "title": "License", 404 | "description": "A license for this descriptor.", 405 | "type": "object", 406 | "properties": { 407 | "name": { 408 | "title": "Open Definition license identifier", 409 | "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", 410 | "type": "string", 411 | "pattern": "^([-a-zA-Z0-9._])+$" 412 | }, 413 | "path": { 414 | "title": "Path", 415 | "description": "A fully qualified URL, or a POSIX file path..", 416 | "type": "string", 417 | "examples": [ 418 | "{\n \"path\": \"file.csv\"\n}\n", 419 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 420 | ], 421 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 422 | }, 423 | "title": { 424 | "title": "Title", 425 | "description": "A human-readable title.", 426 | "type": "string", 427 | "examples": [ 428 | "{\n \"title\": \"My Package Title\"\n}\n" 429 | ] 430 | } 431 | }, 432 | "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." 433 | }, 434 | "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", 435 | "examples": [ 436 | "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"uri\": \"http://opendatacommons.org/licenses/pddl/\"\n }\n ]\n}\n" 437 | ] 438 | }, 439 | "format": { 440 | "propertyOrder": 80, 441 | "title": "Format", 442 | "description": "The file format of this resource.", 443 | "context": "`csv`, `xls`, `json` are examples of common formats.", 444 | "type": "string", 445 | "examples": [ 446 | "{\n \"format\": \"xls\"\n}\n" 447 | ] 448 | }, 449 | "mediatype": { 450 | "propertyOrder": 90, 451 | "title": "Media Type", 452 | "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", 453 | "type": "string", 454 | "pattern": "^(.+)/(.+)$", 455 | "examples": [ 456 | "{\n \"mediatype\": \"text/csv\"\n}\n" 457 | ] 458 | }, 459 | "encoding": { 460 | "propertyOrder": 100, 461 | "title": "Encoding", 462 | "description": "The file encoding of this resource.", 463 | "type": "string", 464 | "default": "utf-8", 465 | "examples": [ 466 | "{\n \"encoding\": \"utf-8\"\n}\n" 467 | ] 468 | }, 469 | "bytes": { 470 | "propertyOrder": 110, 471 | "options": { 472 | "hidden": true 473 | }, 474 | "title": "Bytes", 475 | "description": "The size of this resource in bytes.", 476 | "type": "integer", 477 | "examples": [ 478 | "{\n \"bytes\": 2082\n}\n" 479 | ] 480 | }, 481 | "hash": { 482 | "propertyOrder": 120, 483 | "options": { 484 | "hidden": true 485 | }, 486 | "title": "Hash", 487 | "type": "string", 488 | "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", 489 | "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", 490 | "examples": [ 491 | "{\n \"hash\": \"d25c9c77f588f5dc32059d2da1136c02\"\n}\n", 492 | "{\n \"hash\": \"SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0\"\n}\n" 493 | ] 494 | } 495 | } 496 | }, 497 | "examples": [ 498 | "{\n \"resources\": [\n {\n \"name\": \"my-data\",\n \"data\": [\n \"data.csv\"\n ],\n \"mediatype\": \"text/csv\"\n }\n ]\n}\n" 499 | ] 500 | }, 501 | "sources": { 502 | "propertyOrder": 200, 503 | "options": { 504 | "hidden": true 505 | }, 506 | "title": "Sources", 507 | "description": "The raw sources for this resource.", 508 | "type": "array", 509 | "minItems": 1, 510 | "items": { 511 | "title": "Source", 512 | "description": "A source file.", 513 | "type": "object", 514 | "required": [ 515 | "title" 516 | ], 517 | "properties": { 518 | "title": { 519 | "title": "Title", 520 | "description": "A human-readable title.", 521 | "type": "string", 522 | "examples": [ 523 | "{\n \"title\": \"My Package Title\"\n}\n" 524 | ] 525 | }, 526 | "path": { 527 | "title": "Path", 528 | "description": "A fully qualified URL, or a POSIX file path..", 529 | "type": "string", 530 | "examples": [ 531 | "{\n \"path\": \"file.csv\"\n}\n", 532 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 533 | ], 534 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 535 | }, 536 | "email": { 537 | "title": "Email", 538 | "description": "An email address.", 539 | "type": "string", 540 | "format": "email", 541 | "examples": [ 542 | "{\n \"email\": \"example@example.com\"\n}\n" 543 | ] 544 | } 545 | } 546 | }, 547 | "examples": [ 548 | "{\n \"sources\": [\n {\n \"name\": \"World Bank and OECD\",\n \"uri\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" 549 | ] 550 | } 551 | } 552 | } -------------------------------------------------------------------------------- /src/Validators/schemas/data-resource.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "title": "Data Resource", 4 | "description": "Data Resource.", 5 | "type": "object", 6 | "oneOf": [ 7 | { 8 | "required": [ 9 | "name", 10 | "data" 11 | ] 12 | }, 13 | { 14 | "required": [ 15 | "name", 16 | "path" 17 | ] 18 | } 19 | ], 20 | "properties": { 21 | "profile": { 22 | "propertyOrder": 10, 23 | "default": "data-resource", 24 | "title": "Profile", 25 | "description": "The profile of this descriptor.", 26 | "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `data-package` for Package and `data-resource` for Resource.", 27 | "type": "string", 28 | "examples": [ 29 | "{\n \"profile\": \"tabular-data-package\"\n}\n", 30 | "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" 31 | ] 32 | }, 33 | "name": { 34 | "propertyOrder": 20, 35 | "title": "Name", 36 | "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", 37 | "type": "string", 38 | "pattern": "^([-a-z0-9._/])+$", 39 | "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", 40 | "examples": [ 41 | "{\n \"name\": \"my-nice-name\"\n}\n" 42 | ] 43 | }, 44 | "path": { 45 | "propertyOrder": 30, 46 | "title": "Path", 47 | "description": "A reference to the data for this resource, as either a path as a string, or an array of paths as strings. of valid URIs.", 48 | "oneOf": [ 49 | { 50 | "title": "Path", 51 | "description": "A fully qualified URL, or a POSIX file path..", 52 | "type": "string", 53 | "examples": [ 54 | "{\n \"path\": \"file.csv\"\n}\n", 55 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 56 | ], 57 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 58 | }, 59 | { 60 | "type": "array", 61 | "minItems": 1, 62 | "items": { 63 | "title": "Path", 64 | "description": "A fully qualified URL, or a POSIX file path..", 65 | "type": "string", 66 | "examples": [ 67 | "{\n \"path\": \"file.csv\"\n}\n", 68 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 69 | ], 70 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 71 | }, 72 | "examples": [ 73 | "[ \"file.csv\" ]\n", 74 | "[ \"http://example.com/file.csv\" ]\n" 75 | ] 76 | } 77 | ], 78 | "context": "The dereferenced value of each referenced data source in `path` `MUST` be commensurate with a native, dereferenced representation of the data the resource describes. For example, in a *Tabular* Data Resource, this means that the dereferenced value of `path` `MUST` be an array.", 79 | "examples": [ 80 | "{\n \"path\": [\n \"file.csv\",\n \"file2.csv\"\n ]\n}\n", 81 | "{\n \"path\": [\n \"http://example.com/file.csv\",\n \"http://example.com/file2.csv\"\n ]\n}\n", 82 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 83 | ] 84 | }, 85 | "data": { 86 | "propertyOrder": 230, 87 | "title": "Data", 88 | "description": "Inline data for this resource." 89 | }, 90 | "schema": { 91 | "propertyOrder": 40, 92 | "title": "Schema", 93 | "description": "A schema for this resource.", 94 | "type": "object" 95 | }, 96 | "title": { 97 | "propertyOrder": 50, 98 | "title": "Title", 99 | "description": "A human-readable title.", 100 | "type": "string", 101 | "examples": [ 102 | "{\n \"title\": \"My Package Title\"\n}\n" 103 | ] 104 | }, 105 | "description": { 106 | "propertyOrder": 60, 107 | "format": "textarea", 108 | "title": "Description", 109 | "description": "A text description. Markdown is encouraged.", 110 | "type": "string", 111 | "examples": [ 112 | "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" 113 | ] 114 | }, 115 | "homepage": { 116 | "propertyOrder": 70, 117 | "title": "Home Page", 118 | "description": "The home on the web that is related to this data package.", 119 | "type": "string", 120 | "format": "uri", 121 | "examples": [ 122 | "{\n \"homepage\": \"http://example.com/\"\n}\n" 123 | ] 124 | }, 125 | "sources": { 126 | "propertyOrder": 140, 127 | "options": { 128 | "hidden": true 129 | }, 130 | "title": "Sources", 131 | "description": "The raw sources for this resource.", 132 | "type": "array", 133 | "minItems": 1, 134 | "items": { 135 | "title": "Source", 136 | "description": "A source file.", 137 | "type": "object", 138 | "required": [ 139 | "title" 140 | ], 141 | "properties": { 142 | "title": { 143 | "title": "Title", 144 | "description": "A human-readable title.", 145 | "type": "string", 146 | "examples": [ 147 | "{\n \"title\": \"My Package Title\"\n}\n" 148 | ] 149 | }, 150 | "path": { 151 | "title": "Path", 152 | "description": "A fully qualified URL, or a POSIX file path..", 153 | "type": "string", 154 | "examples": [ 155 | "{\n \"path\": \"file.csv\"\n}\n", 156 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 157 | ], 158 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 159 | }, 160 | "email": { 161 | "title": "Email", 162 | "description": "An email address.", 163 | "type": "string", 164 | "format": "email", 165 | "examples": [ 166 | "{\n \"email\": \"example@example.com\"\n}\n" 167 | ] 168 | } 169 | } 170 | }, 171 | "examples": [ 172 | "{\n \"sources\": [\n {\n \"name\": \"World Bank and OECD\",\n \"uri\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" 173 | ] 174 | }, 175 | "licenses": { 176 | "description": "The license(s) under which the resource is published.", 177 | "propertyOrder": 150, 178 | "options": { 179 | "hidden": true 180 | }, 181 | "title": "Licenses", 182 | "type": "array", 183 | "minItems": 1, 184 | "items": { 185 | "title": "License", 186 | "description": "A license for this descriptor.", 187 | "type": "object", 188 | "properties": { 189 | "name": { 190 | "title": "Open Definition license identifier", 191 | "description": "MUST be an Open Definition license identifier, see http://licenses.opendefinition.org/", 192 | "type": "string", 193 | "pattern": "^([-a-zA-Z0-9._])+$" 194 | }, 195 | "path": { 196 | "title": "Path", 197 | "description": "A fully qualified URL, or a POSIX file path..", 198 | "type": "string", 199 | "examples": [ 200 | "{\n \"path\": \"file.csv\"\n}\n", 201 | "{\n \"path\": \"http://example.com/file.csv\"\n}\n" 202 | ], 203 | "context": "Implementations need to negotiate the type of path provided, and dereference the data accordingly." 204 | }, 205 | "title": { 206 | "title": "Title", 207 | "description": "A human-readable title.", 208 | "type": "string", 209 | "examples": [ 210 | "{\n \"title\": \"My Package Title\"\n}\n" 211 | ] 212 | } 213 | }, 214 | "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." 215 | }, 216 | "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", 217 | "examples": [ 218 | "{\n \"licenses\": [\n {\n \"name\": \"odc-pddl-1.0\",\n \"uri\": \"http://opendatacommons.org/licenses/pddl/\"\n }\n ]\n}\n" 219 | ] 220 | }, 221 | "format": { 222 | "propertyOrder": 80, 223 | "title": "Format", 224 | "description": "The file format of this resource.", 225 | "context": "`csv`, `xls`, `json` are examples of common formats.", 226 | "type": "string", 227 | "examples": [ 228 | "{\n \"format\": \"xls\"\n}\n" 229 | ] 230 | }, 231 | "mediatype": { 232 | "propertyOrder": 90, 233 | "title": "Media Type", 234 | "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", 235 | "type": "string", 236 | "pattern": "^(.+)/(.+)$", 237 | "examples": [ 238 | "{\n \"mediatype\": \"text/csv\"\n}\n" 239 | ] 240 | }, 241 | "encoding": { 242 | "propertyOrder": 100, 243 | "title": "Encoding", 244 | "description": "The file encoding of this resource.", 245 | "type": "string", 246 | "default": "utf-8", 247 | "examples": [ 248 | "{\n \"encoding\": \"utf-8\"\n}\n" 249 | ] 250 | }, 251 | "bytes": { 252 | "propertyOrder": 110, 253 | "options": { 254 | "hidden": true 255 | }, 256 | "title": "Bytes", 257 | "description": "The size of this resource in bytes.", 258 | "type": "integer", 259 | "examples": [ 260 | "{\n \"bytes\": 2082\n}\n" 261 | ] 262 | }, 263 | "hash": { 264 | "propertyOrder": 120, 265 | "options": { 266 | "hidden": true 267 | }, 268 | "title": "Hash", 269 | "type": "string", 270 | "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", 271 | "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", 272 | "examples": [ 273 | "{\n \"hash\": \"d25c9c77f588f5dc32059d2da1136c02\"\n}\n", 274 | "{\n \"hash\": \"SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0\"\n}\n" 275 | ] 276 | } 277 | } 278 | } -------------------------------------------------------------------------------- /src/Validators/schemas/registry.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "data-package", 4 | "title": "Data Package", 5 | "schema": "https://specs.frictionlessdata.io/schemas/data-package.json", 6 | "schema_path": "data-package.json", 7 | "specification": "https://specs.frictionlessdata.io/data-package/" 8 | }, 9 | { 10 | "id": "tabular-data-package", 11 | "title": "Tabular Data Package", 12 | "schema": "https://specs.frictionlessdata.io/schemas/tabular-data-package.json", 13 | "schema_path": "tabular-data-package.json", 14 | "specification": "http://specs.frictionlessdata.io/tabular-data-package/" 15 | }, 16 | { 17 | "id": "fiscal-data-package", 18 | "title": "Fiscal Data Package", 19 | "schema": "https://specs.frictionlessdata.io/schemas/fiscal-data-package.json", 20 | "schema_path": "fiscal-data-package.json", 21 | "specification": "https://specs.frictionlessdata.io/fiscal-data-package/" 22 | }, 23 | { 24 | "id": "data-resource", 25 | "title": "Data Resource", 26 | "schema": "https://specs.frictionlessdata.io/schemas/data-resource.json", 27 | "schema_path": "data-resource.json", 28 | "specification": "https://specs.frictionlessdata.io/data-resource" 29 | }, 30 | { 31 | "id": "tabular-data-resource", 32 | "title": "Tabular Data Resource", 33 | "schema": "https://specs.frictionlessdata.io/schemas/tabular-data-resource.json", 34 | "schema_path": "tabular-data-resource.json", 35 | "specification": "https://specs.frictionlessdata.io/tabular-data-resource" 36 | }, 37 | { 38 | "id": "table-schema", 39 | "title": "Table Schema", 40 | "schema": "https://specs.frictionlessdata.io/schemas/table-schema.json", 41 | "schema_path": "table-schema.json", 42 | "specification": "https://specs.frictionlessdata.io/table-schema/" 43 | } 44 | ] -------------------------------------------------------------------------------- /tests/FactoryTest.php: -------------------------------------------------------------------------------- 1 | 'my-custom-datapackage', 20 | 'resources' => [ 21 | (object) ['name' => 'my-custom-resource', 'path' => ['tests/fixtures/foo.txt']], 22 | ], 23 | ]; 24 | // get a datapackage object based on this descriptor 25 | $datapackage = Factory::datapackage($descriptor); 26 | // the custom datapackage is not used (because the myCustomDatapackage property didn't exist) 27 | $this->assertEquals( 28 | 'frictionlessdata\\datapackage\\Datapackages\\DefaultDatapackage', 29 | get_class($datapackage) 30 | ); 31 | // add the myCustomDatapackage property to the descriptor 32 | $descriptor->myCustomDatapackage = true; 33 | // get a datapackage object 34 | $datapackage = Factory::datapackage($descriptor); 35 | // voila - we got a MyCustomDatapackage class 36 | $this->assertEquals( 37 | 'frictionlessdata\\datapackage\\tests\\Mocks\\MyCustomDatapackage', 38 | get_class($datapackage) 39 | ); 40 | // make sure to clear the custom datapackage class we registered 41 | Factory::clearRegisteredDatapackageClasses(); 42 | // create a datapackage object from the descriptor with the myCustomDatapackage property 43 | $datapackage = Factory::datapackage($descriptor); 44 | // got the normal default datapackage class (because we cleared the custom registered classes) 45 | $this->assertEquals( 46 | 'frictionlessdata\\datapackage\\Datapackages\\DefaultDatapackage', 47 | get_class($datapackage) 48 | ); 49 | } 50 | 51 | public function testRegisterResourceClass() 52 | { 53 | // register a custom resource class which is used for resources that have the goGoPowerRangers property 54 | Factory::registerResourceClass( 55 | 'frictionlessdata\\datapackage\\tests\\Mocks\\MyCustomResource' 56 | ); 57 | // a descriptor without the goGoPowerRangers property 58 | $descriptor = (object) ['name' => 'my-custom-resource', 'path' => ['tests/fixtures/foo.txt']]; 59 | // create a resource object based on the descriptor 60 | $resource = Factory::resource($descriptor); 61 | // got a normal resource 62 | $this->assertEquals( 63 | "frictionlessdata\\datapackage\\Resources\DefaultResource", 64 | get_class($resource) 65 | ); 66 | // add the goGoPowerRangers property 67 | $descriptor->goGoPowerRangers = true; 68 | $resource = Factory::resource($descriptor); 69 | // got the custom resource 70 | $this->assertEquals( 71 | 'frictionlessdata\\datapackage\\tests\\Mocks\\MyCustomResource', 72 | get_class($resource) 73 | ); 74 | // clear the registered classes and ensure it's cleared 75 | Factory::clearRegisteredResourceClasses(); 76 | $resource = Factory::resource($descriptor); 77 | $this->assertEquals( 78 | "frictionlessdata\\datapackage\\Resources\DefaultResource", 79 | get_class($resource) 80 | ); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /tests/Mocks/MockDatapackageValidator.php: -------------------------------------------------------------------------------- 1 | basePath); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/Mocks/MockDefaultDatapackage.php: -------------------------------------------------------------------------------- 1 | basePath); 12 | } 13 | 14 | protected function datapackageValidate() 15 | { 16 | return MockDatapackageValidator::validate($this->descriptor(), $this->basePath); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/Mocks/MockDefaultResource.php: -------------------------------------------------------------------------------- 1 | descriptor(), $this->basePath); 43 | } 44 | 45 | 46 | /** 47 | * @inheritDoc 48 | * @return bool 49 | */ 50 | public function isRemote() 51 | { 52 | $path = $this->path(); 53 | $path_to_check = is_array($path) && count($path) > 0 ? $path[0] : $path; 54 | return strpos($path_to_check, 'mock-http://') === 0 || parent::isRemote(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /tests/Mocks/MockFactory.php: -------------------------------------------------------------------------------- 1 | myCustomDatapackage); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/Mocks/MyCustomResource.php: -------------------------------------------------------------------------------- 1 | goGoPowerRangers); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/RegistryTest.php: -------------------------------------------------------------------------------- 1 | assertDatapackageClassProfile( 15 | 'frictionlessdata\\datapackage\\Datapackages\\DefaultDatapackage', 16 | 'data-package', 17 | (object) [] 18 | ); 19 | } 20 | 21 | public function testDatapackageWithDefaultProfile() 22 | { 23 | $this->assertDatapackageClassProfile( 24 | 'frictionlessdata\\datapackage\\Datapackages\\DefaultDatapackage', 25 | 'data-package', 26 | (object) ['profile' => 'default'] 27 | ); 28 | } 29 | 30 | public function testDatapackageWithDataPackageProfile() 31 | { 32 | $this->assertDatapackageClassProfile( 33 | 'frictionlessdata\\datapackage\\Datapackages\\DefaultDatapackage', 34 | 'data-package', 35 | (object) ['profile' => 'data-package'] 36 | ); 37 | } 38 | 39 | public function testDatapackageWithTabularDataPackageProfile() 40 | { 41 | $this->assertDatapackageClassProfile( 42 | 'frictionlessdata\\datapackage\\Datapackages\\TabularDatapackage', 43 | 'tabular-data-package', 44 | (object) ['profile' => 'tabular-data-package'] 45 | ); 46 | } 47 | 48 | public function testResourceWithoutProfile() 49 | { 50 | $this->assertResourceClassProfile( 51 | 'frictionlessdata\\datapackage\\Resources\\DefaultResource', 52 | 'data-resource', 53 | (object) [] 54 | ); 55 | } 56 | 57 | public function testResourceWithDefaultProfile() 58 | { 59 | $this->assertResourceClassProfile( 60 | 'frictionlessdata\\datapackage\\Resources\\DefaultResource', 61 | 'data-resource', 62 | (object) ['profile' => 'default'] 63 | ); 64 | } 65 | 66 | public function testResourceWithDataResourceProfile() 67 | { 68 | $this->assertResourceClassProfile( 69 | 'frictionlessdata\\datapackage\\Resources\\DefaultResource', 70 | 'data-resource', 71 | (object) ['profile' => 'data-resource'] 72 | ); 73 | } 74 | 75 | public function testResourceWithTabularDataResourceProfile() 76 | { 77 | $this->assertResourceClassProfile( 78 | 'frictionlessdata\\datapackage\\Resources\\TabularResource', 79 | 'tabular-data-resource', 80 | (object) ['profile' => 'tabular-data-resource'] 81 | ); 82 | } 83 | 84 | public function testCustomProfileFromJsonSchemaFile() 85 | { 86 | $descriptor = (object) [ 87 | 'name' => 'custom-datapackage', 88 | 'profile' => 'test-custom-profile.schema.json', 89 | 'resources' => [ 90 | (object) [ 91 | 'name' => 'custom-resource', 92 | 'profile' => 'test-custom-resource-profile.schema.json', 93 | 'path' => ['foo.txt'], 94 | ], 95 | ], 96 | ]; 97 | try { 98 | Factory::datapackage($descriptor, 'tests/fixtures'); 99 | $this->fail(); 100 | } catch (DatapackageValidationFailedException $e) { 101 | $this->assertEquals( 102 | 'Datapackage validation failed: [custom] The property custom is required', 103 | $e->getMessage() 104 | ); 105 | } 106 | $descriptor->resources[0]->custom = [1, 2, 3]; 107 | try { 108 | Factory::datapackage($descriptor, 'tests/fixtures'); 109 | $this->fail(); 110 | } catch (DatapackageValidationFailedException $e) { 111 | $this->assertEquals( 112 | 'Datapackage validation failed: [custom[0]] Integer value found, but a string is required, [custom[1]] Integer value found, but a string is required, [custom[2]] Integer value found, but a string is required', 113 | $e->getMessage() 114 | ); 115 | } 116 | $descriptor->resources[0]->custom = ['1', '2', '3']; 117 | $descriptor->foobar = ''; 118 | try { 119 | Factory::datapackage($descriptor, 'tests/fixtures'); 120 | $this->fail('should raise an exception because test-custom-profile requires foobar attribute (array of strings)'); 121 | } catch (DatapackageValidationFailedException $e) { 122 | $this->assertEquals( 123 | 'Datapackage validation failed: [foobar] String value found, but an array is required', 124 | $e->getMessage() 125 | ); 126 | } 127 | $descriptor->foobar = ['1', '2', '3']; 128 | $datapackage = Factory::datapackage($descriptor, 'tests/fixtures'); 129 | $this->assertEquals((object) [ 130 | 'name' => 'custom-datapackage', 131 | 'profile' => 'test-custom-profile.schema.json', 132 | 'foobar' => ['1', '2', '3'], 133 | 'resources' => [ 134 | (object) [ 135 | 'name' => 'custom-resource', 136 | 'profile' => 'test-custom-resource-profile.schema.json', 137 | 'path' => ['foo.txt'], 138 | 'custom' => ['1', '2', '3'], 139 | ], 140 | ], 141 | ], $datapackage->descriptor()); 142 | } 143 | 144 | // https://github.com/frictionlessdata/datapackage-php/issues/45 145 | // public function testCustomSchemaMustConformToDatapackageSchema() 146 | // { 147 | // $descriptor = (object) [ 148 | // 'profile' => 'http://json-schema.org/schema', 149 | // 'resources' => [], // this is allowed for json-schema.org/schema - but for datapackage it has minimum of 1 150 | // ]; 151 | // try { 152 | // Factory::datapackage($descriptor); 153 | // $this->fail(); 154 | // } catch (DatapackageValidationFailedException $e) { 155 | // $this->assertEquals( 156 | // 'Datapackage validation failed: [resources] There must be a minimum of 1 items in the array', 157 | // $e->getMessage() 158 | // ); 159 | // } 160 | // } 161 | 162 | public function testRegisteredSchema() 163 | { 164 | $descriptor = (object) [ 165 | 'name' => 'custom-datapackage', 166 | 'profile' => 'test-custom-profile', 167 | 'resources' => [ 168 | (object) [ 169 | 'name' => 'custom-resource', 170 | 'profile' => 'test-custom-resource-profile', 171 | 'path' => ['foo.txt'], 172 | 'custom' => ['1', '2', '3'], 173 | ], 174 | ], 175 | ]; 176 | Registry::registerSchema( 177 | 'test-custom-profile', 178 | 'tests/fixtures/test-custom-profile.schema.json' 179 | ); 180 | Registry::registerSchema( 181 | 'test-custom-resource-profile', 182 | 'tests/fixtures/test-custom-resource-profile.schema.json' 183 | ); 184 | $datapackage = Factory::datapackage($descriptor, 'tests/fixtures'); 185 | $this->assertInstanceOf( 186 | 'frictionlessdata\\datapackage\\Datapackages\\CustomDatapackage', 187 | $datapackage 188 | ); 189 | $this->assertInstanceOf( 190 | 'frictionlessdata\\datapackage\\Resources\\CustomResource', 191 | $datapackage->resource('custom-resource') 192 | ); 193 | Registry::clearRegisteredSchemas(); 194 | } 195 | 196 | protected function assertDatapackageClassProfile($expectedClass, $expectedProfile, $descriptor) 197 | { 198 | $this->assertEquals($expectedClass, Factory::getDatapackageClass($descriptor)); 199 | $this->assertEquals($expectedProfile, Registry::getDatapackageValidationProfile($descriptor)); 200 | } 201 | 202 | protected function assertResourceClassProfile($expectedClass, $expectedProfile, $descriptor) 203 | { 204 | $this->assertEquals($expectedClass, Factory::getResourceClass($descriptor)); 205 | $this->assertEquals($expectedProfile, Registry::getResourceValidationProfile($descriptor)); 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /tests/ResourceTest.php: -------------------------------------------------------------------------------- 1 | 'resource-name', 14 | 'path' => [ 15 | 'mock-http://foo.txt', // basePath will not be added to http source 16 | 'foo.txt', // basePath will be added here 17 | ], 18 | ], dirname(__FILE__).DIRECTORY_SEPARATOR.'fixtures'); 19 | $this->assertEquals(['foo', 'foo'], $resource->read()); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/autoload.php: -------------------------------------------------------------------------------- 1 | addPsr4('frictionlessdata\\datapackage\\tests\\', __DIR__, true); 7 | $classLoader->register(); 8 | -------------------------------------------------------------------------------- /tests/fixtures/bar.txt: -------------------------------------------------------------------------------- 1 | BAR! 2 | bar 3 | בר 4 | -------------------------------------------------------------------------------- /tests/fixtures/baz.txt: -------------------------------------------------------------------------------- 1 | בזבזבז 2 | זבזבזב -------------------------------------------------------------------------------- /tests/fixtures/committees/datapackage-lolsv.json: -------------------------------------------------------------------------------- 1 | { 2 | "bytes": 216380647, 3 | "count_of_rows": 2941876, 4 | "description": "Knesset committees and committee meetings data from Knesset dataservice API", 5 | "hash": "5e3087c6129d7b7da03f0e7ec93d1c48", 6 | "name": "knesset_data_committees", 7 | "resources": [ 8 | { 9 | "profile": "tabular-data-resource", 10 | "bytes": 160722, 11 | "count_of_rows": 702, 12 | "dialect": { 13 | "delimiter": "o", 14 | "quoteChar": "L", 15 | "doubleQuote": true 16 | }, 17 | "dpp:streaming": true, 18 | "encoding": "utf-8", 19 | "format": "csv", 20 | "hash": "23974f6e86b9996d4930753f073a67f1", 21 | "name": "kns_committee", 22 | "path": "kns_committee.lolsv", 23 | "schema": { 24 | "fields": [ 25 | { 26 | "description": "\u05e7\u05d5\u05d3 \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 27 | "name": "CommitteeID", 28 | "type": "integer" 29 | }, 30 | { 31 | "description": "\u05e9\u05dd \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 32 | "name": "Name", 33 | "type": "string" 34 | }, 35 | { 36 | "description": "\u05e7\u05d5\u05d3 \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 37 | "name": "CategoryID", 38 | "type": "integer" 39 | }, 40 | { 41 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 \u05d1\u05db\u05dc \u05db\u05e0\u05e1\u05ea, \u05db\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d5\u05ea \u05de\u05d5\u05e7\u05de\u05d5\u05ea \u05de\u05d7\u05d3\u05e9. \u05d4\u05e9\u05d3\u05d4 \u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05db\u05d5\u05dc\u05dc \u05d0\u05ea \u05e8\u05e9\u05d9\u05de\u05ea \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d5\u05ea \u05d4\u05e0\u05d5\u05e9\u05d0\u05d9\u05d5\u05ea \u05e9\u05d0\u05dc\u05d9\u05d4\u05df \u05de\u05e9\u05d5\u05d9\u05db\u05d5\u05ea \u05d4\u05d5\u05d5\u05e2\u05d3\u05d5\u05ea. \u05dc\u05de\u05e9\u05dc \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc \u05d5\u05e2\u05d3\u05ea \u05d4\u05e4\u05e0\u05d9\u05dd \u05d5\u05d4\u05d2\u05e0\u05ea \u05d4\u05e1\u05d1\u05d9\u05d1\u05d4 \u05d4\u05d9\u05d0 \"\u05e4\u05e0\u05d9\u05dd\" \u05d5\u05db\u05da \u05d4\u05d9\u05d4 \u05d2\u05dd \u05db\u05d0\u05e9\u05e8 \u05e9\u05dd \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 \u05d4\u05d9\u05d4 \u05d5\u05e2\u05d3\u05ea \u05d4\u05e4\u05e0\u05d9\u05dd \u05d5\u05d0\u05d9\u05db\u05d5\u05ea \u05d4\u05e1\u05d1\u05d9\u05d1\u05d4. \u05d2\u05dd \u05d5\u05e2\u05d3\u05d5\u05ea \u05d4\u05de\u05e9\u05e0\u05d4 \u05e9\u05dc \u05db\u05dc \u05d5\u05e2\u05d3\u05d4 \u05de\u05e9\u05d5\u05d9\u05db\u05d5\u05ea \u05dc\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc\u05d4. \u05de\u05d3\u05d5\u05d1\u05e8 \u05d1\u05e9\u05d9\u05d5\u05da \u05e0\u05d5\u05e9\u05d0\u05d9 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d5\u05ea.\n", 42 | "name": "CategoryDesc", 43 | "type": "string" 44 | }, 45 | { 46 | "description": "\u05de\u05e1\u05e4\u05e8 \u05d4\u05db\u05e0\u05e1\u05ea", 47 | "name": "KnessetNum", 48 | "type": "integer" 49 | }, 50 | { 51 | "description": "\u05e7\u05d5\u05d3 \u05e1\u05d5\u05d2 \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 52 | "name": "CommitteeTypeID", 53 | "type": "integer" 54 | }, 55 | { 56 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05e1\u05d5\u05d2 \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 (\u05e8\u05d0\u05e9\u05d9\u05ea, \u05de\u05d9\u05d5\u05d7\u05d3\u05ea, \u05de\u05e9\u05e0\u05d4, \u05de\u05e9\u05d5\u05ea\u05e4\u05ea, \u05d4\u05db\u05e0\u05e1\u05ea)\n", 57 | "name": "CommitteeTypeDesc", 58 | "type": "string" 59 | }, 60 | { 61 | "description": "\u05db\u05ea\u05d5\u05d1\u05ea \u05d4\u05d3\u05d5\u05d0\"\u05dc \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 62 | "name": "Email", 63 | "type": "string" 64 | }, 65 | { 66 | "description": "\u05ea\u05d0\u05e8\u05d9\u05da \u05d4\u05ea\u05d7\u05dc\u05d4", 67 | "format": "%Y-%m-%d %H:%M:%S", 68 | "name": "StartDate", 69 | "type": "datetime" 70 | }, 71 | { 72 | "description": "\u05ea\u05d0\u05e8\u05d9\u05da \u05e1\u05d9\u05d5\u05dd", 73 | "format": "%Y-%m-%d %H:%M:%S", 74 | "name": "FinishDate", 75 | "type": "datetime" 76 | }, 77 | { 78 | "description": "\u05e7\u05d5\u05d3 \u05e1\u05d5\u05d2 \u05de\u05e9\u05e0\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 79 | "name": "AdditionalTypeID", 80 | "type": "integer" 81 | }, 82 | { 83 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05e1\u05d5\u05d2 \u05de\u05e9\u05e0\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 (\u05e7\u05d1\u05d5\u05e2\u05d4, \u05de\u05d9\u05d5\u05d7\u05d3\u05ea, \u05d7\u05e7\u05d9\u05e8\u05d4)", 84 | "name": "AdditionalTypeDesc", 85 | "type": "string" 86 | }, 87 | { 88 | "description": "\u05e7\u05d5\u05d3 \u05d5\u05e2\u05d3\u05ea \u05d4\u05d0\u05dd (\u05e8\u05dc\u05d5\u05d5\u05e0\u05d8\u05d9 \u05e8\u05e7 \u05dc\u05d5\u05d5\u05e2\u05d3\u05ea \u05de\u05e9\u05e0\u05d4)", 89 | "name": "ParentCommitteeID", 90 | "type": "integer" 91 | }, 92 | { 93 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05d5\u05e2\u05d3\u05ea \u05d4\u05d0\u05dd", 94 | "name": "CommitteeParentName", 95 | "type": "string" 96 | }, 97 | { 98 | "description": "\u05d4\u05d0\u05dd \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 \u05e4\u05e2\u05d9\u05dc\u05d4?", 99 | "name": "IsCurrent", 100 | "type": "boolean" 101 | }, 102 | { 103 | "description": "\u05ea\u05d0\u05e8\u05d9\u05da \u05e2\u05d3\u05db\u05d5\u05df \u05d0\u05d7\u05e8\u05d5\u05df", 104 | "format": "%Y-%m-%d %H:%M:%S", 105 | "name": "LastUpdatedDate", 106 | "type": "datetime" 107 | } 108 | ], 109 | "primaryKey": [ 110 | "CommitteeID" 111 | ] 112 | } 113 | } 114 | ] 115 | } -------------------------------------------------------------------------------- /tests/fixtures/committees/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "bytes": 216380647, 3 | "count_of_rows": 2941876, 4 | "description": "Knesset committees and committee meetings data from Knesset dataservice API", 5 | "hash": "5e3087c6129d7b7da03f0e7ec93d1c48", 6 | "name": "knesset_data_committees", 7 | "resources": [ 8 | { 9 | "bytes": 160722, 10 | "count_of_rows": 702, 11 | "dialect": { 12 | "delimiter": ",", 13 | "doubleQuote": true, 14 | "lineTerminator": "\r\n", 15 | "quoteChar": "\"", 16 | "skipInitialSpace": false 17 | }, 18 | "dpp:streaming": true, 19 | "encoding": "utf-8", 20 | "format": "csv", 21 | "hash": "23974f6e86b9996d4930753f073a67f1", 22 | "name": "kns_committee", 23 | "path": "kns_committee.csv", 24 | "schema": { 25 | "fields": [ 26 | { 27 | "description": "\u05e7\u05d5\u05d3 \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 28 | "name": "CommitteeID", 29 | "type": "integer" 30 | }, 31 | { 32 | "description": "\u05e9\u05dd \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 33 | "name": "Name", 34 | "type": "string" 35 | }, 36 | { 37 | "description": "\u05e7\u05d5\u05d3 \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 38 | "name": "CategoryID", 39 | "type": "integer" 40 | }, 41 | { 42 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 \u05d1\u05db\u05dc \u05db\u05e0\u05e1\u05ea, \u05db\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d5\u05ea \u05de\u05d5\u05e7\u05de\u05d5\u05ea \u05de\u05d7\u05d3\u05e9. \u05d4\u05e9\u05d3\u05d4 \u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05db\u05d5\u05dc\u05dc \u05d0\u05ea \u05e8\u05e9\u05d9\u05de\u05ea \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d5\u05ea \u05d4\u05e0\u05d5\u05e9\u05d0\u05d9\u05d5\u05ea \u05e9\u05d0\u05dc\u05d9\u05d4\u05df \u05de\u05e9\u05d5\u05d9\u05db\u05d5\u05ea \u05d4\u05d5\u05d5\u05e2\u05d3\u05d5\u05ea. \u05dc\u05de\u05e9\u05dc \u05d4\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc \u05d5\u05e2\u05d3\u05ea \u05d4\u05e4\u05e0\u05d9\u05dd \u05d5\u05d4\u05d2\u05e0\u05ea \u05d4\u05e1\u05d1\u05d9\u05d1\u05d4 \u05d4\u05d9\u05d0 \"\u05e4\u05e0\u05d9\u05dd\" \u05d5\u05db\u05da \u05d4\u05d9\u05d4 \u05d2\u05dd \u05db\u05d0\u05e9\u05e8 \u05e9\u05dd \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 \u05d4\u05d9\u05d4 \u05d5\u05e2\u05d3\u05ea \u05d4\u05e4\u05e0\u05d9\u05dd \u05d5\u05d0\u05d9\u05db\u05d5\u05ea \u05d4\u05e1\u05d1\u05d9\u05d1\u05d4. \u05d2\u05dd \u05d5\u05e2\u05d3\u05d5\u05ea \u05d4\u05de\u05e9\u05e0\u05d4 \u05e9\u05dc \u05db\u05dc \u05d5\u05e2\u05d3\u05d4 \u05de\u05e9\u05d5\u05d9\u05db\u05d5\u05ea \u05dc\u05e7\u05d8\u05d2\u05d5\u05e8\u05d9\u05d4 \u05e9\u05dc\u05d4. \u05de\u05d3\u05d5\u05d1\u05e8 \u05d1\u05e9\u05d9\u05d5\u05da \u05e0\u05d5\u05e9\u05d0\u05d9 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d5\u05ea.\n", 43 | "name": "CategoryDesc", 44 | "type": "string" 45 | }, 46 | { 47 | "description": "\u05de\u05e1\u05e4\u05e8 \u05d4\u05db\u05e0\u05e1\u05ea", 48 | "name": "KnessetNum", 49 | "type": "integer" 50 | }, 51 | { 52 | "description": "\u05e7\u05d5\u05d3 \u05e1\u05d5\u05d2 \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 53 | "name": "CommitteeTypeID", 54 | "type": "integer" 55 | }, 56 | { 57 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05e1\u05d5\u05d2 \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 (\u05e8\u05d0\u05e9\u05d9\u05ea, \u05de\u05d9\u05d5\u05d7\u05d3\u05ea, \u05de\u05e9\u05e0\u05d4, \u05de\u05e9\u05d5\u05ea\u05e4\u05ea, \u05d4\u05db\u05e0\u05e1\u05ea)\n", 58 | "name": "CommitteeTypeDesc", 59 | "type": "string" 60 | }, 61 | { 62 | "description": "\u05db\u05ea\u05d5\u05d1\u05ea \u05d4\u05d3\u05d5\u05d0\"\u05dc \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 63 | "name": "Email", 64 | "type": "string" 65 | }, 66 | { 67 | "description": "\u05ea\u05d0\u05e8\u05d9\u05da \u05d4\u05ea\u05d7\u05dc\u05d4", 68 | "format": "%Y-%m-%d %H:%M:%S", 69 | "name": "StartDate", 70 | "type": "datetime" 71 | }, 72 | { 73 | "description": "\u05ea\u05d0\u05e8\u05d9\u05da \u05e1\u05d9\u05d5\u05dd", 74 | "format": "%Y-%m-%d %H:%M:%S", 75 | "name": "FinishDate", 76 | "type": "datetime" 77 | }, 78 | { 79 | "description": "\u05e7\u05d5\u05d3 \u05e1\u05d5\u05d2 \u05de\u05e9\u05e0\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4", 80 | "name": "AdditionalTypeID", 81 | "type": "integer" 82 | }, 83 | { 84 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05e1\u05d5\u05d2 \u05de\u05e9\u05e0\u05d4 \u05e9\u05dc \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 (\u05e7\u05d1\u05d5\u05e2\u05d4, \u05de\u05d9\u05d5\u05d7\u05d3\u05ea, \u05d7\u05e7\u05d9\u05e8\u05d4)", 85 | "name": "AdditionalTypeDesc", 86 | "type": "string" 87 | }, 88 | { 89 | "description": "\u05e7\u05d5\u05d3 \u05d5\u05e2\u05d3\u05ea \u05d4\u05d0\u05dd (\u05e8\u05dc\u05d5\u05d5\u05e0\u05d8\u05d9 \u05e8\u05e7 \u05dc\u05d5\u05d5\u05e2\u05d3\u05ea \u05de\u05e9\u05e0\u05d4)", 90 | "name": "ParentCommitteeID", 91 | "type": "integer" 92 | }, 93 | { 94 | "description": "\u05ea\u05d9\u05d0\u05d5\u05e8 \u05d5\u05e2\u05d3\u05ea \u05d4\u05d0\u05dd", 95 | "name": "CommitteeParentName", 96 | "type": "string" 97 | }, 98 | { 99 | "description": "\u05d4\u05d0\u05dd \u05d4\u05d5\u05d5\u05e2\u05d3\u05d4 \u05e4\u05e2\u05d9\u05dc\u05d4?", 100 | "name": "IsCurrent", 101 | "type": "boolean" 102 | }, 103 | { 104 | "description": "\u05ea\u05d0\u05e8\u05d9\u05da \u05e2\u05d3\u05db\u05d5\u05df \u05d0\u05d7\u05e8\u05d5\u05df", 105 | "format": "%Y-%m-%d %H:%M:%S", 106 | "name": "LastUpdatedDate", 107 | "type": "datetime" 108 | } 109 | ], 110 | "primaryKey": [ 111 | "CommitteeID" 112 | ] 113 | } 114 | } 115 | ] 116 | } -------------------------------------------------------------------------------- /tests/fixtures/committees/kns_committee.lolsv: -------------------------------------------------------------------------------- 1 | LCommitteeIDLoLNameLoLCategoryIDLoLCategoryDescLoLKnessetNumLoLCommitteeTypeIDLoLCommitteeTypeDescLoLEmailLoLStartDateLoLFinishDateLoLAdditionalTypeIDLoLAdditionalTypeDescLoLParentCommitteeIDLoLCommitteeParentNameLoLIsCurrentLoLLLastUpdatedDateL 2 | L97LoL"ה""ח המדיניות הכלכלית לשנת הכספים 2004"LoLLoLLoL16LoL73LoLועדה משותפתLoLLoL2004-08-12 00:00:00LoLLoLLoLLoLLoLLoLTrueLoL2015-03-20 12:02:57L 3 | L314LoLהמיוחדת לענין לקחי אסון גשר המכביהLoLLoLLoL14LoL72LoLועדה מיוחדתLoLLoL1988-10-19 00:00:00LoLLoL992LoLמיוחדתLoLLoLLoLTrueLoL2015-03-20 12:02:57L 4 | -------------------------------------------------------------------------------- /tests/fixtures/datahub-country-list/data.csv: -------------------------------------------------------------------------------- 1 | Name,Code 2 | Afghanistan,AF 3 | Åland Islands,AX 4 | Albania,AL 5 | Algeria,DZ 6 | American Samoa,AS 7 | Andorra,AD 8 | Angola,AO 9 | Anguilla,AI 10 | Antarctica,AQ 11 | Antigua and Barbuda,AG 12 | Argentina,AR 13 | Armenia,AM 14 | Aruba,AW 15 | Australia,AU 16 | Austria,AT 17 | Azerbaijan,AZ 18 | Bahamas,BS 19 | Bahrain,BH 20 | Bangladesh,BD 21 | Barbados,BB 22 | Belarus,BY 23 | Belgium,BE 24 | Belize,BZ 25 | Benin,BJ 26 | Bermuda,BM 27 | Bhutan,BT 28 | "Bolivia, Plurinational State of",BO 29 | "Bonaire, Sint Eustatius and Saba",BQ 30 | Bosnia and Herzegovina,BA 31 | Botswana,BW 32 | Bouvet Island,BV 33 | Brazil,BR 34 | British Indian Ocean Territory,IO 35 | Brunei Darussalam,BN 36 | Bulgaria,BG 37 | Burkina Faso,BF 38 | Burundi,BI 39 | Cambodia,KH 40 | Cameroon,CM 41 | Canada,CA 42 | Cape Verde,CV 43 | Cayman Islands,KY 44 | Central African Republic,CF 45 | Chad,TD 46 | Chile,CL 47 | China,CN 48 | Christmas Island,CX 49 | Cocos (Keeling) Islands,CC 50 | Colombia,CO 51 | Comoros,KM 52 | Congo,CG 53 | "Congo, the Democratic Republic of the",CD 54 | Cook Islands,CK 55 | Costa Rica,CR 56 | Côte d'Ivoire,CI 57 | Croatia,HR 58 | Cuba,CU 59 | Curaçao,CW 60 | Cyprus,CY 61 | Czech Republic,CZ 62 | Denmark,DK 63 | Djibouti,DJ 64 | Dominica,DM 65 | Dominican Republic,DO 66 | Ecuador,EC 67 | Egypt,EG 68 | El Salvador,SV 69 | Equatorial Guinea,GQ 70 | Eritrea,ER 71 | Estonia,EE 72 | Ethiopia,ET 73 | Falkland Islands (Malvinas),FK 74 | Faroe Islands,FO 75 | Fiji,FJ 76 | Finland,FI 77 | France,FR 78 | French Guiana,GF 79 | French Polynesia,PF 80 | French Southern Territories,TF 81 | Gabon,GA 82 | Gambia,GM 83 | Georgia,GE 84 | Germany,DE 85 | Ghana,GH 86 | Gibraltar,GI 87 | Greece,GR 88 | Greenland,GL 89 | Grenada,GD 90 | Guadeloupe,GP 91 | Guam,GU 92 | Guatemala,GT 93 | Guernsey,GG 94 | Guinea,GN 95 | Guinea-Bissau,GW 96 | Guyana,GY 97 | Haiti,HT 98 | Heard Island and McDonald Islands,HM 99 | Holy See (Vatican City State),VA 100 | Honduras,HN 101 | Hong Kong,HK 102 | Hungary,HU 103 | Iceland,IS 104 | India,IN 105 | Indonesia,ID 106 | "Iran, Islamic Republic of",IR 107 | Iraq,IQ 108 | Ireland,IE 109 | Isle of Man,IM 110 | Israel,IL 111 | Italy,IT 112 | Jamaica,JM 113 | Japan,JP 114 | Jersey,JE 115 | Jordan,JO 116 | Kazakhstan,KZ 117 | Kenya,KE 118 | Kiribati,KI 119 | "Korea, Democratic People's Republic of",KP 120 | "Korea, Republic of",KR 121 | Kuwait,KW 122 | Kyrgyzstan,KG 123 | Lao People's Democratic Republic,LA 124 | Latvia,LV 125 | Lebanon,LB 126 | Lesotho,LS 127 | Liberia,LR 128 | Libya,LY 129 | Liechtenstein,LI 130 | Lithuania,LT 131 | Luxembourg,LU 132 | Macao,MO 133 | "Macedonia, the Former Yugoslav Republic of",MK 134 | Madagascar,MG 135 | Malawi,MW 136 | Malaysia,MY 137 | Maldives,MV 138 | Mali,ML 139 | Malta,MT 140 | Marshall Islands,MH 141 | Martinique,MQ 142 | Mauritania,MR 143 | Mauritius,MU 144 | Mayotte,YT 145 | Mexico,MX 146 | "Micronesia, Federated States of",FM 147 | "Moldova, Republic of",MD 148 | Monaco,MC 149 | Mongolia,MN 150 | Montenegro,ME 151 | Montserrat,MS 152 | Morocco,MA 153 | Mozambique,MZ 154 | Myanmar,MM 155 | Namibia,NA 156 | Nauru,NR 157 | Nepal,NP 158 | Netherlands,NL 159 | New Caledonia,NC 160 | New Zealand,NZ 161 | Nicaragua,NI 162 | Niger,NE 163 | Nigeria,NG 164 | Niue,NU 165 | Norfolk Island,NF 166 | Northern Mariana Islands,MP 167 | Norway,NO 168 | Oman,OM 169 | Pakistan,PK 170 | Palau,PW 171 | "Palestine, State of",PS 172 | Panama,PA 173 | Papua New Guinea,PG 174 | Paraguay,PY 175 | Peru,PE 176 | Philippines,PH 177 | Pitcairn,PN 178 | Poland,PL 179 | Portugal,PT 180 | Puerto Rico,PR 181 | Qatar,QA 182 | Réunion,RE 183 | Romania,RO 184 | Russian Federation,RU 185 | Rwanda,RW 186 | Saint Barthélemy,BL 187 | "Saint Helena, Ascension and Tristan da Cunha",SH 188 | Saint Kitts and Nevis,KN 189 | Saint Lucia,LC 190 | Saint Martin (French part),MF 191 | Saint Pierre and Miquelon,PM 192 | Saint Vincent and the Grenadines,VC 193 | Samoa,WS 194 | San Marino,SM 195 | Sao Tome and Principe,ST 196 | Saudi Arabia,SA 197 | Senegal,SN 198 | Serbia,RS 199 | Seychelles,SC 200 | Sierra Leone,SL 201 | Singapore,SG 202 | Sint Maarten (Dutch part),SX 203 | Slovakia,SK 204 | Slovenia,SI 205 | Solomon Islands,SB 206 | Somalia,SO 207 | South Africa,ZA 208 | South Georgia and the South Sandwich Islands,GS 209 | South Sudan,SS 210 | Spain,ES 211 | Sri Lanka,LK 212 | Sudan,SD 213 | Suriname,SR 214 | Svalbard and Jan Mayen,SJ 215 | Swaziland,SZ 216 | Sweden,SE 217 | Switzerland,CH 218 | Syrian Arab Republic,SY 219 | "Taiwan, Province of China",TW 220 | Tajikistan,TJ 221 | "Tanzania, United Republic of",TZ 222 | Thailand,TH 223 | Timor-Leste,TL 224 | Togo,TG 225 | Tokelau,TK 226 | Tonga,TO 227 | Trinidad and Tobago,TT 228 | Tunisia,TN 229 | Turkey,TR 230 | Turkmenistan,TM 231 | Turks and Caicos Islands,TC 232 | Tuvalu,TV 233 | Uganda,UG 234 | Ukraine,UA 235 | United Arab Emirates,AE 236 | United Kingdom,GB 237 | United States,US 238 | United States Minor Outlying Islands,UM 239 | Uruguay,UY 240 | Uzbekistan,UZ 241 | Vanuatu,VU 242 | "Venezuela, Bolivarian Republic of",VE 243 | Viet Nam,VN 244 | "Virgin Islands, British",VG 245 | "Virgin Islands, U.S.",VI 246 | Wallis and Futuna,WF 247 | Western Sahara,EH 248 | Yemen,YE 249 | Zambia,ZM 250 | Zimbabwe,ZW 251 | -------------------------------------------------------------------------------- /tests/fixtures/datahub-country-list/data_csv.csv: -------------------------------------------------------------------------------- 1 | Name,Code 2 | Afghanistan,AF 3 | Åland Islands,AX 4 | Albania,AL 5 | Algeria,DZ 6 | American Samoa,AS 7 | Andorra,AD 8 | Angola,AO 9 | Anguilla,AI 10 | Antarctica,AQ 11 | Antigua and Barbuda,AG 12 | Argentina,AR 13 | Armenia,AM 14 | Aruba,AW 15 | Australia,AU 16 | Austria,AT 17 | Azerbaijan,AZ 18 | Bahamas,BS 19 | Bahrain,BH 20 | Bangladesh,BD 21 | Barbados,BB 22 | Belarus,BY 23 | Belgium,BE 24 | Belize,BZ 25 | Benin,BJ 26 | Bermuda,BM 27 | Bhutan,BT 28 | "Bolivia, Plurinational State of",BO 29 | "Bonaire, Sint Eustatius and Saba",BQ 30 | Bosnia and Herzegovina,BA 31 | Botswana,BW 32 | Bouvet Island,BV 33 | Brazil,BR 34 | British Indian Ocean Territory,IO 35 | Brunei Darussalam,BN 36 | Bulgaria,BG 37 | Burkina Faso,BF 38 | Burundi,BI 39 | Cambodia,KH 40 | Cameroon,CM 41 | Canada,CA 42 | Cape Verde,CV 43 | Cayman Islands,KY 44 | Central African Republic,CF 45 | Chad,TD 46 | Chile,CL 47 | China,CN 48 | Christmas Island,CX 49 | Cocos (Keeling) Islands,CC 50 | Colombia,CO 51 | Comoros,KM 52 | Congo,CG 53 | "Congo, the Democratic Republic of the",CD 54 | Cook Islands,CK 55 | Costa Rica,CR 56 | Côte d'Ivoire,CI 57 | Croatia,HR 58 | Cuba,CU 59 | Curaçao,CW 60 | Cyprus,CY 61 | Czech Republic,CZ 62 | Denmark,DK 63 | Djibouti,DJ 64 | Dominica,DM 65 | Dominican Republic,DO 66 | Ecuador,EC 67 | Egypt,EG 68 | El Salvador,SV 69 | Equatorial Guinea,GQ 70 | Eritrea,ER 71 | Estonia,EE 72 | Ethiopia,ET 73 | Falkland Islands (Malvinas),FK 74 | Faroe Islands,FO 75 | Fiji,FJ 76 | Finland,FI 77 | France,FR 78 | French Guiana,GF 79 | French Polynesia,PF 80 | French Southern Territories,TF 81 | Gabon,GA 82 | Gambia,GM 83 | Georgia,GE 84 | Germany,DE 85 | Ghana,GH 86 | Gibraltar,GI 87 | Greece,GR 88 | Greenland,GL 89 | Grenada,GD 90 | Guadeloupe,GP 91 | Guam,GU 92 | Guatemala,GT 93 | Guernsey,GG 94 | Guinea,GN 95 | Guinea-Bissau,GW 96 | Guyana,GY 97 | Haiti,HT 98 | Heard Island and McDonald Islands,HM 99 | Holy See (Vatican City State),VA 100 | Honduras,HN 101 | Hong Kong,HK 102 | Hungary,HU 103 | Iceland,IS 104 | India,IN 105 | Indonesia,ID 106 | "Iran, Islamic Republic of",IR 107 | Iraq,IQ 108 | Ireland,IE 109 | Isle of Man,IM 110 | Israel,IL 111 | Italy,IT 112 | Jamaica,JM 113 | Japan,JP 114 | Jersey,JE 115 | Jordan,JO 116 | Kazakhstan,KZ 117 | Kenya,KE 118 | Kiribati,KI 119 | "Korea, Democratic People's Republic of",KP 120 | "Korea, Republic of",KR 121 | Kuwait,KW 122 | Kyrgyzstan,KG 123 | Lao People's Democratic Republic,LA 124 | Latvia,LV 125 | Lebanon,LB 126 | Lesotho,LS 127 | Liberia,LR 128 | Libya,LY 129 | Liechtenstein,LI 130 | Lithuania,LT 131 | Luxembourg,LU 132 | Macao,MO 133 | "Macedonia, the Former Yugoslav Republic of",MK 134 | Madagascar,MG 135 | Malawi,MW 136 | Malaysia,MY 137 | Maldives,MV 138 | Mali,ML 139 | Malta,MT 140 | Marshall Islands,MH 141 | Martinique,MQ 142 | Mauritania,MR 143 | Mauritius,MU 144 | Mayotte,YT 145 | Mexico,MX 146 | "Micronesia, Federated States of",FM 147 | "Moldova, Republic of",MD 148 | Monaco,MC 149 | Mongolia,MN 150 | Montenegro,ME 151 | Montserrat,MS 152 | Morocco,MA 153 | Mozambique,MZ 154 | Myanmar,MM 155 | Namibia,NA 156 | Nauru,NR 157 | Nepal,NP 158 | Netherlands,NL 159 | New Caledonia,NC 160 | New Zealand,NZ 161 | Nicaragua,NI 162 | Niger,NE 163 | Nigeria,NG 164 | Niue,NU 165 | Norfolk Island,NF 166 | Northern Mariana Islands,MP 167 | Norway,NO 168 | Oman,OM 169 | Pakistan,PK 170 | Palau,PW 171 | "Palestine, State of",PS 172 | Panama,PA 173 | Papua New Guinea,PG 174 | Paraguay,PY 175 | Peru,PE 176 | Philippines,PH 177 | Pitcairn,PN 178 | Poland,PL 179 | Portugal,PT 180 | Puerto Rico,PR 181 | Qatar,QA 182 | Réunion,RE 183 | Romania,RO 184 | Russian Federation,RU 185 | Rwanda,RW 186 | Saint Barthélemy,BL 187 | "Saint Helena, Ascension and Tristan da Cunha",SH 188 | Saint Kitts and Nevis,KN 189 | Saint Lucia,LC 190 | Saint Martin (French part),MF 191 | Saint Pierre and Miquelon,PM 192 | Saint Vincent and the Grenadines,VC 193 | Samoa,WS 194 | San Marino,SM 195 | Sao Tome and Principe,ST 196 | Saudi Arabia,SA 197 | Senegal,SN 198 | Serbia,RS 199 | Seychelles,SC 200 | Sierra Leone,SL 201 | Singapore,SG 202 | Sint Maarten (Dutch part),SX 203 | Slovakia,SK 204 | Slovenia,SI 205 | Solomon Islands,SB 206 | Somalia,SO 207 | South Africa,ZA 208 | South Georgia and the South Sandwich Islands,GS 209 | South Sudan,SS 210 | Spain,ES 211 | Sri Lanka,LK 212 | Sudan,SD 213 | Suriname,SR 214 | Svalbard and Jan Mayen,SJ 215 | Swaziland,SZ 216 | Sweden,SE 217 | Switzerland,CH 218 | Syrian Arab Republic,SY 219 | "Taiwan, Province of China",TW 220 | Tajikistan,TJ 221 | "Tanzania, United Republic of",TZ 222 | Thailand,TH 223 | Timor-Leste,TL 224 | Togo,TG 225 | Tokelau,TK 226 | Tonga,TO 227 | Trinidad and Tobago,TT 228 | Tunisia,TN 229 | Turkey,TR 230 | Turkmenistan,TM 231 | Turks and Caicos Islands,TC 232 | Tuvalu,TV 233 | Uganda,UG 234 | Ukraine,UA 235 | United Arab Emirates,AE 236 | United Kingdom,GB 237 | United States,US 238 | United States Minor Outlying Islands,UM 239 | Uruguay,UY 240 | Uzbekistan,UZ 241 | Vanuatu,VU 242 | "Venezuela, Bolivarian Republic of",VE 243 | Viet Nam,VN 244 | "Virgin Islands, British",VG 245 | "Virgin Islands, U.S.",VI 246 | Wallis and Futuna,WF 247 | Western Sahara,EH 248 | Yemen,YE 249 | Zambia,ZM 250 | Zimbabwe,ZW 251 | -------------------------------------------------------------------------------- /tests/fixtures/datahub-country-list/data_json.json: -------------------------------------------------------------------------------- 1 | [{"Code": "AF", "Name": "Afghanistan"},{"Code": "AX", "Name": "\u00c5land Islands"},{"Code": "AL", "Name": "Albania"},{"Code": "DZ", "Name": "Algeria"},{"Code": "AS", "Name": "American Samoa"},{"Code": "AD", "Name": "Andorra"},{"Code": "AO", "Name": "Angola"},{"Code": "AI", "Name": "Anguilla"},{"Code": "AQ", "Name": "Antarctica"},{"Code": "AG", "Name": "Antigua and Barbuda"},{"Code": "AR", "Name": "Argentina"},{"Code": "AM", "Name": "Armenia"},{"Code": "AW", "Name": "Aruba"},{"Code": "AU", "Name": "Australia"},{"Code": "AT", "Name": "Austria"},{"Code": "AZ", "Name": "Azerbaijan"},{"Code": "BS", "Name": "Bahamas"},{"Code": "BH", "Name": "Bahrain"},{"Code": "BD", "Name": "Bangladesh"},{"Code": "BB", "Name": "Barbados"},{"Code": "BY", "Name": "Belarus"},{"Code": "BE", "Name": "Belgium"},{"Code": "BZ", "Name": "Belize"},{"Code": "BJ", "Name": "Benin"},{"Code": "BM", "Name": "Bermuda"},{"Code": "BT", "Name": "Bhutan"},{"Code": "BO", "Name": "Bolivia, Plurinational State of"},{"Code": "BQ", "Name": "Bonaire, Sint Eustatius and Saba"},{"Code": "BA", "Name": "Bosnia and Herzegovina"},{"Code": "BW", "Name": "Botswana"},{"Code": "BV", "Name": "Bouvet Island"},{"Code": "BR", "Name": "Brazil"},{"Code": "IO", "Name": "British Indian Ocean Territory"},{"Code": "BN", "Name": "Brunei Darussalam"},{"Code": "BG", "Name": "Bulgaria"},{"Code": "BF", "Name": "Burkina Faso"},{"Code": "BI", "Name": "Burundi"},{"Code": "KH", "Name": "Cambodia"},{"Code": "CM", "Name": "Cameroon"},{"Code": "CA", "Name": "Canada"},{"Code": "CV", "Name": "Cape Verde"},{"Code": "KY", "Name": "Cayman Islands"},{"Code": "CF", "Name": "Central African Republic"},{"Code": "TD", "Name": "Chad"},{"Code": "CL", "Name": "Chile"},{"Code": "CN", "Name": "China"},{"Code": "CX", "Name": "Christmas Island"},{"Code": "CC", "Name": "Cocos (Keeling) Islands"},{"Code": "CO", "Name": "Colombia"},{"Code": "KM", "Name": "Comoros"},{"Code": "CG", "Name": "Congo"},{"Code": "CD", "Name": "Congo, the Democratic Republic of the"},{"Code": "CK", "Name": "Cook Islands"},{"Code": "CR", "Name": "Costa Rica"},{"Code": "CI", "Name": "C\u00f4te d'Ivoire"},{"Code": "HR", "Name": "Croatia"},{"Code": "CU", "Name": "Cuba"},{"Code": "CW", "Name": "Cura\u00e7ao"},{"Code": "CY", "Name": "Cyprus"},{"Code": "CZ", "Name": "Czech Republic"},{"Code": "DK", "Name": "Denmark"},{"Code": "DJ", "Name": "Djibouti"},{"Code": "DM", "Name": "Dominica"},{"Code": "DO", "Name": "Dominican Republic"},{"Code": "EC", "Name": "Ecuador"},{"Code": "EG", "Name": "Egypt"},{"Code": "SV", "Name": "El Salvador"},{"Code": "GQ", "Name": "Equatorial Guinea"},{"Code": "ER", "Name": "Eritrea"},{"Code": "EE", "Name": "Estonia"},{"Code": "ET", "Name": "Ethiopia"},{"Code": "FK", "Name": "Falkland Islands (Malvinas)"},{"Code": "FO", "Name": "Faroe Islands"},{"Code": "FJ", "Name": "Fiji"},{"Code": "FI", "Name": "Finland"},{"Code": "FR", "Name": "France"},{"Code": "GF", "Name": "French Guiana"},{"Code": "PF", "Name": "French Polynesia"},{"Code": "TF", "Name": "French Southern Territories"},{"Code": "GA", "Name": "Gabon"},{"Code": "GM", "Name": "Gambia"},{"Code": "GE", "Name": "Georgia"},{"Code": "DE", "Name": "Germany"},{"Code": "GH", "Name": "Ghana"},{"Code": "GI", "Name": "Gibraltar"},{"Code": "GR", "Name": "Greece"},{"Code": "GL", "Name": "Greenland"},{"Code": "GD", "Name": "Grenada"},{"Code": "GP", "Name": "Guadeloupe"},{"Code": "GU", "Name": "Guam"},{"Code": "GT", "Name": "Guatemala"},{"Code": "GG", "Name": "Guernsey"},{"Code": "GN", "Name": "Guinea"},{"Code": "GW", "Name": "Guinea-Bissau"},{"Code": "GY", "Name": "Guyana"},{"Code": "HT", "Name": "Haiti"},{"Code": "HM", "Name": "Heard Island and McDonald Islands"},{"Code": "VA", "Name": "Holy See (Vatican City State)"},{"Code": "HN", "Name": "Honduras"},{"Code": "HK", "Name": "Hong Kong"},{"Code": "HU", "Name": "Hungary"},{"Code": "IS", "Name": "Iceland"},{"Code": "IN", "Name": "India"},{"Code": "ID", "Name": "Indonesia"},{"Code": "IR", "Name": "Iran, Islamic Republic of"},{"Code": "IQ", "Name": "Iraq"},{"Code": "IE", "Name": "Ireland"},{"Code": "IM", "Name": "Isle of Man"},{"Code": "IL", "Name": "Israel"},{"Code": "IT", "Name": "Italy"},{"Code": "JM", "Name": "Jamaica"},{"Code": "JP", "Name": "Japan"},{"Code": "JE", "Name": "Jersey"},{"Code": "JO", "Name": "Jordan"},{"Code": "KZ", "Name": "Kazakhstan"},{"Code": "KE", "Name": "Kenya"},{"Code": "KI", "Name": "Kiribati"},{"Code": "KP", "Name": "Korea, Democratic People's Republic of"},{"Code": "KR", "Name": "Korea, Republic of"},{"Code": "KW", "Name": "Kuwait"},{"Code": "KG", "Name": "Kyrgyzstan"},{"Code": "LA", "Name": "Lao People's Democratic Republic"},{"Code": "LV", "Name": "Latvia"},{"Code": "LB", "Name": "Lebanon"},{"Code": "LS", "Name": "Lesotho"},{"Code": "LR", "Name": "Liberia"},{"Code": "LY", "Name": "Libya"},{"Code": "LI", "Name": "Liechtenstein"},{"Code": "LT", "Name": "Lithuania"},{"Code": "LU", "Name": "Luxembourg"},{"Code": "MO", "Name": "Macao"},{"Code": "MK", "Name": "Macedonia, the Former Yugoslav Republic of"},{"Code": "MG", "Name": "Madagascar"},{"Code": "MW", "Name": "Malawi"},{"Code": "MY", "Name": "Malaysia"},{"Code": "MV", "Name": "Maldives"},{"Code": "ML", "Name": "Mali"},{"Code": "MT", "Name": "Malta"},{"Code": "MH", "Name": "Marshall Islands"},{"Code": "MQ", "Name": "Martinique"},{"Code": "MR", "Name": "Mauritania"},{"Code": "MU", "Name": "Mauritius"},{"Code": "YT", "Name": "Mayotte"},{"Code": "MX", "Name": "Mexico"},{"Code": "FM", "Name": "Micronesia, Federated States of"},{"Code": "MD", "Name": "Moldova, Republic of"},{"Code": "MC", "Name": "Monaco"},{"Code": "MN", "Name": "Mongolia"},{"Code": "ME", "Name": "Montenegro"},{"Code": "MS", "Name": "Montserrat"},{"Code": "MA", "Name": "Morocco"},{"Code": "MZ", "Name": "Mozambique"},{"Code": "MM", "Name": "Myanmar"},{"Code": "NA", "Name": "Namibia"},{"Code": "NR", "Name": "Nauru"},{"Code": "NP", "Name": "Nepal"},{"Code": "NL", "Name": "Netherlands"},{"Code": "NC", "Name": "New Caledonia"},{"Code": "NZ", "Name": "New Zealand"},{"Code": "NI", "Name": "Nicaragua"},{"Code": "NE", "Name": "Niger"},{"Code": "NG", "Name": "Nigeria"},{"Code": "NU", "Name": "Niue"},{"Code": "NF", "Name": "Norfolk Island"},{"Code": "MP", "Name": "Northern Mariana Islands"},{"Code": "NO", "Name": "Norway"},{"Code": "OM", "Name": "Oman"},{"Code": "PK", "Name": "Pakistan"},{"Code": "PW", "Name": "Palau"},{"Code": "PS", "Name": "Palestine, State of"},{"Code": "PA", "Name": "Panama"},{"Code": "PG", "Name": "Papua New Guinea"},{"Code": "PY", "Name": "Paraguay"},{"Code": "PE", "Name": "Peru"},{"Code": "PH", "Name": "Philippines"},{"Code": "PN", "Name": "Pitcairn"},{"Code": "PL", "Name": "Poland"},{"Code": "PT", "Name": "Portugal"},{"Code": "PR", "Name": "Puerto Rico"},{"Code": "QA", "Name": "Qatar"},{"Code": "RE", "Name": "R\u00e9union"},{"Code": "RO", "Name": "Romania"},{"Code": "RU", "Name": "Russian Federation"},{"Code": "RW", "Name": "Rwanda"},{"Code": "BL", "Name": "Saint Barth\u00e9lemy"},{"Code": "SH", "Name": "Saint Helena, Ascension and Tristan da Cunha"},{"Code": "KN", "Name": "Saint Kitts and Nevis"},{"Code": "LC", "Name": "Saint Lucia"},{"Code": "MF", "Name": "Saint Martin (French part)"},{"Code": "PM", "Name": "Saint Pierre and Miquelon"},{"Code": "VC", "Name": "Saint Vincent and the Grenadines"},{"Code": "WS", "Name": "Samoa"},{"Code": "SM", "Name": "San Marino"},{"Code": "ST", "Name": "Sao Tome and Principe"},{"Code": "SA", "Name": "Saudi Arabia"},{"Code": "SN", "Name": "Senegal"},{"Code": "RS", "Name": "Serbia"},{"Code": "SC", "Name": "Seychelles"},{"Code": "SL", "Name": "Sierra Leone"},{"Code": "SG", "Name": "Singapore"},{"Code": "SX", "Name": "Sint Maarten (Dutch part)"},{"Code": "SK", "Name": "Slovakia"},{"Code": "SI", "Name": "Slovenia"},{"Code": "SB", "Name": "Solomon Islands"},{"Code": "SO", "Name": "Somalia"},{"Code": "ZA", "Name": "South Africa"},{"Code": "GS", "Name": "South Georgia and the South Sandwich Islands"},{"Code": "SS", "Name": "South Sudan"},{"Code": "ES", "Name": "Spain"},{"Code": "LK", "Name": "Sri Lanka"},{"Code": "SD", "Name": "Sudan"},{"Code": "SR", "Name": "Suriname"},{"Code": "SJ", "Name": "Svalbard and Jan Mayen"},{"Code": "SZ", "Name": "Swaziland"},{"Code": "SE", "Name": "Sweden"},{"Code": "CH", "Name": "Switzerland"},{"Code": "SY", "Name": "Syrian Arab Republic"},{"Code": "TW", "Name": "Taiwan, Province of China"},{"Code": "TJ", "Name": "Tajikistan"},{"Code": "TZ", "Name": "Tanzania, United Republic of"},{"Code": "TH", "Name": "Thailand"},{"Code": "TL", "Name": "Timor-Leste"},{"Code": "TG", "Name": "Togo"},{"Code": "TK", "Name": "Tokelau"},{"Code": "TO", "Name": "Tonga"},{"Code": "TT", "Name": "Trinidad and Tobago"},{"Code": "TN", "Name": "Tunisia"},{"Code": "TR", "Name": "Turkey"},{"Code": "TM", "Name": "Turkmenistan"},{"Code": "TC", "Name": "Turks and Caicos Islands"},{"Code": "TV", "Name": "Tuvalu"},{"Code": "UG", "Name": "Uganda"},{"Code": "UA", "Name": "Ukraine"},{"Code": "AE", "Name": "United Arab Emirates"},{"Code": "GB", "Name": "United Kingdom"},{"Code": "US", "Name": "United States"},{"Code": "UM", "Name": "United States Minor Outlying Islands"},{"Code": "UY", "Name": "Uruguay"},{"Code": "UZ", "Name": "Uzbekistan"},{"Code": "VU", "Name": "Vanuatu"},{"Code": "VE", "Name": "Venezuela, Bolivarian Republic of"},{"Code": "VN", "Name": "Viet Nam"},{"Code": "VG", "Name": "Virgin Islands, British"},{"Code": "VI", "Name": "Virgin Islands, U.S."},{"Code": "WF", "Name": "Wallis and Futuna"},{"Code": "EH", "Name": "Western Sahara"},{"Code": "YE", "Name": "Yemen"},{"Code": "ZM", "Name": "Zambia"},{"Code": "ZW", "Name": "Zimbabwe"}] -------------------------------------------------------------------------------- /tests/fixtures/datahub-country-list/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "datahub": { 3 | "findability": "published", 4 | "hash": "2f0f383b6b82a6825432c7768c644c5f", 5 | "modified": "2017-11-15T07:58:19.911281", 6 | "owner": "core", 7 | "ownerid": "core", 8 | "stats": { 9 | "bytes": 25647, 10 | "rowcount": 249 11 | } 12 | }, 13 | "id": "core/country-list", 14 | "license": "ODC-PDDL", 15 | "name": "country-list", 16 | "profile": "data-package", 17 | "readme": "ISO 3166-1-alpha-2 English country names and code elements. This list states\nthe country names (official short names in English) in alphabetical order as\ngiven in [ISO 3166-1][] and the corresponding ISO 3166-1-alpha-2 code elements.\n\n[ISO 3166-1]: http://www.iso.org/iso/home/standards/country_codes.htm\n\nThis list is updated whenever a change to the official code list in ISO 3166-1\nis effected by the ISO 3166/MA.\n\nIt lists 250 official short names and code elements as of Dec 2012.\n\n## License\n\nThis material is licensed by its maintainers under the Public Domain Dedication\nand License.\n\nNevertheless, it should be noted that this material is ultimately sourced from\nISO and their rights and licensing policy is somewhat unclear. As this is a\nshort, simple database of facts there is a strong argument that no rights can\nsubsist in this collection. However, ISO state on [their\nsite](http://www.iso.org/iso/home/standards/country_codes.htm): \n\n> ISO makes the list of alpha-2 country codes available for internal use and\n> non-commercial purposes free of charge. \n\nThis carries the implication (though not spelled out) that other uses are not\npermitted and that, therefore, there may be rights preventing further general\nuse and reuse.\n\n", 18 | "resources": [ 19 | { 20 | "bytes": 4120, 21 | "datahub": { 22 | "derivedFrom": [ 23 | "data" 24 | ], 25 | "type": "derived/csv" 26 | }, 27 | "dialect": { 28 | "delimiter": ",", 29 | "doubleQuote": true, 30 | "lineTerminator": "\r\n", 31 | "quoteChar": "\"", 32 | "skipInitialSpace": false 33 | }, 34 | "dpp:streamedFrom": "https://s3.amazonaws.com/rawstore.datahub.io/d7c9d7cfb42cb69f4422dec222dbbaa8.csv", 35 | "encoding": "utf-8", 36 | "format": "csv", 37 | "hash": "d7c9d7cfb42cb69f4422dec222dbbaa8", 38 | "name": "data_csv", 39 | "path": "data_csv.csv", 40 | "primaryKey": "Code", 41 | "profile": "data-resource", 42 | "rowcount": 249, 43 | "schema": { 44 | "fields": [ 45 | { 46 | "description": "Country Name", 47 | "name": "Name", 48 | "type": "string" 49 | }, 50 | { 51 | "description": "ISO 2-digit code from ISO 3166-alpha-2", 52 | "name": "Code", 53 | "type": "string" 54 | } 55 | ] 56 | } 57 | }, 58 | { 59 | "bytes": 9576, 60 | "datahub": { 61 | "derivedFrom": [ 62 | "data" 63 | ], 64 | "type": "derived/json" 65 | }, 66 | "dpp:streamedFrom": "https://s3.amazonaws.com/rawstore.datahub.io/d7c9d7cfb42cb69f4422dec222dbbaa8.csv", 67 | "encoding": "utf-8", 68 | "format": "json", 69 | "hash": "8c458f2d15d9f2119654b29ede6e45b8", 70 | "name": "data_json", 71 | "path": "data_json.json", 72 | "primaryKey": "Code", 73 | "profile": "data-resource", 74 | "rowcount": 249, 75 | "schema": { 76 | "fields": [ 77 | { 78 | "description": "Country Name", 79 | "name": "Name", 80 | "type": "string" 81 | }, 82 | { 83 | "description": "ISO 2-digit code from ISO 3166-alpha-2", 84 | "name": "Code", 85 | "type": "string" 86 | } 87 | ] 88 | } 89 | }, 90 | { 91 | "bytes": 8081, 92 | "datahub": { 93 | "type": "derived/zip" 94 | }, 95 | "description": "Compressed versions of dataset. Includes normalized CSV and JSON data with original data and datapackage.json.", 96 | "dpp:streamedFrom": "/tmp/core.country-list.2.zip", 97 | "encoding": "utf-8", 98 | "format": "zip", 99 | "name": "datapackage_zip", 100 | "path": "datapackage_zip.zip", 101 | "profile": "data-resource" 102 | }, 103 | { 104 | "bytes": 3870, 105 | "datahub": { 106 | "type": "source/tabular" 107 | }, 108 | "dpp:streamedFrom": "https://s3.amazonaws.com/rawstore.datahub.io/d7c9d7cfb42cb69f4422dec222dbbaa8.csv", 109 | "encoding": "utf-8", 110 | "format": "csv", 111 | "name": "data", 112 | "path": "data.csv", 113 | "primaryKey": "Code", 114 | "profile": "data-resource", 115 | "schema": { 116 | "fields": [ 117 | { 118 | "description": "Country Name", 119 | "name": "Name", 120 | "type": "string" 121 | }, 122 | { 123 | "description": "ISO 2-digit code from ISO 3166-alpha-2", 124 | "name": "Code", 125 | "type": "string" 126 | } 127 | ] 128 | } 129 | } 130 | ], 131 | "sources": [ 132 | { 133 | "name": "ISO", 134 | "path": "http://www.iso.org/iso/home/standards/country_codes.htm", 135 | "title": "ISO" 136 | } 137 | ], 138 | "title": "List of all countries with their 2 digit codes (ISO 3166-2)", 139 | "views": [ 140 | { 141 | "datahub": { 142 | "type": "preview" 143 | }, 144 | "name": "datahub-preview-data_csv_preview", 145 | "resources": [ 146 | "data" 147 | ], 148 | "specType": "table", 149 | "transform": { 150 | "limit": 2000 151 | } 152 | } 153 | ] 154 | } -------------------------------------------------------------------------------- /tests/fixtures/datahub-country-list/datapackage_zip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/datapackage-php/8825c7dbe0131bd1775bbc4c441804dd5735d569/tests/fixtures/datahub-country-list/datapackage_zip.zip -------------------------------------------------------------------------------- /tests/fixtures/datapackage_zip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frictionlessdata/datapackage-php/8825c7dbe0131bd1775bbc4c441804dd5735d569/tests/fixtures/datapackage_zip.zip -------------------------------------------------------------------------------- /tests/fixtures/default_resource_invalid_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "default-resource-invalid-data", 3 | "resources": [ 4 | { 5 | "name": "default-resource-invalid-data", 6 | "path": ["foo.txt", "--invalid--"] 7 | } 8 | ] 9 | } -------------------------------------------------------------------------------- /tests/fixtures/fiscal-datapackage/budget.csv: -------------------------------------------------------------------------------- 1 | pk,budget,budget_date,payee 2 | 1,10000,01/01/2015,1 3 | 2,20000,01/02/2015,1 -------------------------------------------------------------------------------- /tests/fixtures/fiscal-datapackage/datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "my-openspending-datapackage", 3 | "title": "My OpenSpending Data Package", 4 | "profile": "tabular-data-package", 5 | "resources": [ 6 | { 7 | "name": "budget", 8 | "title": "Budget", 9 | "profile": "tabular-data-resource", 10 | "path": ["budget.csv"], 11 | "schema": { 12 | "fields": [ 13 | { 14 | "name": "id", 15 | "type": "string" 16 | }, 17 | { 18 | "name": "amount", 19 | "type": "number" 20 | }, 21 | { 22 | "name": "date", 23 | "type": "date" 24 | }, 25 | { 26 | "name": "payee", 27 | "type": "string" 28 | } 29 | ], 30 | "primaryKey": ["id"] 31 | }, 32 | "foreignKeys": [ 33 | { 34 | "fields": "payee", 35 | "reference": { 36 | "datapackage": "my-openspending-datapackage", 37 | "resource": "entities", 38 | "fields": "id" 39 | } 40 | } 41 | ] 42 | }, 43 | { 44 | "name": "entities", 45 | "title": "Entities", 46 | "profile": "tabular-data-resource", 47 | "path": ["entities.csv"], 48 | "schema": { 49 | "fields": [ 50 | { 51 | "name": "id", 52 | "type": "string" 53 | }, 54 | { 55 | "name": "title", 56 | "type": "string" 57 | }, 58 | { 59 | "name": "description", 60 | "type": "string" 61 | } 62 | ], 63 | "primaryKey": ["id"] 64 | } 65 | } 66 | ], 67 | "model": { 68 | "measures": { 69 | "amount": { 70 | "source": "budget", 71 | "currency": "USD", 72 | "factor": 1 73 | } 74 | }, 75 | "dimensions": { 76 | "date": { 77 | "dimensionType": "datetime", 78 | "attributes": { 79 | "year": { 80 | "source": "year" 81 | } 82 | }, 83 | "primaryKey": "year" 84 | }, 85 | "payee": { 86 | "dimensionType": "entity", 87 | "attributes": { 88 | "id": { 89 | "resource": "entities", 90 | "source": "id" 91 | }, 92 | "title": { 93 | "resource": "entities", 94 | "source": "title" 95 | }, 96 | "description": { 97 | "resource": "entities", 98 | "source": "description" 99 | } 100 | }, 101 | "primaryKey": ["id"] 102 | } 103 | } 104 | } 105 | } -------------------------------------------------------------------------------- /tests/fixtures/fiscal-datapackage/entities.csv: -------------------------------------------------------------------------------- 1 | id,name,description 2 | 1,Acme 1,They are the first acme company 3 | 2,Acme 2,They are the sceond acme company -------------------------------------------------------------------------------- /tests/fixtures/foo.txt: -------------------------------------------------------------------------------- 1 | foo -------------------------------------------------------------------------------- /tests/fixtures/invalid_tabular_data.csv: -------------------------------------------------------------------------------- 1 | id,email 2 | 1,"good@email.and.nice" 3 | 2,"bad.email" -------------------------------------------------------------------------------- /tests/fixtures/invalid_tabular_resource.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "invalid-tabular-resource", 3 | "resources": [ 4 | { 5 | "profile": "tabular-data-resource", 6 | "name": "invalid-tabular-resource", 7 | "path": ["simple_tabular_data2.csv"], 8 | "schema": { 9 | "foo": "bar" 10 | } 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /tests/fixtures/multi_data_datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "multi-data", 3 | "resources": [ 4 | {"name": "first-resource", "path": ["foo.txt", "bar.txt", "baz.txt"]}, 5 | {"name": "second-resource", "path": ["bar.txt", "baz.txt"]}, 6 | {"name": "third-resource", "path": ["baz.txt"]} 7 | ] 8 | } -------------------------------------------------------------------------------- /tests/fixtures/simple_invalid_datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datapackage-name" 3 | } -------------------------------------------------------------------------------- /tests/fixtures/simple_tabular_data.csv: -------------------------------------------------------------------------------- 1 | id,name 2 | 1,"one" 3 | 2,"two" 4 | 3,"three" -------------------------------------------------------------------------------- /tests/fixtures/simple_valid_datapackage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datapackage-name", 3 | "resources": [ 4 | { "name": "resource-name", "path": ["foo.txt"] } 5 | ] 6 | } -------------------------------------------------------------------------------- /tests/fixtures/simple_valid_datapackage_mock_http_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datapackage-name", 3 | "resources": [ 4 | { "name": "resource-name", "path": ["mock-http://foo.txt", "mock-http://foo.txt"] } 5 | ] 6 | } -------------------------------------------------------------------------------- /tests/fixtures/tabular_resource_invalid_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tabular-resource-invalid-data", 3 | "resources": [ 4 | { 5 | "profile": "tabular-data-resource", 6 | "name": "tabular-resource-invalid-data", 7 | "path": ["valid_emails_tabular_data.csv", "invalid_tabular_data.csv"], 8 | "schema": { 9 | "fields": [ 10 | {"name": "id", "type": "integer"}, 11 | {"name": "email", "type": "string", "format": "email"} 12 | ] 13 | } 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /tests/fixtures/test-custom-resource-profile.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "title": "Data Resource", 4 | "description": "Data Resource.", 5 | "type": "object", 6 | "required": [ 7 | "name", 8 | "path", 9 | "custom" 10 | ], 11 | "properties": { 12 | "custom": { 13 | "type": "array", 14 | "minItems": 1, 15 | "items": { 16 | "type": "string" 17 | } 18 | }, 19 | "profile": { 20 | "propertyOrder": 10, 21 | "title": "Profile", 22 | "description": "The profile of this descriptor.", 23 | "context": "Every Package and Resource descriptor has a profile. The default profile, if none is declared, is `default`. The namespace for the profile is the type of descriptor, so, `default` for a Package descriptor is not the same as `default` for a Resource descriptor.", 24 | "type": "string", 25 | "default": "default", 26 | "examples": [ 27 | "{\n \"profile\": \"tabular-data-package\"\n}\n", 28 | "{\n \"profile\": \"http://example.com/my-profiles-json-schema.json\"\n}\n" 29 | ] 30 | }, 31 | "name": { 32 | "propertyOrder": 20, 33 | "title": "Name", 34 | "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", 35 | "type": "string", 36 | "pattern": "^([-a-z0-9._/])+$", 37 | "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", 38 | "examples": [ 39 | "{\n \"name\": \"my-nice-name\"\n}\n" 40 | ] 41 | }, 42 | "path": { 43 | "propertyOrder": 30, 44 | "title": "Data", 45 | "description": "A reference to the data for this resource. `data` `MUST` be an array of valid URIs.", 46 | "type": "array", 47 | "minItems": 1, 48 | "items": { 49 | "examples": [ 50 | "[ \"file.csv\" ]\n", 51 | "[ \"#/data/my-data\" ]\n", 52 | "[ \"http://example.com/file.csv\" ]\n" 53 | ], 54 | "title": "URI", 55 | "description": "A URI (with some restrictions), being a fully qualified HTTP address, a relative POSIX path, or a JSON Pointer.", 56 | "type": "string", 57 | 58 | "context": "Implementations need to negotiate the type of URI provided, and dereference the data accordingly. There are restrictions imposed on URIs that are POSIX paths: see [the notes on descriptors](#descriptor) for more information." 59 | }, 60 | "context": "The dereferenced value of each referenced data source in the `data` `array` `MUST` be commensurate with a native, dereferenced representation of the data the resource describes. For example, in a *Tabular* Data Resource, this means that the dereferenced value of `data` `MUST` be an array.", 61 | "examples": [ 62 | "{\n \"data\": [\n \"file.csv\",\n \"file2.csv\"\n ]\n}\n", 63 | "{\n \"data\": [\n \"http://example.com/file.csv\",\n \"http://example.com/file2.csv\"\n ]\n}\n", 64 | "{\n \"data\": [\n \"#/data/my-data\",\n \"#/data/my-data2\"\n ]\n}\n" 65 | ] 66 | }, 67 | "schema": { 68 | "propertyOrder": 40, 69 | "title": "Schema", 70 | "description": "A schema for this resource.", 71 | "type": "object" 72 | }, 73 | "title": { 74 | "propertyOrder": 50, 75 | "title": "Title", 76 | "description": "A human-readable title.", 77 | "type": "string", 78 | "examples": [ 79 | "{\n \"title\": \"My Package Title\"\n}\n" 80 | ] 81 | }, 82 | "description": { 83 | "propertyOrder": 60, 84 | "format": "textarea", 85 | "title": "Description", 86 | "description": "A text description. Markdown is encouraged.", 87 | "type": "string", 88 | "examples": [ 89 | "{\n \"description\": \"# My Package description\\nAll about my package.\"\n}\n" 90 | ] 91 | }, 92 | "homepage": { 93 | "propertyOrder": 70, 94 | "title": "Home Page", 95 | "description": "The home on the web that is related to this data package.", 96 | "type": "object", 97 | "properties": { 98 | "name": { 99 | "title": "Name", 100 | "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", 101 | "type": "string", 102 | "pattern": "^([-a-z0-9._/])+$", 103 | "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", 104 | "examples": [ 105 | "{\n \"name\": \"my-nice-name\"\n}\n" 106 | ] 107 | }, 108 | "uri": { 109 | "title": "URI", 110 | "description": "A URI (with some restrictions), being a fully qualified HTTP address, a relative POSIX path, or a JSON Pointer.", 111 | "type": "string", 112 | 113 | "examples": [ 114 | "{\n \"uri\": \"file.csv\"\n}\n", 115 | "{\n \"uri\": \"#/data/my-data\"\n}\n", 116 | "{\n \"uri\": \"http://example.com/file.csv\"\n}\n" 117 | ], 118 | "context": "Implementations need to negotiate the type of URI provided, and dereference the data accordingly. There are restrictions imposed on URIs that are POSIX paths: see [the notes on descriptors](#descriptor) for more information." 119 | }, 120 | "title": { 121 | "title": "Title", 122 | "description": "A human-readable title.", 123 | "type": "string", 124 | "examples": [ 125 | "{\n \"title\": \"My Package Title\"\n}\n" 126 | ] 127 | } 128 | }, 129 | "examples": [ 130 | "{\n \"homepage\": {\n \"name\": \"My Web Page\",\n \"uri\": \"http://example.com/\"\n }\n}\n" 131 | ] 132 | }, 133 | "sources": { 134 | "propertyOrder": 140, 135 | "options": { 136 | "hidden": true 137 | }, 138 | "title": "Sources", 139 | "description": "The raw sources for this resource.", 140 | "type": "array", 141 | "minItems": 1, 142 | "items": { 143 | "title": "Source", 144 | "description": "A source file.", 145 | "type": "object", 146 | "required": [ 147 | "uri" 148 | ], 149 | "properties": { 150 | "name": { 151 | "title": "Name", 152 | "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", 153 | "type": "string", 154 | "pattern": "^([-a-z0-9._/])+$", 155 | "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", 156 | "examples": [ 157 | "{\n \"name\": \"my-nice-name\"\n}\n" 158 | ] 159 | }, 160 | "uri": { 161 | "title": "URI", 162 | "description": "A URI (with some restrictions), being a fully qualified HTTP address, a relative POSIX path, or a JSON Pointer.", 163 | "type": "string", 164 | 165 | "examples": [ 166 | "{\n \"uri\": \"file.csv\"\n}\n", 167 | "{\n \"uri\": \"#/data/my-data\"\n}\n", 168 | "{\n \"uri\": \"http://example.com/file.csv\"\n}\n" 169 | ], 170 | "context": "Implementations need to negotiate the type of URI provided, and dereference the data accordingly. There are restrictions imposed on URIs that are POSIX paths: see [the notes on descriptors](#descriptor) for more information." 171 | }, 172 | "email": { 173 | "title": "Email", 174 | "description": "An email address.", 175 | "type": "string", 176 | "format": "email", 177 | "examples": [ 178 | "{\n \"email\": \"example@example.com\"\n}\n" 179 | ] 180 | } 181 | } 182 | }, 183 | "examples": [ 184 | "{\n \"sources\": [\n {\n \"name\": \"World Bank and OECD\",\n \"uri\": \"http://data.worldbank.org/indicator/NY.GDP.MKTP.CD\"\n }\n ]\n}\n" 185 | ] 186 | }, 187 | "licenses": { 188 | "description": "The license(s) under which the resource is published.", 189 | "propertyOrder": 150, 190 | "options": { 191 | "hidden": true 192 | }, 193 | "title": "Licenses", 194 | "type": "array", 195 | "minItems": 1, 196 | "items": { 197 | "title": "License", 198 | "description": "A license for this descriptor.", 199 | "type": "object", 200 | "required": [ 201 | "uri" 202 | ], 203 | "properties": { 204 | "name": { 205 | "title": "Name", 206 | "description": "An identifier string. Lower case characters with `.`, `_`, `-` and `/` are allowed.", 207 | "type": "string", 208 | "pattern": "^([-a-z0-9._/])+$", 209 | "context": "This is ideally a url-usable and human-readable name. Name `SHOULD` be invariant, meaning it `SHOULD NOT` change when its parent descriptor is updated.", 210 | "examples": [ 211 | "{\n \"name\": \"my-nice-name\"\n}\n" 212 | ] 213 | }, 214 | "uri": { 215 | "title": "URI", 216 | "description": "A URI (with some restrictions), being a fully qualified HTTP address, a relative POSIX path, or a JSON Pointer.", 217 | "type": "string", 218 | 219 | "examples": [ 220 | "{\n \"uri\": \"file.csv\"\n}\n", 221 | "{\n \"uri\": \"#/data/my-data\"\n}\n", 222 | "{\n \"uri\": \"http://example.com/file.csv\"\n}\n" 223 | ], 224 | "context": "Implementations need to negotiate the type of URI provided, and dereference the data accordingly. There are restrictions imposed on URIs that are POSIX paths: see [the notes on descriptors](#descriptor) for more information." 225 | }, 226 | "title": { 227 | "title": "Title", 228 | "description": "A human-readable title.", 229 | "type": "string", 230 | "examples": [ 231 | "{\n \"title\": \"My Package Title\"\n}\n" 232 | ] 233 | } 234 | }, 235 | "context": "Use of this property does not imply that the person was the original creator of, or a contributor to, the data in the descriptor, but refers to the composition of the descriptor itself." 236 | }, 237 | "context": "This property is not legally binding and does not guarantee that the package is licensed under the terms defined herein.", 238 | "examples": [ 239 | "{\n \"licenses\": [\n {\n \"name\": \"ODC-PDDL-1.0\",\n \"uri\": \"http://opendatacommons.org/licenses/pddl/\"\n }\n ]\n}\n" 240 | ] 241 | }, 242 | "format": { 243 | "propertyOrder": 80, 244 | "title": "Format", 245 | "description": "The file format of this resource.", 246 | "context": "`csv`, `xls`, `json` are examples of common formats.", 247 | "type": "string", 248 | "examples": [ 249 | "{\n \"format\": \"xls\"\n}\n" 250 | ] 251 | }, 252 | "mediatype": { 253 | "propertyOrder": 90, 254 | "title": "Media Type", 255 | "description": "The media type of this resource. Can be any valid media type listed with [IANA](https://www.iana.org/assignments/media-types/media-types.xhtml).", 256 | "type": "string", 257 | "pattern": "^(.+)/(.+)$", 258 | "examples": [ 259 | "{\n \"mediatype\": \"text/csv\"\n}\n" 260 | ] 261 | }, 262 | "encoding": { 263 | "propertyOrder": 100, 264 | "title": "Encoding", 265 | "description": "The file encoding of this resource.", 266 | "type": "string", 267 | "default": "utf-8", 268 | "examples": [ 269 | "{\n \"encoding\": \"utf-8\"\n}\n" 270 | ] 271 | }, 272 | "bytes": { 273 | "propertyOrder": 110, 274 | "options": { 275 | "hidden": true 276 | }, 277 | "title": "Bytes", 278 | "description": "The size of this resource in bytes.", 279 | "type": "integer", 280 | "examples": [ 281 | "{\n \"bytes\": 2082\n}\n" 282 | ] 283 | }, 284 | "hash": { 285 | "propertyOrder": 120, 286 | "options": { 287 | "hidden": true 288 | }, 289 | "title": "Hash", 290 | "type": "string", 291 | "description": "The MD5 hash of this resource. Indicate other hashing algorithms with the {algorithm}:{hash} format.", 292 | "pattern": "^([^:]+:[a-fA-F0-9]+|[a-fA-F0-9]{32}|)$", 293 | "examples": [ 294 | "{\n \"hash\": \"d25c9c77f588f5dc32059d2da1136c02\"\n}\n", 295 | "{\n \"hash\": \"SHA256:5262f12512590031bbcc9a430452bfd75c2791ad6771320bb4b5728bfb78c4d0\"\n}\n" 296 | ] 297 | } 298 | } 299 | } -------------------------------------------------------------------------------- /tests/fixtures/valid_emails_tabular_data.csv: -------------------------------------------------------------------------------- 1 | id,email 2 | 1,"good@email.and.nice" 3 | 2,"another.good@email.com" -------------------------------------------------------------------------------- /update_registry.php: -------------------------------------------------------------------------------- 1 | schema_path; 17 | $old_schema = file_exists($filename) ? file_get_contents($filename) : 'FORCE UPDATE'; 18 | echo "downloading schema from {$schema->schema}\n"; 19 | $new_schema = file_get_contents($schema->schema); 20 | if ($old_schema == $new_schema) { 21 | echo "no update needed\n"; 22 | } else { 23 | echo "schema changed - updating local file\n"; 24 | file_put_contents($filename, $new_schema); 25 | file_put_contents($base_filename.'CHANGELOG', "\n\nChanges to {$schema->id} schema\n".date('c')."\n* check the git diff and summarize the spec changes here\n* \n\n", FILE_APPEND); 26 | if ($schema->id == 'registry') { 27 | echo "registry was updated, re-running update to fetch latest files from registry\n\n"; 28 | 29 | return update(); 30 | } 31 | ++$numUpdated; 32 | } 33 | } 34 | echo "\n{$numUpdated} schemas updated\n"; 35 | 36 | return 0; 37 | } 38 | 39 | exit(update()); 40 | --------------------------------------------------------------------------------