├── .gitignore
├── .travis.yml
├── README.md
├── bin
└── dpm
├── doc
├── command-ckan.md
├── command-dpmrc.md
├── command-help.md
├── command-identifier.md
├── command-info.md
├── command-init.md
├── command-install.md
└── command-validate.md
├── lib
├── index.js
├── tree.js
└── util.js
├── package.json
└── test
├── all.js
└── fixtures
├── datapackage-example-inline
├── README.md
└── datapackage.json
├── mydpkg-invalid-test
└── datapackage.json
├── mydpkg-test
├── datapackage.json
├── scripts
│ └── test.r
├── x1.csv
└── x2.csv
└── req-test
└── package.json
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 |
3 | lib-cov
4 | *.seed
5 | *.log
6 | *.csv
7 | *.dat
8 | *.out
9 | *.pid
10 | *.gz
11 |
12 | pids/
13 | logs/
14 | results/
15 | node_modules/
16 | .idea/
17 |
18 | npm-debug.log
19 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 | node_js:
3 | - 0.12
4 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 | DEPRECATED and REPLACED - SEP 2017
3 | data tool has replaced dpm
4 |
5 | data is our new command line tool
6 |
7 | https://datahub.io/docs/features/data-cli
8 |
9 | https://github.com/datahq/data-cli
10 |
11 |
12 | ---
13 | ---
14 |
15 | # Data Package Manager - in JavaScript
16 |
17 | [](https://nodei.co/npm/dpmjs/)
18 |
19 | [](https://travis-ci.org/frictionlessdata/dpm-js)
20 |
21 | dpm is a library and command line manager for [Data Packages](http://dataprotocols.org/data-packages/).
22 |
23 | > Starting from v0.8.0 package on NPM has been renamed to `dpmjs`.
24 |
25 | ## Install
26 |
27 | `dpm` is implemented in node, so to install `dpm` just do:
28 |
29 | ```bash
30 | npm install dpmjs -g
31 | ```
32 |
33 | ## Command Line Usage
34 |
35 | To get an overview and list of commands check out the command line help:
36 |
37 | ```bash
38 | dpm --help
39 | ```
40 |
41 | ## Using DPM programaticaly
42 |
43 | You can also use `dpm` programatically.
44 |
45 | ```javascript
46 | var Dpm = require('dpmjs');
47 | var dpm = new Dpm(conf);
48 |
49 | dpm.install(['mydpkg@0.0.0', 'mydata@1.0.0'], {cache: true}, function(err, dpkgs){
50 | //done!
51 | });
52 | dpm.on('log', console.log); //if you like stuff on stdout
53 | ```
54 |
55 | ## Changelog
56 |
57 | * v0.8.0: renamed to `dpmjs` on NPM
58 | * v0.7.0: new ckan command
59 | * v0.6.0: much better validation via v0.2 of datapackage-validate
60 |
61 | ## References
62 |
63 | Previous `dpm` (python-based) can still be found at
64 | http://github.com/okfn/dpm-old.
65 |
--------------------------------------------------------------------------------
/bin/dpm:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 |
3 | var path = require('path')
4 | , dpinit = require('datapackage-init')
5 | , util = require('util')
6 | , request = require('request')
7 | , fs = require('fs')
8 | , Dpm = require('../lib/index')
9 | , optimist = require('optimist')
10 | , read = require('read')
11 | , async = require('async')
12 | , rimraf = require('rimraf')
13 | , split = require('split')
14 | , tree = require('../lib/tree')
15 | , readdirpSync = require('fs-readdir-recursive')
16 | , os = require('os')
17 | , colors = require('colors');
18 |
19 |
20 | function getHelpDoc(cmd) {
21 | var fp = path.join(path.dirname(__dirname), 'doc', 'command-' + cmd + '.md')
22 | , info = fs.readFileSync(fp, 'utf8')
23 | ;
24 | info = info.replace('$0', 'dpm');
25 | return info;
26 | }
27 |
28 | var argv = optimist
29 | .usage(
30 | getHelpDoc('help')
31 | )
32 | .alias('f', 'force').describe('f', 'just do it')
33 | .alias('s', 'save').describe('s', 'data packages will appear in your dataDependencies')
34 | .alias('c', 'cache').describe('c', 'store the resources content on the disk in a data/ directory')
35 | .alias('h', 'help').describe('h', 'print usage')
36 | .alias('v', 'version').describe('v', 'print version number')
37 | .argv;
38 |
39 | if(argv.h){
40 | console.log(optimist.help());
41 | process.exit(0);
42 | }
43 |
44 | if(argv.v){
45 | console.log(require('../package.json').version);
46 | process.exit(0);
47 | }
48 |
49 | var conf = require('rc')('dpm', {}, process.cwd());
50 | var dpm = new Dpm(conf)
51 |
52 | if(argv._[0] !== 'cat'){
53 | dpm.on('log', console.log);
54 | }
55 |
56 | if (argv._[0] === 'init') {
57 | dpinit.init(dpm.root, function(err, dpkg) {
58 | var path_ = path.join(dpm.root, 'datapackage.json');
59 | console.log('datapackage.json written to ' + path_);
60 | });
61 | } else if (argv._[0] === 'validate') {
62 | if (argv._.length > 1) {
63 | dpm.validate(argv._[1]);
64 | } else {
65 | dpm.validate();
66 | }
67 | } else if(argv._[0] === 'install'){
68 | var dpkgIds = argv._.slice(1)
69 | , options = {cache: argv.cache, force: argv.force}
70 | ;
71 | dpm.installFromArgString(dpkgIds, options, function(err, dpkgs) {
72 | if (err) _fail(err);
73 | else console.log(tree.deps(dpkgs));
74 | });
75 | } else if(argv._[0] === 'ckan'){
76 | dpm.ckan(argv, function(err, msg) {
77 | if (err) _fail(err);
78 | else console.log(msg);
79 | });
80 | } else if(argv._[0] === 'info'){
81 | argv.identifier = argv._[1];
82 | dpm.info(argv, function(err, info) {
83 | if (err) _fail(err);
84 | else console.log(info.plain);
85 | });
86 | } else if (argv._[0] === 'help'){
87 | var cmd = argv._[1];
88 | if (cmd == 'ident') cmd = 'identifier'
89 | if(cmd && ['init', 'cat', 'ckan', 'clone', 'dpmrc', 'get', 'info', 'install', 'identifier', 'validate'].indexOf(cmd) !== -1){
90 | console.log(os.EOL + getHelpDoc(cmd));
91 | } else {
92 | console.log(optimist.help());
93 | }
94 | } else if (!argv._[0]){
95 | console.log(optimist.help());
96 | _fail(new Error('missing command'));
97 | } else {
98 | _fail(new Error('invalid command'));
99 | };
100 |
101 | function _fail(err){
102 | if (typeof(err) === 'string') {
103 | err = new Error(err);
104 | }
105 | if(err){
106 | console.error('dpm'.grey + ' ERR! '.red + err.message + (('code' in err) ? ' (' + err.code + ')': ''));
107 | process.exit(1);
108 | }
109 | };
110 |
--------------------------------------------------------------------------------
/doc/command-ckan.md:
--------------------------------------------------------------------------------
1 | $0 ckan {ckan-instance-url}
2 |
3 | Push the Data Package in the current directory into CKAN instance at
4 | {ckan-instance-url}.
5 |
6 | This will also import data from the Data Package resources into the DataStore
7 | (if of appropriate format i.e. CSVs).
8 |
9 | If you need to set the owner organization in CKAN use --owner_org e.g.
10 |
11 | dpm ckan http://datahub.io --owner_org=my-organization
12 |
13 | ## Configuration - Important
14 |
15 | To push to CKAN you will need to set an API Key for that instance. To do this
16 | add ckan config to your .dpmrc file as follows (see "dpm help dpmrc" for more
17 | on configuration files):
18 |
19 | ```
20 | ; you have one 'ckan.XXX' section per ckan instance you want to interact with
21 | ; XXX can be anything but must not contain '.' and can be used as shorthand
22 | ; e.g. you can do
23 | ; dpm push datahub
24 | ; rather than
25 | ; dpm push http://datahub.io/
26 | [ckan.datahub]
27 | url = http://datahub.io/
28 | apikey = XXX
29 |
30 | [ckan.local]
31 | url = http://localhost:5000
32 | apikey = XXX
33 | ```
34 |
35 |
36 |
--------------------------------------------------------------------------------
/doc/command-dpmrc.md:
--------------------------------------------------------------------------------
1 | dpm supports loading common configuration using the rc library:
2 | https://www.npmjs.org/package/rc
3 |
4 | It therefore can load configuration via any of the methods supported by rc.
5 |
6 | The recommended method is to create an .dpmrc file in your home directory in
7 | ini format.
8 |
9 | Here's an example:
10 |
11 | ```
12 | ; start of the file
13 |
14 | ; you have to have something at top level or subsections are not correctly loaded
15 | doesNotMatter = 1
16 |
17 | [mysubsection]
18 | foo = bar
19 | ```
20 |
--------------------------------------------------------------------------------
/doc/command-help.md:
--------------------------------------------------------------------------------
1 | Usage: dpm
2 |
3 | where is one of:
4 |
5 | init
6 | info
7 | install
8 | validate
9 | ckan
10 | help
11 |
12 | dpm help detailed help on
13 |
14 | Specify configs in the ini-formatted file:
15 |
16 | ${HOME}/.dpmrc
17 |
18 | For more detail do:
19 |
20 | dpm help dpmrc
21 |
--------------------------------------------------------------------------------
/doc/command-identifier.md:
--------------------------------------------------------------------------------
1 | Data Packages are frequently identified by a Data Package Ident(ifier) String.
2 |
3 | An Identifier String is a single string identifying a Data Package (and,
4 | implicitly, its location). Examples:
5 |
6 | * A direct URL to a datapackage (with or without the `datapackage.json`) e.g.
7 |
8 | http://mywebsite.com/mydatapackage/
9 | // or with the datapackage.json
10 | http://mywebsite.com/mydatapackage/datapackage.json
11 |
12 | * A Github URL
13 |
14 | http://github.com/datasets/gold-prices
15 |
16 | * A single name
17 |
18 | gold-prices
19 |
20 | In this case the data package must be in the core datasets in the primary registry.
21 |
22 | For full details see http://dataprotocols.org/data-package-identifier/
23 |
24 |
--------------------------------------------------------------------------------
/doc/command-info.md:
--------------------------------------------------------------------------------
1 | $0 info {identifier}
2 |
3 | Print out information about Data Package identified by {identifier}
4 |
5 | (For more on identifiers see dpm help identifier).
6 |
7 | For example:
8 |
9 | # print out info about Data Package in current directory
10 | dpm info
11 |
12 | # print out info about this Data Package online (on GitHub)
13 | dpm info https://github.com/datasets/gdp
14 |
15 | # print out info about this Data Package online
16 | dpm info http://data.openspending.org/datasets/gb-cra
17 |
18 |
--------------------------------------------------------------------------------
/doc/command-init.md:
--------------------------------------------------------------------------------
1 | $0 init
2 |
3 | Initialize (create) a Data Package by creating a `datapackage.json` file in the
4 | current directory.
5 |
6 | It only covers the most common items, and tries to guess sane defaults. See
7 | http://data.okfn.org/doc/data-package for definitive documentation on
8 | these fields, and exactly what they do.
9 |
10 |
--------------------------------------------------------------------------------
/doc/command-install.md:
--------------------------------------------------------------------------------
1 | $0 install {url}
2 |
3 | Install Data Package at {url} into subdirectory of current directory named
4 | "datapackages".
5 |
6 | The list of data packages to install either comes from stdin, or (if
7 | no data packages are specified to stdin), from the "dataDependencies"
8 | object of the datapackage.json file of the current working directory.
9 |
10 | By default, each data package is installed in a directory (named after
11 | the data package "name" property) and located inside the datapackages
12 | directory.
13 |
14 |
--------------------------------------------------------------------------------
/doc/command-validate.md:
--------------------------------------------------------------------------------
1 | $0 validate [{path}]
2 |
3 | Validate Data Package in current directory (or at {path} if specified) by
4 | checking, in particular, that its `datapackage.json` is valid.
5 |
--------------------------------------------------------------------------------
/lib/index.js:
--------------------------------------------------------------------------------
1 | var crypto = require('crypto')
2 | , url = require('url')
3 | , semver = require('semver')
4 | , uniq = require('lodash.uniq')
5 | , flatten = require('lodash.flatten')
6 | , glob = require('glob')
7 | , querystring = require('querystring')
8 | , cookie = require('cookie')
9 | , EventEmitter = require('events').EventEmitter
10 | , util = require('util')
11 | , request = require('request')
12 | , path = require('path')
13 | , mime = require('mime')
14 | , rimraf = require('rimraf')
15 | , mkdirp = require('mkdirp')
16 | , async = require('async')
17 | , fs = require('fs')
18 | , zlib = require('zlib')
19 | , tar = require('tar')
20 | , once = require('once')
21 | , concat = require('concat-stream')
22 | , validator = require('datapackage-validate')
23 | , dpSpec = require('datapackage-identifier')
24 | , dpRead = require('datapackage-read')
25 | , dpCkan = require('datapackage-ckan')
26 | , colors = require('colors')
27 | ;
28 |
29 |
30 | var Dpm = module.exports = function(config, root){
31 | EventEmitter.call(this);
32 |
33 | this.root = root || process.cwd();
34 | this.config = config;
35 | };
36 |
37 | util.inherits(Dpm, EventEmitter);
38 |
39 | Dpm.prototype.validate = function(path_) {
40 | var dpjsonPath = path_ || this.root;
41 | // strip datapackage.json if already there
42 | dpjsonPath = dpjsonPath.replace(/datapackage.json$/, '');
43 | // now add it on
44 | dpjsonPath = path.join(dpjsonPath, 'datapackage.json');
45 | if (!fs.existsSync(dpjsonPath)) {
46 | console.log(('No DataPackage at path ' + dpjsonPath).red);
47 | return;
48 | }
49 | var out = fs.readFileSync(dpjsonPath, {encoding: 'utf8'});
50 | return validator.validate(out).then(function (results) {
51 | if (results.valid) {
52 | console.log('DataPackage.json is Valid'.green);
53 | } else {
54 | console.log('DataPackage.json is Invalid'.red);
55 | console.log(JSON.stringify(results, null, 2));
56 | }
57 | return results;
58 | });
59 | };
60 |
61 |
62 | //TODO: remove
63 | Dpm.prototype.url = function(path, queryObj){
64 | return this.config.protocol + '://' + this.config.hostname + ':' + this.config.port + path + ( (queryObj) ? '?' + querystring.stringify(queryObj): '');
65 | };
66 |
67 | Dpm.prototype.logHttp = function(methodCode, reqUrl){
68 | this.emit('log', 'dpm'.grey + ' http '.green + methodCode.toString().magenta + ' ' + reqUrl.replace(/:80\/|:443\//, '/'));
69 | };
70 |
71 | Dpm.prototype.resolveDeps = function(dataDependencies, callback){
72 | var deps = [];
73 | dataDependencies = dataDependencies || {};
74 | for (var name in dataDependencies){
75 | deps.push({name: name, range: dataDependencies[name]});
76 | }
77 |
78 | async.map(deps, function(dep, cb){
79 |
80 | var rurl = this.url('/' + dep.name);
81 | this.logHttp('GET', rurl);
82 |
83 | request(this.rOpts(rurl), function(err, res, versions){
84 | if(err) return cb(err);
85 |
86 | this.logHttp(res.statusCode, rurl);
87 | if (res.statusCode >= 400){
88 | var err = new Error('fail');
89 | err.code = res.statusCode;
90 | return cb(err);
91 | }
92 |
93 | versions = JSON.parse(versions);
94 | var version = semver.maxSatisfying(versions, dep.range);
95 |
96 | cb(null, dep.name + '@' + version);
97 |
98 | }.bind(this));
99 |
100 | }.bind(this), callback);
101 |
102 | };
103 |
104 |
105 | // TODO: reintroduce support for --save to list new installs as dependencies
106 | Dpm.prototype.installFromArgString = function(dpkgIds, options, callback) {
107 | var self = this
108 | , dpkg = null
109 | ;
110 |
111 | if(!dpkgIds.length) { //get deps for a package.json
112 | var dpPath = path.join(this.root, 'datapackage.json');
113 | try {
114 | dpkg = JSON.parse(fs.readFileSync(dpPath));
115 | } catch(e){
116 | callback(new Error('could not find datapackage at '+ dpPath));
117 | }
118 | dpm.resolveDeps(dpkg.dataDependencies, function(err, dpkgIds){
119 | if (err) callback(new Error(err));
120 | else next(dpkgIds);
121 | });
122 | } else {
123 | next(dpkgIds);
124 | }
125 |
126 | function next(dpkgIds){
127 | self.install(dpkgIds, options, callback);
128 | }
129 | }
130 |
131 | // dpkgIds are data package "specs":
132 | //
133 | // - url to a datapackage
134 | // - path to a datapackage
135 | // - package name
136 | Dpm.prototype.install = function(dpkgIds, opts, callback){
137 | if(arguments.length === 2){
138 | callback = opts;
139 | opts = {};
140 | }
141 | async.map(dpkgIds, this.get.bind(this), callback);
142 | };
143 |
144 | // retrieve a (remote) data package into local datapackages directory
145 | Dpm.prototype.get = function(dpkgId, opts, callback) {
146 | if(arguments.length === 2){
147 | callback = opts;
148 | opts = {};
149 | }
150 |
151 | var self = this
152 | , spec = dpSpec.parse(dpkgId)
153 | , root = path.join(this.root, 'datapackages')
154 | , pkgPath = path.join(root, spec.name)
155 | , dpjsonUrl = spec.url.replace(/\/$/, '') + '/datapackage.json'
156 | ;
157 |
158 | if (!fs.existsSync(pkgPath)) {
159 | mkdirp.sync(pkgPath);
160 | }
161 |
162 | if (!spec.url) {
163 | callback(new Error('We only support installing from URLs at the moment and "' + dpkgId + '" is not a URL'));
164 | return;
165 | }
166 |
167 | request.get(dpjsonUrl, function(err, res, body) {
168 | if(err) return callback(err);
169 |
170 | var dpkg = JSON.parse(body)
171 | , dest = path.join(pkgPath, 'datapackage.json');
172 |
173 | fs.writeFileSync(dest, JSON.stringify(dpkg, null, 2));
174 |
175 | var resources = dpkg.resources;
176 | // TODO: convert path to urls ...
177 |
178 | async.each(resources, function(r, cb) {
179 | cb = once(cb);
180 | if(r.data) return cb(null);
181 |
182 | var resourceUrl = r.url ? r.url : spec.url.replace(/\/$/, '') + '/' + r.path;
183 |
184 | self.logHttp('GET', resourceUrl);
185 | var req = request(resourceUrl);
186 | req.on('error', cb);
187 | req.on('response', function(resp){
188 | self.logHttp(resp.statusCode, resourceUrl);
189 | if(resp.statusCode >= 400){
190 | resp.pipe(concat(function(body){
191 | var err = new Error(body.toString);
192 | err.code = resp.statusCode;
193 | cb(err);
194 | }));
195 | return;
196 | }
197 |
198 | var defaultDataRootPath = path.join(pkgPath, 'data')
199 | , destPath = ''
200 | ;
201 | if (r.path) {
202 | destPath = path.join(pkgPath, r.path);
203 | } else {
204 | var filename = url.parse(r.url).pathname.split('/').pop();
205 | destPath = path.join(defaultDataRootPath, filename);
206 | // TODO: store this into local datapackage (??)
207 | }
208 |
209 | mkdirp.sync(path.dirname(destPath));
210 | resp
211 | .pipe(fs.createWriteStream(destPath))
212 | .on('finish', function(){
213 | cb(null);
214 | });
215 | });
216 | },
217 | // what we call when async.each finishes
218 | function(err) {
219 | callback(err, dpkg);
220 | }
221 | );
222 | });
223 | };
224 |
225 | Dpm.prototype.ckan = function(argv, callback) {
226 | var ckanUrlOrName = argv._[1]
227 | , dpPath = this.root
228 | ;
229 |
230 | // convert configs keyed by name to ones keyed by urls ...
231 | var ckanConfigs = {};
232 | for(key in this.config.ckan) {
233 | ckanConfigs[this.config.ckan[key].url] = this.config.ckan[key];
234 | };
235 | var config = ckanConfigs[ckanUrlOrName] || this.config.ckan ? this.config.ckan[ckanUrlOrName] : null;
236 |
237 | if (!config) {
238 | var msg = 'You need to set a CKAN API Key for CKAN instance ' + ckanUrlOrName +'. See help on ckan command for how to do this.';
239 | callback(msg);
240 | return;
241 | }
242 |
243 | var ckanIt = new dpCkan.Pusher(config.url, config.apikey);
244 | ckanIt.push(dpPath, argv, callback);
245 | };
246 |
247 | Dpm.prototype.info = function(argv, callback) {
248 | if (!argv.identifier) {
249 | argv.identifier = path.resolve('.');
250 | }
251 | out = dpSpec.parse(argv.identifier);
252 | if (out.originalType == 'path') {
253 | dpRead.load(out.path, handleIt);
254 | } else {
255 | dpRead.loadUrl(out.url, handleIt);
256 | }
257 | function handleIt(err, info) {
258 | if (!err) {
259 | var out = {
260 | json: info,
261 | plain: JSON.stringify(info, null, 2),
262 | html: 'TODO'
263 | }
264 | }
265 | callback(err, out);
266 | }
267 | }
268 |
--------------------------------------------------------------------------------
/lib/tree.js:
--------------------------------------------------------------------------------
1 | var archy = require('archy')
2 | , flat = require('flat')
3 | , path = require('path');
4 |
5 | exports.deps = function (dpkgs) {
6 |
7 | var tree = {
8 | label: '.',
9 | nodes: [{
10 | label: 'datapackages',
11 | nodes: []
12 | }]
13 | };
14 |
15 | dpkgs.forEach(function (dpkg) {
16 | tree.nodes[0].nodes.push(getDpkgNode(dpkg));
17 | });
18 |
19 | return archy(tree);
20 | };
21 |
22 | exports.dpkg = function (dpkg) {
23 |
24 | var tree = {
25 | label: '.',
26 | nodes: [{
27 | label: dpkg.name,
28 | nodes: ['datapackage.json']
29 | }]
30 | };
31 |
32 | var dataNodes = getDataNodes(dpkg);
33 | if (dataNodes) {
34 | tree.nodes[0].nodes.push(dataNodes);
35 | }
36 |
37 | return archy(tree);
38 | };
39 |
40 | exports.clone = function (paths) {
41 | return archy(reformat(unflatten(paths))[0]);
42 | };
43 |
44 | function getDpkgNode(dpkg) {
45 | var node = {
46 | label: dpkg.name,
47 | nodes: ['datapackage.json']
48 | };
49 |
50 | var dataNodes = getDataNodes(dpkg);
51 | if (dataNodes) {
52 | node.nodes.push(dataNodes);
53 | }
54 | return node;
55 | }
56 |
57 | function getDataNodes(dpkg) {
58 |
59 | var dataNodes = [];
60 | dpkg.resources = dpkg.resources || [];
61 | dpkg.resources.forEach(function (r) {
62 | if ('path' in r) {
63 | dataNodes.push(path.basename(r.path));
64 | }
65 | });
66 |
67 | if (dataNodes.length) {
68 | return {
69 | label: 'data',
70 | nodes: dataNodes
71 | };
72 | }
73 | }
74 |
75 | function unflatten(paths) {
76 |
77 | var obj = {};
78 |
79 | paths.forEach(function (p) {
80 | obj[p] = path.basename(p);
81 | });
82 |
83 | return flat.unflatten(obj, { delimiter: path.sep });
84 | }
85 |
86 | /**
87 | * inspired from https://github.com/hughsk/file-tree
88 | */
89 | function reformat(object) {
90 | if (typeof object !== 'object') return object;
91 |
92 | var entries = [];
93 | var entry;
94 |
95 | for (var key in object) {
96 | entry = reformat(object[key]);
97 | if (typeof entry === 'string') {
98 | entry.label = key;
99 | entries.push(entry);
100 | } else {
101 | entry = { nodes: entry, label: key };
102 | entries.push(entry);
103 | }
104 | }
105 | return entries;
106 | }
107 |
--------------------------------------------------------------------------------
/lib/util.js:
--------------------------------------------------------------------------------
1 | var urlmod = require('url');
2 |
3 | exports.parseSpecString = function (specString) {
4 | out = {
5 | url: '',
6 | name: '',
7 | version: ''
8 | }
9 | var path = specString;
10 | if (specString.indexOf('http') != -1) {
11 | out.url = specString;
12 | var urlparts = urlmod.parse(specString);
13 | path = urlparts.pathname;
14 | }
15 | var parts = path.split('/');
16 | var name = parts.pop();
17 | var _tmp = name.split('@');
18 | out.name = _tmp[0];
19 | if (_tmp.length > 1) {
20 | out.version = _tmp.split('@');
21 | }
22 | return out;
23 | }
24 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "dpmjs",
3 | "version": "0.8.2",
4 | "description": "Data Package manager (dpm) library and command line tool",
5 | "main": "index.js",
6 | "bin": {
7 | "dpm": "bin/dpm"
8 | },
9 | "preferGlobal": true,
10 | "scripts": {
11 | "test": "node_modules/.bin/mocha"
12 | },
13 | "repository": {
14 | "type": "git",
15 | "url": "git://github.com/frictionlessdata/dpm-js.git"
16 | },
17 | "keywords": [
18 | "hyperdata",
19 | "package",
20 | "datapackage",
21 | "client",
22 | "data",
23 | "registry",
24 | "npm"
25 | ],
26 | "author": "Rufus Pollock (http://rufuspollock.org)",
27 | "license": "MIT",
28 | "bugs": {
29 | "url": "https://github.com/frictionlessdata/dpm-js/issues"
30 | },
31 | "homepage": "https://github.com/frictionlessdata/dpm-js",
32 | "dependencies": {
33 | "temp": "~0.6.0",
34 | "fstream-ignore": "0.0.7",
35 | "tar": "~0.1.18",
36 | "async": "~0.2.9",
37 | "mime": "~1.2.11",
38 | "rc": "~0.3.2",
39 | "optimist": "~0.6.0",
40 | "cookie": "~0.1.0",
41 | "clone": "~0.1.11",
42 | "read": "~1.0.5",
43 | "colors": "~0.6.2",
44 | "split": "~0.2.10",
45 | "semver": "~2.2.1",
46 | "request": "~2.30.0",
47 | "rimraf": "~2.2.5",
48 | "mkdirp": "~0.3.5",
49 | "concat-stream": "~1.2.1",
50 | "once": "~1.3.0",
51 | "archy": "0.0.2",
52 | "flat": "~1.0.0",
53 | "fs-readdir-recursive": "0.0.1",
54 | "datapackage-init": ">=0.2.3",
55 | "datapackage-validate": ">=0.2.5",
56 | "datapackage-read": ">=0.2.1",
57 | "datapackage-identifier": ">=0.3.0",
58 | "datapackage-ckan": ">=0.2.1",
59 | "glob": "~3.2.7",
60 | "lodash.uniq": "~2.4.1",
61 | "lodash.flatten": "~2.4.1",
62 | "jts-infer": "0.0.0"
63 | },
64 | "devDependencies": {
65 | "lodash.difference": "~2.4.1",
66 | "mocha": "~2.4.5",
67 | "sinon": "~1.17.3"
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/test/all.js:
--------------------------------------------------------------------------------
1 | var assert = require('assert')
2 | , sinon = require('sinon')
3 | , colors = require('colors')
4 | , path = require('path')
5 | , fs = require('fs')
6 | , rimraf = require('rimraf')
7 | , dpm = require('../lib/index')
8 | , util = require('../lib/util')
9 | ;
10 |
11 | describe('get', function() {
12 | var root = path.join('/tmp', 'dpm-test');
13 | this.timeout(8000);
14 |
15 | beforeEach(function(done) {
16 | rimraf(root, done);
17 | });
18 |
19 | it('works on urls', function(done) {
20 | var ours = new dpm({}, root);
21 | var url = 'http://data.okfn.org/data/country-codes';
22 | var dpjson = path.join(root, 'datapackages', 'country-codes', 'datapackage.json')
23 | var datapath = path.join(root, 'datapackages', 'country-codes', 'data', 'country-codes.csv')
24 | // var url = 'http://localhost/datasets/country-codes';
25 | ours.get(url, function(err) {
26 | if (err) done(err);
27 | var dpkg = JSON.parse(fs.readFileSync(dpjson));
28 | var data = fs.readFileSync(datapath, 'utf8');
29 | assert.equal(dpkg.name, 'country-codes');
30 | assert.equal(data.slice(0,18), 'name,official_name');
31 | done();
32 | });
33 | });
34 |
35 | it('works on github', function(done) {
36 | var ours = new dpm({}, root);
37 | var url = 'https://github.com/datasets/country-codes';
38 | var dpjson = path.join(root, 'datapackages', 'country-codes', 'datapackage.json')
39 | var datapath = path.join(root, 'datapackages', 'country-codes', 'data', 'country-codes.csv')
40 | // var url = 'http://localhost/datasets/country-codes';
41 | ours.get(url, function(err) {
42 | if (err) done(err);
43 | var dpkg = JSON.parse(fs.readFileSync(dpjson));
44 | var data = fs.readFileSync(datapath, 'utf8');
45 | assert.equal(dpkg.name, 'country-codes');
46 | assert.equal(data.slice(0,18), 'name,official_name');
47 | done();
48 | });
49 | });
50 |
51 | it('should ignore data resource', function(done) {
52 | var ours = new dpm({}, root);
53 | var url = 'https://raw.githubusercontent.com/okfn/dpm/master/test/fixtures/datapackage-example-inline';
54 | var dpjson = path.join(root, 'datapackages', 'datapackage-example-inline', 'datapackage.json');
55 |
56 | ours.get(url, function(err) {
57 | if (err) return done(err);
58 |
59 | var dpkg = JSON.parse(fs.readFileSync(dpjson));
60 | assert.equal(dpkg.name, 'datapackage-example-inline');
61 | done();
62 | });
63 | });
64 | });
65 |
66 | describe('info', function() {
67 | it('fixture', function(done) {
68 | var ours = new dpm({}, root);
69 | var argv = {
70 | identifier: path.resolve('test/fixtures/mydpkg-test')
71 | };
72 | ours.info(argv, function(err, result) {
73 | assert(!err, err);
74 | assert.equal(result.json.name, 'mydpkg-test');
75 | done();
76 | });
77 | });
78 | });
79 |
80 | describe('validate', function() {
81 | var consoleLogSpy;
82 | beforeEach(function() {
83 | consoleLogSpy = sinon.spy(console, 'log');
84 | });
85 |
86 | afterEach(function() {
87 | consoleLogSpy.restore();
88 | });
89 |
90 | it('should return valid with a valid datapackage.json', function(done) {
91 | var _dpm = new dpm({}, root);
92 | var _path = path.resolve('test/fixtures/mydpkg-test');
93 | return _dpm.validate(_path).then(function (results) {
94 | assert.equal(results.valid, true);
95 | assert.equal(results.errors.length, 0);
96 | assert.ok(consoleLogSpy.calledWith('DataPackage.json is Valid'.green));
97 | done();
98 | });
99 | });
100 |
101 | it('should return invalid with a invalid datapackage.json', function(done) {
102 | var _dpm = new dpm({}, root);
103 | var _path = path.resolve('test/fixtures/mydpkg-invalid-test');
104 | var expectedResult = {
105 | "valid": false,
106 | "errors": [
107 | {
108 | "message": "Missing required property: name",
109 | "params": {
110 | "key": "name"
111 | },
112 | "code": 302,
113 | "dataPath": "",
114 | "schemaPath": "/required/0",
115 | "subErrors": null,
116 | "type": "schema"
117 | }
118 | ]
119 | };
120 |
121 | return _dpm.validate(_path).then(function (results) {
122 | assert.equal(results.valid, false);
123 | assert.equal(results.errors.length, 1);
124 | assert.ok(consoleLogSpy.calledWith('DataPackage.json is Invalid'.red));
125 | assert.ok(consoleLogSpy.calledWith(JSON.stringify(expectedResult, null, 2)));
126 | done();
127 | });
128 | });
129 | });
130 |
--------------------------------------------------------------------------------
/test/fixtures/datapackage-example-inline/README.md:
--------------------------------------------------------------------------------
1 | This is a basic example of using an inline data element in a datapackage. Adapted from the documentation.
2 |
--------------------------------------------------------------------------------
/test/fixtures/datapackage-example-inline/datapackage.json:
--------------------------------------------------------------------------------
1 | {
2 | "name" : "datapackage-example-inline",
3 | "license" : "CC0",
4 | "resources": [
5 | {
6 | "format": "csv",
7 | "data": "A,B,C\n1,2,3\n4,5,6"
8 | }
9 | ]
10 | }
11 |
--------------------------------------------------------------------------------
/test/fixtures/mydpkg-invalid-test/datapackage.json:
--------------------------------------------------------------------------------
1 | {
2 | "description": "my datapackage",
3 | "version": "0.0.0",
4 | "keywords": ["test", "datapackage"],
5 |
6 | "resources": [
7 | {
8 | "name": "inline",
9 | "schema": {
10 | "fields": [
11 | {"name": "a", "type": "string"},
12 | {"name": "b", "type": "integer"},
13 | {"name": "c", "type": "number"}
14 | ]
15 | },
16 | "data": [
17 | {"a": "a", "b": 1, "c": 1.2},
18 | {"a": "x", "b": 2, "c": 2.3},
19 | {"a": "y", "b": 3, "c": 3.4}
20 | ]
21 | },
22 | {
23 | "name": "csv1",
24 | "format": "csv",
25 | "schema": {
26 | "fields": [
27 | {"name": "a", "type": "integer"},
28 | {"name": "b", "type": "integer"}
29 | ]
30 | },
31 | "path": "x1.csv"
32 | },
33 | {
34 | "name": "csv2",
35 | "format": "csv",
36 | "schema": {
37 | "fields": [
38 | {"name": "c", "type": "integer"},
39 | {"name": "d", "type": "integer"}
40 | ]
41 | },
42 | "path": "x2.csv"
43 | }
44 | ]
45 | }
46 |
--------------------------------------------------------------------------------
/test/fixtures/mydpkg-test/datapackage.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mydpkg-test",
3 | "description": "my datapackage",
4 | "version": "0.0.0",
5 | "keywords": ["test", "datapackage"],
6 |
7 | "resources": [
8 | {
9 | "name": "inline",
10 | "schema": {
11 | "fields": [
12 | {"name": "a", "type": "string"},
13 | {"name": "b", "type": "integer"},
14 | {"name": "c", "type": "number"}
15 | ]
16 | },
17 | "data": [
18 | {"a": "a", "b": 1, "c": 1.2},
19 | {"a": "x", "b": 2, "c": 2.3},
20 | {"a": "y", "b": 3, "c": 3.4}
21 | ]
22 | },
23 | {
24 | "name": "csv1",
25 | "format": "csv",
26 | "schema": {
27 | "fields": [
28 | {"name": "a", "type": "integer"},
29 | {"name": "b", "type": "integer"}
30 | ]
31 | },
32 | "path": "x1.csv"
33 | },
34 | {
35 | "name": "csv2",
36 | "format": "csv",
37 | "schema": {
38 | "fields": [
39 | {"name": "c", "type": "integer"},
40 | {"name": "d", "type": "integer"}
41 | ]
42 | },
43 | "path": "x2.csv"
44 | }
45 | ]
46 | }
47 |
--------------------------------------------------------------------------------
/test/fixtures/mydpkg-test/scripts/test.r:
--------------------------------------------------------------------------------
1 | print('hello');
2 |
--------------------------------------------------------------------------------
/test/fixtures/mydpkg-test/x1.csv:
--------------------------------------------------------------------------------
1 | "a","b"
2 | 1,2
3 | 3,4
4 |
--------------------------------------------------------------------------------
/test/fixtures/mydpkg-test/x2.csv:
--------------------------------------------------------------------------------
1 | "c","d"
2 | 5,6
3 | 7,8
4 |
--------------------------------------------------------------------------------
/test/fixtures/req-test/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "req-test",
3 | "description": "a test for require",
4 | "dataDependencies": {
5 | "mydpkg-test": "0.0.0"
6 | },
7 | "version": "0.0.0",
8 | "keywords": ["test", "datapackage"],
9 |
10 | "resources": [
11 | {
12 | "name": "azerty",
13 | "require": {"datapackage": "mydpkg-test", "resource": "csv1", "fields": ["a"]}
14 | }
15 | ]
16 | }
17 |
--------------------------------------------------------------------------------