16 | {% if status %}
17 | HTTP status {{status}}:
18 | {% endif %}
19 | {% if text %}
20 | {{text}}
21 | {% endif %}
22 |
23 |
24 |
25 |
--------------------------------------------------------------------------------
/tools/dspl2/dspl2/templates/render.html:
--------------------------------------------------------------------------------
1 | {% macro render(obj) %}
2 | {% if obj.items %}
3 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
28 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/tools/dspltools/examples/dsplgen/dsplgen_advanced.csv:
--------------------------------------------------------------------------------
1 | date[type=date;format=MM/dd/yyyy],first_category[slice_role=dimension;rollup=true;total_val=total],second_category[slice_role=dimension;rollup=true],first_value[slice_role=metric;type=integer],second_value[slice_role=metric;type=float]
2 | 1/1/2010,red,tall,10,23
3 | 1/1/2010,red,short,90,1
4 | 1/1/2010,blue,tall,12,31
5 | 1/1/2010,blue,short,21,231
6 | 1/1/2010,green,short,20,212
7 | 1/1/2010,total,tall,10,98
8 | 1/1/2010,total,short,-30,39
9 | 1/2/2010,red,tall,10,91
10 | 1/2/2010,red,short,32,123
11 | 1/2/2010,blue,tall,22,121
12 | 1/2/2010,blue,short,20,32
13 | 1/2/2010,green,short,1,19
14 | 1/2/2010,total,short,2,10
15 | 1/3/2010,red,short,10,34
16 | 1/3/2010,red,tall,10,34
17 | 1/3/2010,blue,short,93,21
18 | 1/3/2010,blue,tall,39,12
19 | 1/3/2010,green,short,31,31
20 | 1/3/2010,green,tall,21,31
21 | 1/3/2010,total,short,13,123
22 | 1/4/2010,red,tall,40,21
23 | 1/4/2010,red,short,22,12
24 | 1/4/2010,blue,tall,39,21
25 | 1/4/2010,blue,short,10,12
26 | 1/4/2010,green,tall,30,23
27 | 1/4/2010,green,short,10,123
28 | 1/4/2010,total,tall,-10,23
29 | 1/4/2010,total,short,31,661
30 |
--------------------------------------------------------------------------------
/tools/dspltools/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Setup script for the DSPLtools suite."""
10 |
11 | import setuptools
12 | from distutils.core import setup
13 |
14 |
15 | setup(name='dspltools',
16 | version='0.5.0',
17 | description='Suite of command-line tools for generating DSPL datasets',
18 | author='Public Statistics',
19 | author_email='public-data-import-feedback@google.com',
20 | url='http://github.com/google/dspl',
21 | packages=['dspllib', 'dspllib.data_sources',
22 | 'dspllib.model', 'dspllib.validation'],
23 | package_dir={'dspllib': 'packages/dspllib'},
24 | package_data={'dspllib.validation': ['schemas/*.xsd',
25 | 'test_dataset/*.csv',
26 | 'test_dataset/*.xml']},
27 | scripts=['scripts/dsplcheck.py', 'scripts/dsplgen.py',
28 | 'scripts/run_all_tests.py'])
29 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/data_sources/csv_data_source_test.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Tests of csv_data_source module."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken
'
13 |
14 | import unittest
15 |
16 | import csv_data_source
17 | import csv_sources_test_suite
18 |
19 |
20 | class CSVDataSourceTests(csv_sources_test_suite.CSVSourcesTests):
21 | """Tests of the CSVDataSource object."""
22 |
23 | def setUp(self):
24 | self.data_source_class = csv_data_source.CSVDataSource
25 |
26 | super(CSVDataSourceTests, self).setUp()
27 |
28 |
29 | class CSVDataSourceErrorTests(csv_sources_test_suite.CSVSourcesErrorTests):
30 | """Tests of the CSVDataSource object under various error conditions."""
31 |
32 | def setUp(self):
33 | self.data_source_class = csv_data_source.CSVDataSource
34 |
35 | super(CSVDataSourceErrorTests, self).setUp()
36 |
37 |
38 | if __name__ == '__main__':
39 | unittest.main()
40 |
--------------------------------------------------------------------------------
/tools/dspl2/dspl2/templates/viewer.js:
--------------------------------------------------------------------------------
1 | for (var td of document.querySelectorAll('td:first-child')) {
2 | var sibling = td.nextElementSibling;
3 | if (sibling) {
4 | if (sibling.querySelector('table')) {
5 | if (sibling.children.length < 20) {
6 | td.classList.toggle('open');
7 | } else {
8 | td.classList.toggle('closed');
9 | sibling.classList.toggle('hidden');
10 | }
11 | td.addEventListener('click', (ev) => {
12 | ev.target.classList.toggle('open');
13 | ev.target.classList.toggle('closed');
14 | ev.target.nextElementSibling.classList.toggle('hidden');
15 | });
16 | }
17 | }
18 | }
19 |
20 | function onclick(ev) {
21 | document.querySelectorAll('h2').forEach((elt) => {
22 | elt.classList.remove('active');
23 | });
24 | ev.target.classList.add('active');
25 |
26 | document.querySelectorAll('div').forEach((elt) => {
27 | elt.classList.add('hidden');
28 | });
29 | document.querySelector('div#'+ev.target.textContent.trim().toLowerCase()).classList.remove('hidden');
30 | }
31 |
32 | document.querySelectorAll('h2').forEach((elt) => {
33 | elt.addEventListener('click', onclick);
34 | });
35 |
--------------------------------------------------------------------------------
/samples/us_census/retail_sales/businesses.csv:
--------------------------------------------------------------------------------
1 | "business","name","parent"
2 | "44x72","Retail and Food services",
3 | "44000","Retail services","44x72"
4 | "44100","Motor Vehicle and Parts Dealers","44000"
5 | "44200","Furniture and Home Furnishings Stores","44000"
6 | "44300","Electronics and Appliance Stores","44000"
7 | "44400","Building Material and Garden Equipment and Supplies Dealers","44000"
8 | "44500","Food and Beverage Stores","44000"
9 | "44510","Grocery Stores","44500"
10 | "44600","Health and Personal Care Stores","44000"
11 | "44700","Gasoline Stations","44000"
12 | "44800","Clothing and Clothing Accessories Stores","44000"
13 | "45100","Sporting Goods, Hobby, Book, and Music Stores","44000"
14 | "45200","General Merchandise Stores","44000"
15 | "45210","Department Stores (excluding leased department stores)","45200"
16 | "45300","Miscellaneous Store Retailers","44000"
17 | "45400","Nonstore Retailers","44000"
18 | "72200","Food Services and Drinking Places","44x72"
19 | "44xxx","Other Aggregates",
20 | "44y72","Retail and Food services (excluding motor vehicles)","44xxx"
21 | "4400a","Retail Services (excluding Motor Vehicle and Parts Dealers)","44xxx"
22 | "441x0","Auto and other Motor Vehicle","44xxx"
23 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement. You (or your employer) retain the copyright to your contribution;
10 | this simply gives us permission to use and redistribute your contributions as
11 | part of the project. Head over to to see
12 | your current agreements on file or to sign a new one.
13 |
14 | You generally only need to submit a CLA once, so if you've already submitted one
15 | (even if it was for a different project), you probably don't need to do it
16 | again.
17 |
18 | ## Code reviews
19 |
20 | All submissions, including submissions by project members, require review. We
21 | use GitHub pull requests for this purpose. Consult
22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
23 | information on using pull requests.
24 |
25 | ## Community Guidelines
26 |
27 | This project follows [Google's Open Source Community
28 | Guidelines](https://opensource.google.com/conduct/).
29 |
--------------------------------------------------------------------------------
/samples/bls/unemployment/footnotes.csv:
--------------------------------------------------------------------------------
1 | codeValue,description
2 | 1,Data affected by changes in population controls.
3 | 2,Constructed on the 2002 Census Industry Classification from data originally coded on earlier classifications. Official series was not revised.
4 | 3,2000 forward coded on the 2002 Census Occupation Classification. 1983-99 constructed from data originally coded on earlier classifications.
5 | 4,2000 forward coded on the 2002 Census Industry Classification. 1983-99 constructed from data originally coded on earlier classifications.
6 | 7,Data do not meet publication criteria.
7 | 8,This series id code has been discontinued; data are available using the database tool at www.bls.gov/webapps/legacy/cpsatab8.htm.
8 | 9,Data from 1994 through 2002 were revised in February 2014 with updated seasonal adjustments.
9 | A,Area boundaries do not reflect official OMB definitions.
10 | N,Not available.
11 | P,Preliminary.
12 | V,The survey was not conducted due to bad weather. Interpolated data were seasonally adjusted.
13 | W,The household survey was not conducted for this month due to bad weather. Data were interpolated.
14 | Y,Data reflect controlling to interpolated statewide totals because the survey was not conducted.
15 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/data_sources/csv_data_source_sqlite_test.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Tests of csv_data_source_sqlite module."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | import unittest
15 |
16 | import csv_data_source_sqlite
17 | import csv_sources_test_suite
18 |
19 |
20 | class CSVDataSourceSqliteTests(csv_sources_test_suite.CSVSourcesTests):
21 | """Tests of the CSVDataSourceSqlite object."""
22 |
23 | def setUp(self):
24 | self.data_source_class = csv_data_source_sqlite.CSVDataSourceSqlite
25 |
26 | super(CSVDataSourceSqliteTests, self).setUp()
27 |
28 |
29 | class CSVDataSourceSqliteErrorTests(
30 | csv_sources_test_suite.CSVSourcesErrorTests):
31 | """Tests of the CSVDataSourceSqlite object under various error conditions."""
32 |
33 | def setUp(self):
34 | self.data_source_class = csv_data_source_sqlite.CSVDataSourceSqlite
35 |
36 | super(CSVDataSourceSqliteErrorTests, self).setUp()
37 |
38 |
39 | if __name__ == '__main__':
40 | unittest.main()
41 |
--------------------------------------------------------------------------------
/docs/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | ---
2 | title: Contributing to Data Set Publishing Language, Version 2.0
3 | author: Google
4 | ---
5 | # How to Contribute
6 |
7 | We'd love to accept your patches and contributions to this project. There are
8 | just a few small guidelines you need to follow.
9 |
10 | ## Contributor License Agreement
11 |
12 | Contributions to this project must be accompanied by a Contributor License
13 | Agreement. You (or your employer) retain the copyright to your contribution;
14 | this simply gives us permission to use and redistribute your contributions as
15 | part of the project. Head over to to see
16 | your current agreements on file or to sign a new one.
17 |
18 | You generally only need to submit a CLA once, so if you've already submitted one
19 | (even if it was for a different project), you probably don't need to do it
20 | again.
21 |
22 | ## Code reviews
23 |
24 | All submissions, including submissions by project members, require review. We
25 | use GitHub pull requests for this purpose. Consult
26 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
27 | information on using pull requests.
28 |
29 | ## Community Guidelines
30 |
31 | This project follows [Google's Open Source Community
32 | Guidelines](https://opensource.google.com/conduct/).
33 |
--------------------------------------------------------------------------------
/tools/dspltools/scripts/run_all_tests.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Run all tests defined in the DSPL Tools code."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | import unittest
15 |
16 | _TEST_MODULE_NAMES = [
17 | 'dsplcheck_test',
18 | 'dsplgen_test',
19 | 'dspllib.data_sources.csv_data_source_test',
20 | 'dspllib.data_sources.csv_data_source_sqlite_test',
21 | 'dspllib.data_sources.data_source_test',
22 | 'dspllib.data_sources.data_source_to_dspl_test',
23 | 'dspllib.model.dspl_model_loader_test',
24 | 'dspllib.model.dspl_model_test',
25 | 'dspllib.validation.dspl_validation_test',
26 | 'dspllib.validation.xml_validation_test']
27 |
28 |
29 | def main():
30 | """Run all DSPL Tools tests and print the results to stderr."""
31 | test_suite = unittest.TestSuite()
32 |
33 | for test_module_name in _TEST_MODULE_NAMES:
34 | test_suite.addTests(
35 | unittest.defaultTestLoader.loadTestsFromName(test_module_name))
36 |
37 | unittest.TextTestRunner().run(test_suite)
38 |
39 |
40 | if __name__ == '__main__':
41 | main()
42 |
--------------------------------------------------------------------------------
/tools/dspl2/dspl2/templates/display.html:
--------------------------------------------------------------------------------
1 | {% from 'render.html' import render %}
2 |
3 |
4 | DSPL 2 Viewer
5 |
8 |
9 |
10 | DSPL 2 Viewer
11 | Dataset
12 | Dimensions
13 | Measures
14 | Footnotes
15 | Slices
16 |
17 | {{render(dataset)}}
18 |
19 |
20 |
21 |
22 | | dimensions |
23 | {{ render(dimension) }} |
24 |
25 |
26 |
28 |
29 |
30 | | measures |
31 | {{ render(measure) }} |
32 |
33 |
34 |
46 |
47 |
48 | | slices |
49 | {{ render(slice) }} |
50 |
51 |
52 |
85 |
86 |
87 |
88 |
89 |
90 | US
91 |
92 |
93 | states.csv
94 |
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/tools/dspl2/dspl2/tests/test_expander.py:
--------------------------------------------------------------------------------
1 | from dspl2.expander import Dspl2JsonLdExpander, Dspl2RdfExpander
2 | from dspl2.rdfutil import SCHEMA
3 | from io import StringIO
4 | import rdflib
5 | import unittest
6 |
7 |
8 | class DummyGetter(object):
9 | def __init__(self, graph):
10 | self.graph = graph
11 | self.data = {}
12 |
13 | def Set(self, filename, data):
14 | self.data[filename] = StringIO(data)
15 |
16 | def Fetch(self, filename):
17 | return self.data.get(filename, StringIO(''))
18 |
19 |
20 | class ExpanderTests(unittest.TestCase):
21 | def test_Dspl2RdfExpander_ExpandDimensionValue(self):
22 | graph = rdflib.Graph()
23 | getter = DummyGetter(graph)
24 | expander = Dspl2RdfExpander(getter)
25 | dim = rdflib.URIRef('http://foo.invalid/test.json#dim')
26 | equiv_types = [SCHEMA.Place]
27 | row = {
28 | 'codeValue': 'cv',
29 | 'key1': 'val1',
30 | 'key2': 'val2',
31 | }
32 | row_id = rdflib.URIRef(str(dim) + '=' + row['codeValue'])
33 | expander._ExpandDimensionValue(dim, equiv_types, row_id, row)
34 | self.assertEqual(set(graph.objects(subject=dim, predicate=SCHEMA.codeList)),
35 | {row_id})
36 | self.assertEqual(set(graph.objects(subject=row_id,
37 | predicate=rdflib.RDF.type)),
38 | {SCHEMA.DimensionValue, SCHEMA.Place})
39 | self.assertEqual(set(graph.objects(subject=row_id,
40 | predicate=rdflib.RDF.type)),
41 | {SCHEMA.DimensionValue, SCHEMA.Place})
42 | self.assertEqual(set(graph.objects(subject=row_id, predicate=SCHEMA.key1)),
43 | {rdflib.Literal('val1')})
44 | self.assertEqual(set(graph.objects(subject=row_id, predicate=SCHEMA.key2)),
45 | {rdflib.Literal('val2')})
46 | self.assertEqual(set(graph.objects(subject=row_id,
47 | predicate=SCHEMA.codeValue)),
48 | {rdflib.Literal('cv')})
49 |
50 | def test_Dspl2RdfExpander_ExpandFootnotes(self):
51 | graph = rdflib.Graph()
52 | dim = rdflib.URIRef('#ds')
53 | graph.add((dim, rdflib.RDF.type, SCHEMA.StatisticalDataset))
54 | graph.add((dim, SCHEMA.footnote, rdflib.Literal('foo')))
55 | getter = DummyGetter(graph)
56 | getter.Set('foo', 'codeValue,name,description\np,predicted,Value is predicted rather than measured.\n')
57 | expander = Dspl2RdfExpander(getter)
58 | expander._ExpandFootnotes()
59 | for triple in graph:
60 | print(triple)
61 | footnote_id = rdflib.URIRef('#footnote=p')
62 | self.assertEqual(set(graph.objects(subject=dim,
63 | predicate=SCHEMA.footnote)),
64 | {footnote_id})
65 | self.assertEqual(set(graph.objects(subject=footnote_id,
66 | predicate=SCHEMA.description)),
67 | {rdflib.term.Literal('Value is predicted rather than measured.')})
68 | self.assertEqual(set(graph.objects(subject=footnote_id,
69 | predicate=SCHEMA.name)),
70 | {rdflib.term.Literal('predicted')})
71 | self.assertEqual(set(graph.objects(subject=footnote_id,
72 | predicate=rdflib.RDF.type)),
73 | {SCHEMA.StatisticalAnnotation})
74 | self.assertEqual(set(graph.objects(subject=footnote_id,
75 | predicate=SCHEMA.codeValue)),
76 | {rdflib.term.Literal('p')})
77 |
78 | def test_Dspl2RdfExpander_ExpandSliceData(self):
79 | pass
80 |
81 | def test_Dspl2JsonLdExpander_ExpandCodeList(self):
82 | pass
83 |
84 | def test_Dspl2JsonLdExpander_ExpandFootnotes(self):
85 | pass
86 |
87 | def test_Dspl2JsonLdExpander_ExpandSliceData(self):
88 | pass
89 |
90 |
91 | if __name__ == '__main__':
92 | unittest.main()
93 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/validation/xml_validation.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Validate a DSPL XML file."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | from lxml import etree
15 | import os.path
16 | import re
17 |
18 |
19 | # The number of lines of context to show around XML errors
20 | _CONTEXT_LINES = 3
21 |
22 | _SCHEMA_PATH = os.path.join(os.path.split(__file__)[0], 'schemas')
23 | _DSPL_SCHEMA_FILE = 'dspl.xsd'
24 |
25 |
26 | def GetErrorContext(xml_string, error_line_number):
27 | """Generate a string that shows the context of an XML error.
28 |
29 | Args:
30 | xml_string: String containing the contents of an XML file
31 | error_line_number: 1-indexed line number on which error has been detected
32 |
33 | Returns:
34 | A pretty-printed string containing the lines around the error
35 | """
36 | min_error_start_line = (error_line_number - 1) - _CONTEXT_LINES
37 | max_error_end_line = (error_line_number - 1) + _CONTEXT_LINES
38 |
39 | error_context_lines = []
40 |
41 | for l, line in enumerate(xml_string.splitlines()):
42 | if l >= min_error_start_line:
43 | line_string = '%5d' % (l + 1)
44 |
45 | # Highlight the error line with asterisks
46 | if (l + 1) == error_line_number:
47 | line_string = line_string.replace(' ', '*')
48 |
49 | error_context_lines.append('%s: %s' % (line_string, line.rstrip()))
50 |
51 | if l >= max_error_end_line:
52 | break
53 |
54 | return '\n'.join(error_context_lines)
55 |
56 |
57 | def GetErrorLineNumber(error_string):
58 | """Parse out the line number from a minixsv error message.
59 |
60 | Args:
61 | error_string: String returned by minixsv exception
62 |
63 | Returns:
64 | Integer line number on which error was detected
65 | """
66 | line_match = re.search(': line ([0-9]+)', error_string)
67 |
68 | return int(line_match.group(1))
69 |
70 |
71 | def RunValidation(xml_file, schema_file=None, verbose=True):
72 | """Run the validation process and return a message with the result.
73 |
74 | Args:
75 | xml_file: An XML input file
76 | schema_file: A DSPL schema file; if not given, the default 'dspl.xsd' is
77 | used.
78 | verbose: Include helpful, extra information about validation
79 |
80 | Returns:
81 | String containing result of validation process
82 | """
83 | result = ''
84 |
85 | xml_file_text = xml_file.read()
86 |
87 | if schema_file:
88 | schema_file_text = schema_file.read()
89 | else:
90 | schema_file = open(os.path.join(_SCHEMA_PATH, _DSPL_SCHEMA_FILE), 'r')
91 | schema_file_text = schema_file.read()
92 | schema_file.close()
93 |
94 | # Insert proper paths into XSD schemaLocation tags
95 | substitution_function = (
96 | lambda m: 'schemaLocation="%s"' % os.path.join(_SCHEMA_PATH, m.group(1)))
97 |
98 | schema_file_text = re.sub(
99 | 'schemaLocation="([a-zA-Z_0-9.]+)"',
100 | substitution_function,
101 | schema_file_text, 2)
102 |
103 | # Parse the schema file into an etree
104 | schema_file_xml = etree.XML(schema_file_text)
105 |
106 | try:
107 | schema = etree.XMLSchema(schema_file_xml)
108 | parser = etree.XMLParser(schema=schema)
109 | etree.fromstring(xml_file_text, parser)
110 | except etree.XMLSyntaxError as xml_error:
111 | # XML parsing error
112 | error_string = str(xml_error)
113 | if verbose:
114 | result = ('Input does not validate against DSPL schema\n\n%s\n%s' %
115 | (error_string, GetErrorContext(
116 | xml_file_text,
117 | xml_error.lineno)))
118 | else:
119 | result = error_string
120 | else:
121 | if verbose:
122 | result = 'XML file validates successfully!'
123 |
124 | return result
125 |
--------------------------------------------------------------------------------
/samples/google/canonical/geo.xml:
--------------------------------------------------------------------------------
1 |
2 |
32 |
35 |
36 |
37 |
38 |
39 |
40 | Geographical concepts
41 |
42 |
43 | Canonical concepts for geographical data.
44 |
45 |
46 | http://code.google.com/apis/publicdata/docs/canonical/geo.html
47 |
48 |
49 |
50 |
51 |
52 | Google Inc.
53 |
54 |
55 | Google Inc.
56 |
57 |
58 | http://www.google.com
59 |
60 |
61 |
62 |
63 |
64 |
65 | Location
66 |
67 | Base concept for locations.
68 |
69 |
70 |
71 |
72 |
73 | Latitude
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 | Longitude
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 | Country or Territory
92 |
93 |
94 | A country or territory, identified by its ISO-3166-1 2-letter code.
95 |
96 |
97 | Countries
98 |
99 |
100 | World
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 | countries.csv
115 |
116 |
117 |
118 |
119 |
--------------------------------------------------------------------------------
/samples/google/canonical/currencies.csv:
--------------------------------------------------------------------------------
1 | currency,name,symbol
2 | AED,"UAE Dirham",
3 | AFN,Afghani,؋
4 | ALL,Lek,Lek
5 | AMD,"Armenian Dram",
6 | ANG,"Netherlands Antillian Guilder",ƒ
7 | AOA,Kwanza,
8 | ARS,"Argentine Peso",$
9 | AUD,"Australian Dollar",$
10 | AWG,"Aruban Guilder",ƒ
11 | AZN,"Azerbaijanian Manat",ман
12 | BAM,"Convertible Marks",KM
13 | BBD,"Barbados Dollar",$
14 | BDT,Taka,
15 | BGN,"Bulgarian Lev",лв
16 | BHD,"Bahraini Dinar",
17 | BIF,"Burundi Franc",
18 | BMD,"Bermudian Dollar (customarily known as Bermuda Dollar)",$
19 | BND,"Brunei Dollar",$
20 | "BOB BOV","Boliviano Mvdol",$b
21 | BRL,"Brazilian Real",R$
22 | BSD,"Bahamian Dollar",$
23 | BWP,Pula,P
24 | BYR,"Belarussian Ruble",p.
25 | BZD,"Belize Dollar",BZ$
26 | CAD,"Canadian Dollar",$
27 | CDF,"Congolese Franc",
28 | CHF,"Swiss Franc",CHF
29 | "CLP CLF","Chilean Peso Unidades de fomento",$
30 | CNY,"Yuan Renminbi",¥
31 | "COP COU","Colombian Peso Unidad de Valor Real",$
32 | CRC,"Costa Rican Colon",₡
33 | "CUP CUC","Cuban Peso Peso Convertible",₱
34 | CVE,"Cape Verde Escudo",
35 | CZK,"Czech Koruna",Kč
36 | DJF,"Djibouti Franc",
37 | DKK,"Danish Krone",kr
38 | DOP,"Dominican Peso",RD$
39 | DZD,"Algerian Dinar",
40 | EEK,Kroon,
41 | EGP,"Egyptian Pound",£
42 | ERN,Nakfa,
43 | ETB,"Ethiopian Birr",
44 | EUR,Euro,€
45 | FJD,"Fiji Dollar",$
46 | FKP,"Falkland Islands Pound",£
47 | GBP,"Pound Sterling",£
48 | GEL,Lari,
49 | GHS,Cedi,
50 | GIP,"Gibraltar Pound",£
51 | GMD,Dalasi,
52 | GNF,"Guinea Franc",
53 | GTQ,Quetzal,Q
54 | GYD,"Guyana Dollar",$
55 | HKD,"Hong Kong Dollar",$
56 | HNL,Lempira,L
57 | HRK,"Croatian Kuna",kn
58 | "HTG USD","Gourde US Dollar",
59 | HUF,Forint,Ft
60 | IDR,Rupiah,Rp
61 | ILS,"New Israeli Sheqel",₪
62 | INR,"Indian Rupee",
63 | "INR BTN","Indian Rupee Ngultrum",
64 | IQD,"Iraqi Dinar",
65 | IRR,"Iranian Rial",﷼
66 | ISK,"Iceland Krona",kr
67 | JMD,"Jamaican Dollar",J$
68 | JOD,"Jordanian Dinar",
69 | JPY,Yen,¥
70 | KES,"Kenyan Shilling",
71 | KGS,Som,лв
72 | KHR,Riel,៛
73 | KMF,"Comoro Franc",
74 | KPW,"North Korean Won",₩
75 | KRW,Won,₩
76 | KWD,"Kuwaiti Dinar",
77 | KYD,"Cayman Islands Dollar",$
78 | KZT,Tenge,лв
79 | LAK,Kip,₭
80 | LBP,"Lebanese Pound",£
81 | LKR,"Sri Lanka Rupee",₨
82 | LRD,"Liberian Dollar",$
83 | LTL,"Lithuanian Litas",Lt
84 | LVL,"Latvian Lats",Ls
85 | LYD,"Libyan Dinar",
86 | MAD,"Moroccan Dirham",
87 | MDL,"Moldovan Leu",
88 | MGA,"Malagasy Ariary",
89 | MKD,Denar,ден
90 | MMK,Kyat,
91 | MNT,Tugrik,₮
92 | MOP,Pataca,
93 | MRO,Ouguiya,
94 | MUR,"Mauritius Rupee",₨
95 | MVR,Rufiyaa,
96 | MWK,Kwacha,
97 | "MXN MXV","Mexican Peso Mexican Unidad de Inversion (UDI)",$
98 | MYR,"Malaysian Ringgit",RM
99 | MZN,Metical,MT
100 | NGN,Naira,₦
101 | NIO,"Cordoba Oro",C$
102 | NOK,"Norwegian Krone",kr
103 | NPR,"Nepalese Rupee",₨
104 | NZD,"New Zealand Dollar",$
105 | OMR,"Rial Omani",﷼
106 | "PAB USD","Balboa US Dollar",B/.
107 | PEN,"Nuevo Sol",S/.
108 | PGK,Kina,
109 | PHP,"Philippine Peso",Php
110 | PKR,"Pakistan Rupee",₨
111 | PLN,Zloty,zł
112 | PYG,Guarani,Gs
113 | QAR,"Qatari Rial",﷼
114 | RON,"New Leu",lei
115 | RSD,"Serbian Dinar",Дин.
116 | RUB,"Russian Ruble",руб
117 | RWF,"Rwanda Franc",
118 | SAR,"Saudi Riyal",﷼
119 | SBD,"Solomon Islands Dollar",$
120 | SCR,"Seychelles Rupee",₨
121 | SDG,"Sudanese Pound",
122 | SEK,"Swedish Krona",kr
123 | SGD,"Singapore Dollar",$
124 | SHP,"Saint Helena Pound",£
125 | SLL,Leone,
126 | SOS,"Somali Shilling",S
127 | SRD,"Surinam Dollar",$
128 | STD,Dobra,
129 | "SVC USD","El Salvador Colon US Dollar",$
130 | SYP,"Syrian Pound",£
131 | SZL,Lilangeni,
132 | THB,Baht,฿
133 | TJS,Somoni,
134 | TMT,Manat,
135 | TND,"Tunisian Dinar",
136 | TOP,Pa'anga,
137 | TRY,"Turkish Lira",TL
138 | TTD,"Trinidad and Tobago Dollar",TT$
139 | TWD,"New Taiwan Dollar",NT$
140 | TZS,"Tanzanian Shilling",
141 | UAH,Hryvnia,₴
142 | UGX,"Uganda Shilling",
143 | USD,"US Dollar",$
144 | "UYU UYI","Peso Uruguayo Uruguay Peso en Unidades Indexadas",$U
145 | UZS,"Uzbekistan Sum",лв
146 | VEF,"Bolivar Fuerte",Bs
147 | VND,Dong,₫
148 | VUV,Vatu,
149 | WST,Tala,
150 | XAF,"CFA Franc BEAC",
151 | XAG,Silver,
152 | XAU,Gold,
153 | XBA,"Bond Markets Units European Composite Unit (EURCO)",
154 | XBB,"European Monetary Unit (E.M.U.-6)",
155 | XBC,"European Unit of Account 9(E.U.A.-9)",
156 | XBD,"European Unit of Account 17(E.U.A.-17)",
157 | XCD,"East Caribbean Dollar",$
158 | XDR,SDR,
159 | XFU,UIC-Franc,
160 | XOF,"CFA Franc BCEAO",
161 | XPD,Palladium,
162 | XPF,"CFP Franc",
163 | XPT,Platinum,
164 | XTS,"Codes specifically reserved for testing purposes",
165 | YER,"Yemeni Rial",﷼
166 | ZAR,Rand,R
167 | "ZAR LSL","Rand Loti",
168 | "ZAR NAD","Rand Namibia Dollar",
169 | ZMK,"Zambian Kwacha",
170 | ZWL,"Zimbabwe Dollar",
171 |
--------------------------------------------------------------------------------
/tools/dspl2/dspl2/rdfutil.py:
--------------------------------------------------------------------------------
1 | # Copyright 2018 Google LLC
2 | #
3 | # Use of this source code is governed by a BSD-style
4 | # license that can be found in the LICENSE file or at
5 | # https://developers.google.com/open-source/licenses/bsd
6 |
7 | import json
8 | from pathlib import Path
9 | from pyld import jsonld
10 | from rdflib import Graph, Namespace
11 | from rdflib.serializer import Serializer
12 | import sys
13 |
14 | from dspl2.jsonutil import AsList
15 |
16 |
17 | SCHEMA = Namespace('http://schema.org/')
18 |
19 |
20 | _Schema = {}
21 | _Context = {}
22 | _DataFileFrame = {
23 | '@context': [_Context, {'schema': 'http://schema.org/'}],
24 | '@type': 'StatisticalDataset',
25 | }
26 | _FullFrame = {
27 | '@context': [_Context, {'schema': 'http://schema.org/'}],
28 | '@type': 'StatisticalDataset',
29 | 'slice': {
30 | 'dimension': {
31 | '@embed': '@never'
32 | },
33 | 'measure': {
34 | '@embed': '@never'
35 | },
36 | 'tableMapping': {
37 | 'sourceEntity': {
38 | '@embed': '@never'
39 | }
40 | },
41 | 'data': {
42 | 'dimensionValue': {
43 | 'dimension': {
44 | '@embed': '@never'
45 | }
46 | },
47 | 'measureValue': {
48 | 'measure': {
49 | '@embed': '@never'
50 | },
51 | 'footnote': {
52 | '@embed': '@never'
53 | }
54 | }
55 | }
56 | }
57 | }
58 | _Initialized = False
59 | _Module_path = Path(__file__).parent
60 | _RdfPrefixes = {
61 | 'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
62 | 'rdfs': 'http://www.w3.org/2000/01/rdf-schema#',
63 | 'schema': 'http://schema.org/',
64 | }
65 |
66 |
67 | def _Init(context, schema):
68 | global _Context, _Schema, _Initialized
69 | if not _Initialized:
70 | with schema.open() as schema:
71 | _Schema.update(json.load(schema))
72 | with context.open() as context:
73 | _Context.update(json.load(context))
74 | del _Context['@context']['id']
75 | del _Context['@context']['type']
76 | _Initialized = True
77 |
78 |
79 | def _LoadJsonLd(json_val, public_id):
80 | _Init(_Module_path / 'schema' / 'jsonldcontext.json',
81 | _Module_path / 'schema' / 'schema.jsonld')
82 | json_val['@context'] = _Context
83 | graph = Graph().parse(
84 | data=json.dumps(json_val).encode('utf-8'),
85 | format='json-ld',
86 | publicID=public_id
87 | )
88 | return graph
89 |
90 |
91 | def LoadGraph(input, public_id):
92 | if isinstance(input, dict):
93 | data = input
94 | elif isinstance(input, str):
95 | data = json.loads(input)
96 | else:
97 | data = json.load(input)
98 |
99 | return _LoadJsonLd(data, public_id)
100 |
101 |
102 | def FrameGraph(graph, frame=_FullFrame):
103 | serialized = graph.serialize(format='json-ld')
104 | json_val = json.loads(serialized)
105 | json_val = {
106 | '@context': _Context,
107 | '@graph': AsList(json_val)
108 | }
109 | framed = jsonld.frame(json_val, frame, {'embed': '@always'})
110 | framed['@context'] = 'http://schema.org'
111 | for items in framed['@graph']:
112 | framed.update(items)
113 | del framed['@graph']
114 | return framed
115 |
116 |
117 | def _N3(obj, namespace_manager):
118 | if isinstance(obj, str):
119 | return obj
120 | return obj.n3(namespace_manager=namespace_manager)
121 |
122 |
123 | def MakeSparqlSelectQuery(*constraints,
124 | ns_manager=None,
125 | rdf_prefixes=_RdfPrefixes):
126 | ret = ''
127 | for prefix, url in rdf_prefixes.items():
128 | ret += f'PREFIX {prefix}: <{url}>\n'
129 | ret += 'SELECT * WHERE {\n'
130 | for constraint in constraints:
131 | sub, pred, obj = (_N3(field, ns_manager)
132 | for field in constraint)
133 | ret += f' {sub} {pred} {obj} .\n'
134 | ret += '}'
135 | return ret
136 |
137 |
138 | def SelectFromGraph(graph, *constraints):
139 | result = graph.query(
140 | MakeSparqlSelectQuery(
141 | *constraints,
142 | ns_manager=graph.namespace_manager))
143 | return list({str(k): str(v)
144 | for k, v in binding.items()}
145 | for binding in result.bindings)
146 |
147 |
148 | def main(args, context, schema):
149 | with open(args[1]) as f:
150 | normalized = FrameGraph(LoadGraph(f, args[1]))
151 | json.dump(normalized, sys.stdout, indent=2)
152 |
153 |
154 | if __name__ == '__main__':
155 | main(sys.argv)
156 |
--------------------------------------------------------------------------------
/samples/eurostat/population_density/eurostat_population_density.json:
--------------------------------------------------------------------------------
1 | {
2 | "@context": "http://schema.org",
3 | "@type": "StatisticalDataset",
4 | "@id": "",
5 | "url": "https://data.europa.eu/euodp/en/data/dataset/bAzn6fiusnRFOBwUeIo78w",
6 | "identifier": "met_d3dens",
7 | "name": "Eurostat Population Density",
8 | "description": "Population density by metropolitan regions",
9 | "dateCreated": "2015-10-16",
10 | "dateModified": "2019-06-18",
11 | "temporalCoverage": "1990-01-01/2016-01-01",
12 | "distribution": {
13 | "@type": "DataDownload",
14 | "contentUrl": "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?file=data/met_d3dens.tsv.gz&unzip=true",
15 | "encodingFormat": "text/tab-separated-values"
16 | },
17 | "spatialCoverage":{
18 | "@type":"Place",
19 | "geo":{
20 | "@type":"GeoShape",
21 | "name": "European Union",
22 | "box":"34.633285 -10.468556 70.096054 34.597916"
23 | }
24 | },
25 | "license": "https://ec.europa.eu/eurostat/about/policies/copyright",
26 | "creator":{
27 | "@type":"Organization",
28 | "url": "https://ec.europa.eu/eurostat",
29 | "name":"Eurostat"
30 | },
31 | "publisher": {
32 | "@type": "Organization",
33 | "name": "Eurostat",
34 | "url": "https://ec.europa.eu/eurostat",
35 | "contactPoint": {
36 | "@type": "ContactPoint",
37 | "contactType": "User Support",
38 | "url": "https://ec.europa.eu/eurostat/help/support"
39 | }
40 | },
41 | "dimension": [
42 | {
43 | "@type": "CategoricalDimension",
44 | "@id": "#metroreg",
45 | "dataset": {"@id": ""},
46 | "codeList": "metroreg.csv"
47 | },
48 | {
49 | "@type": "TimeDimension",
50 | "@id": "#year",
51 | "dataset": {"@id": ""},
52 | "name": "year",
53 | "equivalentType": "xsd:Year",
54 | "dateFormat": "yyyy"
55 | }
56 | ],
57 | "measure": [
58 | {
59 | "@type": "StatisticalMeasure",
60 | "@id": "#density",
61 | "dataset": {"@id": ""},
62 | "name": "Population density",
63 | "unitText": "persons per square kilometre"
64 | }
65 | ],
66 | "footnote": [
67 | {
68 | "@type": "StatisticalAnnotation",
69 | "@id": "#footnote=b",
70 | "dataset": {"@id": ""},
71 | "codeValue": "b",
72 | "description": "break in time series"
73 | },
74 | {
75 | "@type": "StatisticalAnnotation",
76 | "@id": "#footnote=c",
77 | "dataset": {"@id": ""},
78 | "codeValue": "c",
79 | "description": "confidential"
80 | },
81 | {
82 | "@type": "StatisticalAnnotation",
83 | "@id": "#footnote=d",
84 | "dataset": {"@id": ""},
85 | "codeValue": "d",
86 | "description": "definition differs, see metadata"
87 | },
88 | {
89 | "@type": "StatisticalAnnotation",
90 | "@id": "#footnote=e",
91 | "dataset": {"@id": ""},
92 | "codeValue": "e",
93 | "description": "estimated"
94 | },
95 | {
96 | "@type": "StatisticalAnnotation",
97 | "@id": "#footnote=f",
98 | "dataset": {"@id": ""},
99 | "codeValue": "f",
100 | "description": "forecast"
101 | },
102 | {
103 | "@type": "StatisticalAnnotation",
104 | "@id": "#footnote=n",
105 | "dataset": {"@id": ""},
106 | "codeValue": "n",
107 | "description": "not significant"
108 | },
109 | {
110 | "@type": "StatisticalAnnotation",
111 | "@id": "#footnote=p",
112 | "dataset": {"@id": ""},
113 | "codeValue": "p",
114 | "description": "provisional"
115 | },
116 | {
117 | "@type": "StatisticalAnnotation",
118 | "@id": "#footnote=r",
119 | "dataset": {"@id": ""},
120 | "codeValue": "r",
121 | "description": "revised"
122 | },
123 | {
124 | "@type": "StatisticalAnnotation",
125 | "@id": "#footnote=s",
126 | "dataset": {"@id": ""},
127 | "codeValue": "s",
128 | "description": "Eurostat estimate"
129 | },
130 | {
131 | "@type": "StatisticalAnnotation",
132 | "@id": "#footnote=u",
133 | "dataset": {"@id": ""},
134 | "codeValue": "u",
135 | "description": "low reliability"
136 | },
137 | {
138 | "@type": "StatisticalAnnotation",
139 | "@id": "#footnote=z",
140 | "dataset": {"@id": ""},
141 | "codeValue": "z",
142 | "description": "not applicable"
143 | }
144 | ],
145 | "slice": {
146 | "@type": "DataSlice",
147 | "@id": "#metroreg_year",
148 | "dataset": {"@id": ""},
149 | "dimension": ["#metroreg", "#year"],
150 | "measure": {"@id": "#density"},
151 | "data": {"@id": "met_d3dens.csv"}
152 | }
153 | }
154 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/data_sources/data_source_test.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Tests of data_source module."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | import unittest
15 |
16 | import data_source
17 |
18 |
19 | class DataSourceColumnBundleTests(unittest.TestCase):
20 | """Tests of DataSourceColumnBundle object."""
21 |
22 | def setUp(self):
23 | self.column_bundle = data_source.DataSourceColumnBundle(
24 | [data_source.DataSourceColumn(column_id='col1'),
25 | data_source.DataSourceColumn(column_id='col2'),
26 | data_source.DataSourceColumn(column_id='col3')])
27 |
28 | def testAddColumn(self):
29 | self.column_bundle.AddColumn(
30 | data_source.DataSourceColumn(column_id='col4'))
31 | self.assertEqual(self.column_bundle.GetColumnByID('col4').column_id,
32 | 'col4')
33 |
34 | def testGetColumnByID(self):
35 | column = self.column_bundle.GetColumnByID('col2')
36 | self.assertEqual(column.column_id, 'col2')
37 |
38 | def testGetColumnByOrder(self):
39 | column = self.column_bundle.GetColumnByOrder(2)
40 | self.assertEqual(column.column_id, 'col3')
41 |
42 | def testGetNumColumns(self):
43 | self.assertEqual(self.column_bundle.GetNumColumns(), 3)
44 |
45 | def testGetColumnIterator(self):
46 | column_iterator = self.column_bundle.GetColumnIterator()
47 | column_id_list = [c.column_id for c in column_iterator]
48 | self.assertEqual(column_id_list, ['col1', 'col2', 'col3'])
49 |
50 |
51 | class TableDataTest(unittest.TestCase):
52 | """Tests of TableData object."""
53 |
54 | def setUp(self):
55 | self.table_data = data_source.TableData(
56 | [[1, 2, 3], [4, 5, 6]])
57 |
58 | def testMergeValues(self):
59 | another_table_data = data_source.TableData([[4, 5, 6], [6, 7, 8]])
60 | merged_table_data = self.table_data.MergeValues(
61 | another_table_data, num_columns=2)
62 | self.assertEqual(merged_table_data.rows,
63 | [[1, 2, 3, 4, 5], [4, 5, 6, 6, 7]])
64 |
65 | def testMergeContant(self):
66 | merged_table_data = self.table_data.MergeConstant('abcd')
67 | self.assertEqual(merged_table_data.rows,
68 | [[1, 2, 3, 'abcd'], [4, 5, 6, 'abcd']])
69 |
70 |
71 | class DataGuessingTest(unittest.TestCase):
72 | """Test of data type / format guessing functions."""
73 |
74 | def setUp(self):
75 | pass
76 |
77 | def testGuessType(self):
78 | self.assertEqual(data_source.GuessDataType('312332'), 'integer')
79 | self.assertEqual(data_source.GuessDataType('1999', 'year'), 'date')
80 | self.assertEqual(data_source.GuessDataType('3123.32'), 'float')
81 | self.assertEqual(data_source.GuessDataType('-3399332'), 'integer')
82 | self.assertEqual(data_source.GuessDataType('-3.0'), 'float')
83 | self.assertEqual(data_source.GuessDataType('1/1/11'), 'date')
84 | self.assertEqual(data_source.GuessDataType('01/1932'), 'date')
85 | self.assertEqual(data_source.GuessDataType('2-3-1932'), 'date')
86 | self.assertEqual(data_source.GuessDataType('something'), 'string')
87 | self.assertEqual(data_source.GuessDataType('3278.23728.223'), 'string')
88 |
89 | def testGuessDateFormat(self):
90 | self.assertEqual(data_source.GuessDateFormat('2819'), 'yyyy')
91 | self.assertEqual(data_source.GuessDateFormat('3/1990'), 'MM/yyyy')
92 | self.assertEqual(data_source.GuessDateFormat('1990-3'), 'yyyy-MM')
93 | self.assertEqual(data_source.GuessDateFormat('01-2-1981'), 'MM-dd-yyyy')
94 | self.assertEqual(data_source.GuessDateFormat('1990/2/3'), 'yyyy/MM/dd')
95 |
96 | self.assertRaises(data_source.DataSourceError,
97 | data_source.GuessDateFormat, '1990.12')
98 | self.assertRaises(data_source.DataSourceError,
99 | data_source.GuessDateFormat, 'Jan 1981')
100 |
101 | def testGuessDateConcept(self):
102 | self.assertEqual(data_source.GuessDateConcept('yyyy'), 'time:year')
103 | self.assertEqual(data_source.GuessDateConcept('yyyy-MM'), 'time:month')
104 | self.assertEqual(data_source.GuessDateConcept('yy.MM.dd'), 'time:day')
105 | self.assertEqual(data_source.GuessDateConcept('dd/MM/yyyy'), 'time:day')
106 |
107 | self.assertRaises(data_source.DataSourceError,
108 | data_source.GuessDateConcept, 'yy-mm')
109 | self.assertRaises(data_source.DataSourceError,
110 | data_source.GuessDateConcept, 'GG yyyy')
111 |
112 |
113 | if __name__ == '__main__':
114 | unittest.main()
115 |
--------------------------------------------------------------------------------
/tools/dspl2/dspl2/tests/test_rdfutil.py:
--------------------------------------------------------------------------------
1 | from dspl2.rdfutil import (LoadGraph, FrameGraph, SelectFromGraph)
2 | from io import StringIO
3 | import json
4 | import rdflib
5 | import rdflib.compare
6 | import unittest
7 |
8 |
9 | _SampleJson = '''{
10 | "@context": "http://schema.org",
11 | "@type": "StatisticalDataset",
12 | "@id": "",
13 | "url": "https://data.europa.eu/euodp/en/data/dataset/bAzn6fiusnRFOBwUeIo78w",
14 | "identifier": "met_d3dens",
15 | "name": "Eurostat Population Density",
16 | "description": "Population density by metropolitan regions",
17 | "dateCreated": "2015-10-16",
18 | "dateModified": "2019-06-18",
19 | "temporalCoverage": "1990-01-01/2016-01-01",
20 | "distribution": {
21 | "@type": "DataDownload",
22 | "contentUrl": "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?file=data/met_d3dens.tsv.gz&unzip=true",
23 | "encodingFormat": "text/tab-separated-values"
24 | },
25 | "spatialCoverage":{
26 | "@type":"Place",
27 | "geo":{
28 | "@type":"GeoShape",
29 | "name": "European Union",
30 | "box":"34.633285 -10.468556 70.096054 34.597916"
31 | }
32 | },
33 | "license": "https://ec.europa.eu/eurostat/about/policies/copyright",
34 | "creator":{
35 | "@type":"Organization",
36 | "url": "https://ec.europa.eu/eurostat",
37 | "name":"Eurostat"
38 | },
39 | "publisher": {
40 | "@type": "Organization",
41 | "name": "Eurostat",
42 | "url": "https://ec.europa.eu/eurostat",
43 | "contactPoint": {
44 | "@type": "ContactPoint",
45 | "contactType": "User Support",
46 | "url": "https://ec.europa.eu/eurostat/help/support"
47 | }
48 | }
49 | }'''
50 |
51 |
52 | class RdfUtilTests(unittest.TestCase):
53 | def test_LoadGraph(self):
54 | graph1 = LoadGraph(_SampleJson, '')
55 | graph2 = LoadGraph(json.loads(_SampleJson), '')
56 | graph3 = LoadGraph(StringIO(_SampleJson), '')
57 | self.assertTrue(rdflib.compare.isomorphic(graph1, graph2))
58 | self.assertTrue(rdflib.compare.isomorphic(graph1, graph3))
59 |
60 | def test_FrameGraph(self):
61 | json_val = FrameGraph(LoadGraph(_SampleJson, ''))
62 | self.assertEqual(json_val['@context'], 'http://schema.org')
63 | self.assertEqual(json_val['@type'], 'StatisticalDataset')
64 | self.assertEqual(json_val['url'], 'https://data.europa.eu/euodp/en/data/dataset/bAzn6fiusnRFOBwUeIo78w')
65 | self.assertEqual(json_val['identifier'], 'met_d3dens')
66 | self.assertEqual(json_val['name'], 'Eurostat Population Density')
67 | self.assertEqual(json_val['description'], 'Population density by metropolitan regions')
68 | self.assertEqual(json_val['dateCreated'], '2015-10-16')
69 | self.assertEqual(json_val['dateModified'], '2019-06-18')
70 | self.assertEqual(json_val['temporalCoverage'], '1990-01-01/2016-01-01')
71 | self.assertEqual(json_val['distribution']['@type'], 'DataDownload')
72 | self.assertEqual(json_val['distribution']['contentUrl'], 'http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?file=data/met_d3dens.tsv.gz&unzip=true')
73 | self.assertEqual(json_val['distribution']['encodingFormat'], 'text/tab-separated-values')
74 | self.assertEqual(json_val['spatialCoverage']['@type'], "Place")
75 | self.assertEqual(json_val['spatialCoverage']['geo']['@type'], "GeoShape")
76 | self.assertEqual(json_val['spatialCoverage']['geo']['name'], 'European Union')
77 | self.assertEqual(json_val['spatialCoverage']['geo']['box'], '34.633285 -10.468556 70.096054 34.597916')
78 | self.assertEqual(json_val['license'], 'https://ec.europa.eu/eurostat/about/policies/copyright')
79 | self.assertEqual(json_val['creator']['@type'], "Organization")
80 | self.assertEqual(json_val['creator']['url'], 'https://ec.europa.eu/eurostat')
81 | self.assertEqual(json_val['creator']['name'], 'Eurostat')
82 | self.assertEqual(json_val['publisher']['@type'], 'Organization')
83 | self.assertEqual(json_val['publisher']['name'], 'Eurostat')
84 | self.assertEqual(json_val['publisher']['url'], 'https://ec.europa.eu/eurostat')
85 | self.assertEqual(json_val['publisher']['contactPoint']['@type'], 'ContactPoint')
86 | self.assertEqual(json_val['publisher']['contactPoint']['contactType'], 'User Support')
87 | self.assertEqual(json_val['publisher']['contactPoint']['url'], 'https://ec.europa.eu/eurostat/help/support')
88 |
89 | def test_SelectFromGraph(self):
90 | graph = LoadGraph(_SampleJson, '')
91 | results = list(SelectFromGraph(
92 | graph,
93 | ('?ds', 'rdf:type', 'schema:StatisticalDataset'),
94 | ('?ds', 'schema:name', '?name')))
95 | self.assertEqual(len(results), 1)
96 | self.assertEqual(results[0]['name'], 'Eurostat Population Density')
97 |
98 |
99 | if __name__ == '__main__':
100 | unittest.main()
101 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/validation/schemas/xml_1998.xsd:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | See http://www.w3.org/XML/1998/namespace.html and
7 | http://www.w3.org/TR/REC-xml for information about this namespace.
8 |
9 | This schema document describes the XML namespace, in a form
10 | suitable for import by other schema documents.
11 |
12 | Note that local names in this namespace are intended to be defined
13 | only by the World Wide Web Consortium or its subgroups. The
14 | following names are currently defined in this namespace and should
15 | not be used with conflicting semantics by any Working Group,
16 | specification, or document instance:
17 |
18 | base (as an attribute name): denotes an attribute whose value
19 | provides a URI to be used as the base for interpreting any
20 | relative URIs in the scope of the element on which it
21 | appears; its value is inherited. This name is reserved
22 | by virtue of its definition in the XML Base specification.
23 |
24 | lang (as an attribute name): denotes an attribute whose value
25 | is a language code for the natural language of the content of
26 | any element; its value is inherited. This name is reserved
27 | by virtue of its definition in the XML specification.
28 |
29 | space (as an attribute name): denotes an attribute whose
30 | value is a keyword indicating what whitespace processing
31 | discipline is intended for the content of the element; its
32 | value is inherited. This name is reserved by virtue of its
33 | definition in the XML specification.
34 |
35 | Father (in any context at all): denotes Jon Bosak, the chair of
36 | the original XML Working Group. This name is reserved by
37 | the following decision of the W3C XML Plenary and
38 | XML Coordination groups:
39 |
40 | In appreciation for his vision, leadership and dedication
41 | the W3C XML Plenary on this 10th day of February, 2000
42 | reserves for Jon Bosak in perpetuity the XML name
43 | xml:Father
44 |
45 |
46 |
47 |
48 | This schema defines attributes and an attribute group
49 | suitable for use by
50 | schemas wishing to allow xml:base, xml:lang or xml:space attributes
51 | on elements they define.
52 |
53 | To enable this, such a schema must import this schema
54 | for the XML namespace, e.g. as follows:
55 | <schema . . .>
56 | . . .
57 | <import namespace="http://www.w3.org/XML/1998/namespace"
58 | schemaLocation="http://www.w3.org/2001/03/xml.xsd"/>
59 |
60 | Subsequently, qualified reference to any of the attributes
61 | or the group defined below will have the desired effect, e.g.
62 |
63 | <type . . .>
64 | . . .
65 | <attributeGroup ref="xml:specialAttrs"/>
66 |
67 | will define a type which will schema-validate an instance
68 | element with any of those attributes
69 |
70 |
71 |
72 | In keeping with the XML Schema WG's standard versioning
73 | policy, this schema document will persist at
74 | http://www.w3.org/2001/03/xml.xsd.
75 | At the date of issue it can also be found at
76 | http://www.w3.org/2001/xml.xsd.
77 | The schema document at that URI may however change in the future,
78 | in order to remain compatible with the latest version of XML Schema
79 | itself. In other words, if the XML Schema namespace changes, the version
80 | of this document at
81 | http://www.w3.org/2001/xml.xsd will change
82 | accordingly; the version at
83 | http://www.w3.org/2001/03/xml.xsd will not change.
84 |
85 |
86 |
87 |
88 |
89 | In due course, we should install the relevant ISO 2- and 3-letter
90 | codes as the enumerated possible values . . .
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 | See http://www.w3.org/TR/xmlbase/ for
106 | information about this attribute.
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/validation/xml_validation_test.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Tests of xml_validation module."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | import re
15 | import StringIO
16 | import unittest
17 |
18 | import xml_validation
19 |
20 |
21 | _DSPL_CONTENT_VALID = (
22 | """
23 |
25 |
26 |
27 |
28 | Dataset Name
29 |
30 |
31 |
32 |
33 | Provider Name
34 |
35 |
36 | """)
37 |
38 |
39 | _DSPL_CONTENT_XML_ERROR = (
40 | """
41 |
43 |
44 |
45 |
46 | Dataset Name
47 |
48 |
49 |
50 |
51 | Provider Name
52 |
53 |
54 | """)
55 |
56 |
57 | _DSPL_CONTENT_SCHEMA_ERROR = (
58 | """
59 |
61 |
62 |
63 |
64 | Dataset Name
65 |
66 |
67 |
68 |
69 | Provider Name
70 |
71 |
72 | """)
73 |
74 | _DSPL_BILLION_LAUGHS = (
75 | """
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 | ]>
87 |
88 |
89 |
90 | &lol9;
91 |
92 |
93 |
94 |
95 | Provider Name
96 |
97 |
98 | """)
99 |
100 |
101 | class XMLValidationTests(unittest.TestCase):
102 | """Test case for xml_validation module."""
103 |
104 | def setUp(self):
105 | pass
106 |
107 | def testXMLValidationGoodXML(self):
108 | """A simple end-to-end test of the valid XML case."""
109 | valid_input_file = StringIO.StringIO(_DSPL_CONTENT_VALID)
110 |
111 | result = xml_validation.RunValidation(valid_input_file)
112 | self.assertTrue(re.search('validates successfully', result))
113 |
114 | valid_input_file.close()
115 |
116 | def testXMLValidationXMLError(self):
117 | """A simple end-to-end test of the bad XML case."""
118 | xml_error_input_file = StringIO.StringIO(_DSPL_CONTENT_XML_ERROR)
119 |
120 | result = xml_validation.RunValidation(xml_error_input_file)
121 | self.assertTrue(
122 | re.search('XML declaration allowed only.*line 1', result, flags=re.DOTALL))
123 |
124 | xml_error_input_file.close()
125 |
126 | def testXMLValidationSchemaError(self):
127 | """A simple end-to-end test of the non-conforming XML case."""
128 | schema_error_input_file = StringIO.StringIO(_DSPL_CONTENT_SCHEMA_ERROR)
129 |
130 | result = xml_validation.RunValidation(schema_error_input_file)
131 | # TODO: this validation failure has lineno 0; look into why lxml is not
132 | # returning the right location.
133 | self.assertTrue(re.search('The attribute \'illegalproperty\' is not allowed',
134 | result, flags=re.DOTALL))
135 |
136 | schema_error_input_file.close()
137 |
138 | def testXMLBillionLaughsAttack(self):
139 | """A simple test to verify that the validation routine is not susceptible
140 | to the billion laughs attack.
141 | """
142 | billion_laughs_input_file = StringIO.StringIO(_DSPL_BILLION_LAUGHS)
143 | result = xml_validation.RunValidation(billion_laughs_input_file)
144 | self.assertTrue(re.search('Detected an entity reference loop', result))
145 |
146 | billion_laughs_input_file.close()
147 |
148 |
149 | if __name__ == '__main__':
150 | unittest.main()
151 |
--------------------------------------------------------------------------------
/tools/dspltools/scripts/dsplcheck.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Check a DSPL dataset for likely import errors."""
10 | from __future__ import print_function
11 |
12 |
13 | __author__ = 'Benjamin Yolken '
14 |
15 | import optparse
16 | import os
17 | import shutil
18 | import sys
19 | import tempfile
20 | import time
21 | import zipfile
22 |
23 | from dspllib.model import dspl_model_loader
24 | from dspllib.validation import dspl_validation
25 | from dspllib.validation import xml_validation
26 |
27 |
28 | def LoadOptionsFromFlags(argv):
29 | """Parse command-line arguments.
30 |
31 | Args:
32 | argv: The program argument vector (excluding the script name)
33 |
34 | Returns:
35 | A dictionary with key-value pairs for each of the options
36 | """
37 | usage_string = 'python dsplcheck.py [options] [DSPL XML file or zip archive]'
38 |
39 | parser = optparse.OptionParser(usage=usage_string)
40 |
41 | parser.set_defaults(verbose=True)
42 | parser.add_option('-q', '--quiet',
43 | action='store_false', dest='verbose',
44 | help='Quiet mode')
45 |
46 | parser.add_option(
47 | '-l', '--checking_level', dest='checking_level', type='choice',
48 | choices=['schema_only', 'schema_and_model', 'full'], default='full',
49 | help='Level of checking to do (default: full)')
50 |
51 | (options, args) = parser.parse_args(args=argv)
52 |
53 | if not len(args) == 1:
54 | parser.error('An XML file or DSPL zip archive is required')
55 |
56 | return {'verbose': options.verbose,
57 | 'checking_level': options.checking_level,
58 | 'file_path': args[0]}
59 |
60 |
61 | def GetInputFilePath(input_file_path):
62 | """Parse the input file path, extracting a zip file if necessary.
63 |
64 | Args:
65 | input_file_path: String path to dsplcheck input file
66 |
67 | Returns:
68 | Dictionary containing final XML file path (post-extraction) and directory
69 | into which zip was extracted (or '' if input was not a zip).
70 | """
71 | if zipfile.is_zipfile(input_file_path):
72 | # Extract files to temporary directory and search for dataset XML
73 | zip_dir = tempfile.mkdtemp()
74 |
75 | zip_file = zipfile.ZipFile(input_file_path, 'r')
76 | zip_file.extractall(zip_dir)
77 |
78 | xml_file_paths = []
79 |
80 | for (dirpath, unused_dirnames, filenames) in os.walk(zip_dir):
81 | for file_name in filenames:
82 | if file_name[-4:] == '.xml':
83 | xml_file_paths.append(os.path.join(dirpath, file_name))
84 |
85 | if not xml_file_paths:
86 | print('Error: zip does not have any XML files')
87 | sys.exit(2)
88 | elif len(xml_file_paths) > 1:
89 | print('Error: zip contains multiple XML files')
90 | sys.exit(2)
91 | else:
92 | xml_file_path = xml_file_paths[0]
93 |
94 | zip_file.close()
95 | else:
96 | xml_file_path = input_file_path
97 | zip_dir = ''
98 |
99 | return {'xml_file_path': xml_file_path,
100 | 'zip_dir': zip_dir}
101 |
102 |
103 | def main(argv):
104 | """Parse command-line flags and run XML validator.
105 |
106 | Args:
107 | argv: The program argument vector (excluding the script name)
108 | """
109 | start_time = time.time()
110 |
111 | options = LoadOptionsFromFlags(argv)
112 | file_paths = GetInputFilePath(options['file_path'])
113 |
114 | try:
115 | xml_file = open(file_paths['xml_file_path'], 'r')
116 | except IOError as io_error:
117 | print('Error opening XML file\n\n%s' % io_error)
118 | sys.exit(2)
119 |
120 | if options['verbose']:
121 | print('==== Checking XML file against DSPL schema....')
122 |
123 | result = xml_validation.RunValidation(
124 | xml_file,
125 | verbose=options['verbose'])
126 |
127 | print(result)
128 |
129 | if 'validates successfully' not in result:
130 | # Stop if XML validation not successful
131 | sys.exit(2)
132 |
133 | if options['checking_level'] != 'schema_only':
134 | if options['verbose']:
135 | print('\n==== Parsing DSPL dataset....')
136 |
137 | if options['checking_level'] == 'full':
138 | full_data_check = True
139 | else:
140 | full_data_check = False
141 |
142 | try:
143 | dataset = dspl_model_loader.LoadDSPLFromFiles(
144 | file_paths['xml_file_path'], load_all_data=full_data_check)
145 | except dspl_model_loader.DSPLModelLoaderError as loader_error:
146 | print('Error while trying to parse DSPL dataset\n\n%s' % loader_error)
147 | sys.exit(2)
148 |
149 | if options['verbose']:
150 | print('Parsing completed.')
151 |
152 | if full_data_check:
153 | print('\n==== Checking DSPL model and data....')
154 | else:
155 | print('\n==== Checking DSPL model....')
156 |
157 | dspl_validator = dspl_validation.DSPLDatasetValidator(
158 | dataset, full_data_check=full_data_check)
159 |
160 | print(dspl_validator.RunValidation(options['verbose']))
161 |
162 | xml_file.close()
163 |
164 | if file_paths['zip_dir']:
165 |
166 | shutil.rmtree(file_paths['zip_dir'])
167 |
168 | if options['verbose']:
169 | print('\nCompleted in %0.2f seconds' % (time.time() - start_time))
170 |
171 |
172 | if __name__ == '__main__':
173 | main(sys.argv[1:])
174 |
--------------------------------------------------------------------------------
/tools/dspltools/examples/dsplcheck/invalid_dspl/invalid_dspl.xml:
--------------------------------------------------------------------------------
1 |
2 |
32 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 | My statistics
47 |
48 |
49 | Some very interesting statistics about countries
50 |
51 |
52 | http://www.stats-bureau.com/mystats/info.html
53 |
54 |
55 |
56 |
57 |
58 | Bureau of Statistics
59 |
60 |
61 | http://www.stats-bureau.com
62 |
63 |
64 |
65 |
66 |
67 |
68 | Geography
69 |
70 |
71 |
72 |
73 | Social indicators
74 |
75 |
76 |
77 | Population indicators
78 |
79 |
80 |
81 |
82 | Poverty & income
83 |
84 |
85 |
86 |
87 | Health
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 | Population
99 |
100 |
101 | Size of the resident population.
102 |
103 |
104 |
105 |
106 |
107 |
108 |
112 |
113 |
114 |
115 | Country
116 |
117 |
118 | My list of countries
119 |
120 |
121 |
122 |
123 |
124 | Country name
125 |
126 | The official name of the country
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 | countries.csv
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 | country_slice.csv
161 |
162 |
163 |
164 |
165 |
166 |
--------------------------------------------------------------------------------
/tools/dspltools/examples/dsplcheck/valid_dataset/valid_dataset.xml:
--------------------------------------------------------------------------------
1 |
2 |
32 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 | My statistics
47 |
48 |
49 | Some very interesting statistics about countries
50 |
51 |
52 | http://www.stats-bureau.com/mystats/info.html
53 |
54 |
55 |
56 |
57 |
58 | Bureau of Statistics
59 |
60 |
61 | http://www.stats-bureau.com
62 |
63 |
64 |
65 |
66 |
67 |
68 | Geography
69 |
70 |
71 |
72 |
73 | Social indicators
74 |
75 |
76 |
77 | Population indicators
78 |
79 |
80 |
81 |
82 | Poverty & income
83 |
84 |
85 |
86 |
87 | Health
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 | Population
99 |
100 |
101 | Size of the resident population.
102 |
103 |
104 |
105 |
106 |
107 |
108 |
112 |
113 |
114 |
115 | Country
116 |
117 |
118 | My list of countries
119 |
120 |
121 |
122 |
123 |
124 | Country name
125 |
126 | The official name of the country
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 | countries.csv
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 | country_slice.csv
161 |
162 |
163 |
164 |
165 |
166 |
--------------------------------------------------------------------------------
/tools/dspl2viz/static/dspl2viz.js:
--------------------------------------------------------------------------------
1 | var View;
2 |
3 | function drawChart(event) {
4 | try {
5 | var spec = JSON.parse(event.target.value);
6 | spec.height = document.querySelector('#chart-container').scrollHeight;
7 | spec.width = document.querySelector('#chart-container').scrollWidth;
8 | event.target.value = JSON.stringify(spec, null, 2);
9 | vegaEmbed("#chart", spec)
10 | // result.view provides access to the Vega View API
11 | .then(result => {View =result})
12 | .catch(console.warn);
13 | } catch(e) {
14 | console.warn(e);
15 | }
16 | }
17 |
18 | var DatasetId = 'file:///usr/local/google/home/nkrishnaswami/dspl/samples/bls/unemployment/bls-unemployment.jsonld';
19 | var SliceId = '#statesUnemploymentMonthly';
20 | var MeasureId = '#unemployment_rate';
21 | var DimValues = {
22 | seasonality: 'S',
23 | state: 'ST0100000000000',
24 | };
25 |
26 | function setSpec() {
27 | var vlSpec = {
28 | "$schema": "https://vega.github.io/schema/vega-lite/v4.0.0-beta.10.json",
29 | "description": "A simple bar chart with embedded data.",
30 | "autosize": {
31 | "type": "fit",
32 | "resize": true
33 | },
34 | "data": {
35 | "url": "/api/series",
36 | "format": {
37 | "type": "csv"
38 | }
39 | },
40 | "mark": "line",
41 | "encoding": {
42 | "x": {
43 | "field": "month",
44 | "type": "ordinal"
45 | },
46 | "y": {
47 | "field": "unemployment_rate",
48 | "type": "quantitative"
49 | },
50 | "color": {
51 | "field": "state",
52 | "type": "ordinal"
53 | }
54 | }
55 | }
56 | vlSpec.data.url += '?dataset=' + encodeURIComponent(DatasetId);
57 | vlSpec.data.url += '&slice=' + encodeURIComponent(SliceId);
58 | vlSpec.data.url += '&measure=' + encodeURIComponent(MeasureId);
59 | vlSpec.data.url += '&dimension_value=';
60 | for (var idx = 0; idx < Object.keys(DimValues).length; ++idx) {
61 | if (idx != 0) {
62 | vlSpec.data.url += ',';
63 | }
64 | var key = Object.keys(DimValues)[idx];
65 | var val = DimValues[key];
66 | vlSpec.data.url += encodeURIComponent(`#${key}:${val}`);
67 | }
68 |
69 | var input = document.querySelector("#vegalite-input");
70 | input.value = JSON.stringify(vlSpec, null, 2);
71 | input.dispatchEvent(new Event('change'));
72 | }
73 |
74 | function processMeasures(data) {
75 | let measure_container = document.querySelector('#measure-explorer');
76 | measure_container.innerText = "Measures:";
77 | console.log(measure_container);
78 | let ul = document.createElement('ul');
79 | measure_container.appendChild(ul);
80 | for(let measure of data) {
81 | let id = $('', { href: measure['@id'] }).prop('hash').substring(1);
82 | console.log("Processing ", measure.name, 'id:', id, measure);
83 | let li = document.createElement('ul');
84 | li.innerText = measure.name;
85 | if (measure.description) {
86 | li.title = measure.description;
87 | }
88 | li.addEventListener('click', function (event) {
89 | for(var elt of ul.children) {
90 | elt.style.fontWeight = 'normal';
91 | }
92 | event.target.style.fontWeight = 'bold';
93 | MeasureId = '#' + id;
94 | setSpec();
95 | });
96 | ul.appendChild(li);
97 | }
98 | }
99 |
100 | function processSlices(data) {
101 | let slice_container = document.querySelector('#slice-explorer');
102 | slice_container.innerText = "Slices:";
103 | console.log(slice_container);
104 | let ul = document.createElement('ul');
105 | slice_container.appendChild(ul);
106 | for(let slice of data) {
107 | let id = $('', { href: slice['@id'] }).prop('hash').substring(1);
108 | console.log("Processing ", slice.name, 'id:', id, slice);
109 | let li = document.createElement('ul');
110 | li.innerText = slice.name;
111 | if (slice.description) {
112 | li.title = slice.description;
113 | }
114 | li.addEventListener('click', function (event) {
115 | for(var elt of ul.children) {
116 | elt.style.fontWeight = 'normal';
117 | }
118 | event.target.style.fontWeight = 'bold';
119 | SliceId = '#' + id;
120 | setSpec();
121 | });
122 | ul.appendChild(li);
123 | }
124 | }
125 |
126 |
127 |
128 |
129 | function processDimensionValues(dimension) {
130 | let id = $('', { href: dimension['@id'] }).prop('hash').substring(1);
131 | console.log("Processing ", dimension.name, 'id:', id);
132 | let div = document.createElement('div');
133 | let dimension_container = document.querySelector('#dimension-explorer');
134 | dimension_container.appendChild(div);
135 | div.innerText = dimension.name;
136 | if (dimension.description) {
137 | div.title = dimension.description;
138 | }
139 | let ul = document.createElement('ul');
140 | div.appendChild(ul);
141 | dimension.codes = {};
142 | for(let dimensionValue of dimension.codeList) {
143 | dimension.codes[dimensionValue.codeValue] = dimensionValue;
144 | let li = document.createElement('li');
145 | li.innerText = dimensionValue.name;
146 | if (dimensionValue.description) {
147 | li.title = dimensionValue.description;
148 | }
149 | li.addEventListener('click', function (event) {
150 | for(var elt of ul.children) {
151 | elt.style.fontWeight = 'normal';
152 | }
153 | event.target.style.fontWeight = 'bold';
154 | DimValues[id] = dimensionValue.codeValue;
155 | setSpec()
156 | });
157 | ul.appendChild(li);
158 | }
159 | }
160 |
161 |
162 | function processDimensions(data) {
163 | for(let dimension of data) {
164 | if (dimension.name != 'States' && dimension.name != 'Seasonality') {
165 | continue;
166 | }
167 | $.getJSON('/api/dimension_values?dataset=file:///usr/local/google/home/nkrishnaswami/dspl/samples/bls/unemployment/bls-unemployment.jsonld&dimension='+encodeURIComponent(dimension['@id']),
168 | processDimensionValues);
169 | }
170 | }
171 |
172 | document.querySelector("#vegalite-input").addEventListener('change', drawChart);
173 | setSpec();
174 |
175 |
176 | $.getJSON('/api/measures?dataset=' + encodeURIComponent(DatasetId), processMeasures);
177 | // $.getJSON('/api/slices_for_measure?dataset=' + encodeURIComponent(DatasetId), processMeasures);
178 | $.getJSON('/api/dimensions?dataset=' + encodeURIComponent(DatasetId), processDimensions);
179 |
--------------------------------------------------------------------------------
/samples/google/canonical/unit.xml:
--------------------------------------------------------------------------------
1 |
2 |
32 |
35 |
36 |
37 |
38 |
39 |
40 | Unit concepts
41 |
42 |
43 | Concepts for to representing units.
44 |
45 |
46 | http://code.google.com/apis/publicdata/docs/canonical/unit.html
47 |
48 |
49 |
50 |
51 |
52 | Google Inc.
53 |
54 |
55 | Google Inc.
56 |
57 |
58 | http://www.google.com
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 | Unit
67 |
68 |
69 |
70 | Specifies the unit associated with a metric concept.
71 |
72 | Example:
73 |
74 |
75 |
76 |
77 | Area in square kilometers
78 |
79 |
80 |
81 | ]]>
82 |
83 | The table contains a single row that contains the property values:
84 |
85 | symbol,symbol_position,unit_text
86 | km²,END,square kilometers
87 |
88 | One can then use this unit in defining a metric concept:
89 |
90 |
91 |
92 |
93 | Country area in square kilometers
94 |
95 |
96 |
97 |
98 | ]]>
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 | The symbol associated with a unit.
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 | Unit text
116 |
117 |
118 |
119 | Descriptive text that can be displayed next to a value.
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 | Unit symbol position
130 |
131 |
132 | Unit symbol position
133 |
134 |
135 |
136 | END
137 |
138 |
139 |
140 |
141 |
142 |
143 | Currency unit
144 |
145 |
146 |
147 | Specifies the currency associated with a metric concept.
148 | Each currency is identified by its 3-letter currency code (ISO 4217).
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 | unit_symbol_positions.csv
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 | currencies.csv
170 |
171 |
172 |
173 |
174 |
--------------------------------------------------------------------------------
/tools/dspltools/scripts/dsplcheck_test.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Tests of dsplcheck module."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | import os
15 | import os.path
16 | import re
17 | import shutil
18 | import StringIO
19 | import sys
20 | import tempfile
21 | import unittest
22 | import zipfile
23 |
24 | import dsplcheck
25 |
26 |
27 | _DSPL_CONTENT = (
28 | """
29 |
31 |
32 |
33 |
34 | Dataset Name
35 |
36 |
37 |
38 |
39 | Provider Name
40 |
41 |
42 | """)
43 |
44 |
45 | _DSPL_CONTENT_BAD_CSV_PATH = (
46 | """
47 |
49 |
50 |
51 |
52 | Dataset Name
53 |
54 |
55 |
56 |
57 | Provider Name
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 | non_existent_file.csv
66 |
67 |
68 |
69 | """)
70 |
71 |
72 | class DSPLCheckTests(unittest.TestCase):
73 | """Test case for dsplcheck module."""
74 |
75 | def setUp(self):
76 | self.input_dir = tempfile.mkdtemp()
77 | self.valid_dspl_file_path = (
78 | os.path.join(self.input_dir, 'valid_dataset.xml'))
79 |
80 | self.valid_dspl_file = open(
81 | self.valid_dspl_file_path, 'w')
82 | self.valid_dspl_file.write(_DSPL_CONTENT)
83 | self.valid_dspl_file.close()
84 |
85 | def tearDown(self):
86 | shutil.rmtree(self.input_dir)
87 |
88 | def testValidDataset(self):
89 | """Test basic case of dataset that validates and parses correctly."""
90 | self._StdoutTestHelper(
91 | dsplcheck.main, [self.valid_dspl_file_path],
92 | 'validates successfully.*Parsing completed.*'
93 | 'Checking DSPL model and data.*Completed')
94 |
95 | def testBadXMLFilePath(self):
96 | """Test case where bad XML file path is passed in."""
97 | self._StdoutTestHelper(
98 | dsplcheck.main, ['nonexistent_input_file.xml'],
99 | 'Error opening XML file', expect_exit=True)
100 |
101 | def testBadCSVFilePath(self):
102 | """Test case where DSPL file has bad CSV reference."""
103 | bad_csv_dspl_file_path = (
104 | os.path.join(self.input_dir, 'invalid_csv_dataset.xml'))
105 |
106 | bad_csv_dspl_file = open(bad_csv_dspl_file_path, 'w')
107 | bad_csv_dspl_file.write(_DSPL_CONTENT_BAD_CSV_PATH)
108 | bad_csv_dspl_file.close()
109 |
110 | self._StdoutTestHelper(
111 | dsplcheck.main, [bad_csv_dspl_file_path],
112 | 'Error while trying to parse', expect_exit=True)
113 |
114 | def testSchemaOnlyOption(self):
115 | """Test that 'schema only' checking level option works correctly."""
116 | self._StdoutTestHelper(
117 | dsplcheck.main, [self.valid_dspl_file_path, '-l', 'schema_only'],
118 | 'validates successfully\W*Completed')
119 |
120 | def testSchemaAndModelOption(self):
121 | """Test that 'schema and model' checking level option works correctly."""
122 | self._StdoutTestHelper(
123 | dsplcheck.main, [self.valid_dspl_file_path, '-l', 'schema_and_model'],
124 | 'Checking DSPL model(?! and data)')
125 |
126 | def testZipInput(self):
127 | """Test that module properly handles zipped input."""
128 | zip_path = os.path.join(self.input_dir, 'dataset.zip')
129 |
130 | zip_file = zipfile.ZipFile(zip_path, 'w')
131 | zip_file.write(self.valid_dspl_file_path)
132 | zip_file.close()
133 |
134 | self._StdoutTestHelper(
135 | dsplcheck.main, [zip_path],
136 | 'validates successfully.*Parsing completed.*'
137 | 'Checking DSPL model and data.*Completed')
138 |
139 | def testZipMissingXML(self):
140 | """Test that zip file without an XML file produces error."""
141 | zip_path = os.path.join(self.input_dir, 'dataset.zip')
142 |
143 | zip_file = zipfile.ZipFile(zip_path, 'w')
144 | zip_file.writestr('test.txt', 'Text')
145 | zip_file.close()
146 |
147 | self._StdoutTestHelper(
148 | dsplcheck.main, [zip_path],
149 | 'does not have any XML', expect_exit=True)
150 |
151 | def testZipMultipleXMLFiles(self):
152 | """Test that zip file with multiple XML files produces error."""
153 | zip_path = os.path.join(self.input_dir, 'dataset.zip')
154 |
155 | zip_file = zipfile.ZipFile(zip_path, 'w')
156 | zip_file.writestr('test.xml', 'Text')
157 | zip_file.writestr('test2.xml', 'Text')
158 | zip_file.close()
159 |
160 | self._StdoutTestHelper(
161 | dsplcheck.main, [zip_path],
162 | 'multiple XML files', expect_exit=True)
163 |
164 | def _StdoutTestHelper(self, function, args,
165 | expected_output, expect_exit=False):
166 | """Check the stdout output of a function against its expected value.
167 |
168 | Args:
169 | function: A function to execute
170 | args: The arguments to pass to the function
171 | expected_output: A regular expression expected to match the stdout output
172 | expect_exit: Boolean indicating whether the function execution should
173 | trigger a system exit
174 | """
175 | saved_stdout = sys.stdout
176 |
177 | redirected_output = StringIO.StringIO()
178 | sys.stdout = redirected_output
179 |
180 | if expect_exit:
181 | self.assertRaises(SystemExit, function, args)
182 | else:
183 | function(args)
184 |
185 | self.assertTrue(
186 | re.search(expected_output, redirected_output.getvalue(), re.DOTALL))
187 |
188 | redirected_output.close()
189 | sys.stdout = saved_stdout
190 |
191 |
192 | if __name__ == '__main__':
193 | unittest.main()
194 |
--------------------------------------------------------------------------------
/tools/dspl2viz/dspl2viz.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import dspl2
3 | from flask import (
4 | Flask, render_template, request, Response)
5 | from functools import lru_cache
6 | from icu import SimpleDateFormat
7 | from io import StringIO
8 | import json
9 | import os.path
10 | import pandas as pd
11 | from urllib.parse import urlparse
12 |
13 |
14 | app = Flask(__name__)
15 |
16 |
17 | @app.route('/')
18 | def main():
19 | return render_template('dspl2viz.html')
20 |
21 |
22 | @app.route('/api/measures')
23 | def api_measures():
24 | dataset = request.args.get('dataset')
25 | if dataset is None:
26 | return Response("Dataset not specified", status=400)
27 | try:
28 | getter = dspl2.LocalFileGetter(
29 | os.path.expanduser('~/dspl/samples/bls/unemployment/bls-unemployment.jsonld'))
30 | expander = dspl2.Dspl2JsonLdExpander(getter)
31 | ds = expander.Expand(expandSlices=False)
32 | return Response(json.dumps(ds['measure'], indent=2), mimetype='application/json')
33 | except Exception as e:
34 | app.logger.warn(e)
35 | return Response("Unable to find requested dataset", status=404)
36 |
37 |
38 | @app.route('/api/dimensions')
39 | def api_dimensions():
40 | dataset = request.args.get('dataset')
41 | if dataset is None:
42 | return Response("Dataset not specified", status=400)
43 | try:
44 | getter = dspl2.HybridFileGetter(dataset)
45 | expander = dspl2.Dspl2JsonLdExpander(getter)
46 | ds = expander.Expand(expandSlices=False, expandDimensions=False)
47 | return Response(json.dumps(ds['dimension'], indent=2), mimetype='application/json')
48 | except Exception as e:
49 | app.logger.warn(e)
50 | return Response("Unable to find requested dataset", status=404)
51 |
52 |
53 | @app.route('/api/dimension_values')
54 | def api_dimension_values():
55 | dataset = request.args.get('dataset')
56 | if dataset is None:
57 | return Response("Dataset not specified", status=400)
58 | dimension = request.args.get('dimension')
59 | if dimension is None:
60 | return Response("Dimension not specified", status=400)
61 | try:
62 | getter = dspl2.HybridFileGetter(dataset)
63 | expander = dspl2.Dspl2JsonLdExpander(getter)
64 | ds = expander.Expand(expandSlices=False, expandDimensions=True)
65 | for dim in ds['dimension']:
66 | if (dimension == dspl2.GetUrl(dim) or
67 | urlparse(dimension).fragment == urlparse(dspl2.GetUrl(dim)).fragment):
68 | return Response(json.dumps(dim, indent=2), mimetype='application/json')
69 | return Response("Unable to find requested dimension", status=404)
70 | except Exception as e:
71 | app.logger.warn(e)
72 | return Response("Unable to find requested dataset", status=404)
73 |
74 |
75 | @app.route('/api/slices_for_measure')
76 | def api_slices_for_measure():
77 | dataset = request.args.get('dataset')
78 | if dataset is None:
79 | return Response("Dataset not specified", status=400)
80 | measure = request.args.get('measure')
81 | if measure is None:
82 | return Response("Measure not specified", status=400)
83 | try:
84 | getter = dspl2.HybridFileGetter(dataset)
85 | expander = dspl2.Dspl2JsonLdExpander(getter)
86 | ds = expander.Expand(expandSlices=False, expandDimensions=False)
87 | slices = []
88 | for slice in ds['slice']:
89 | for sliceMeasure in slice['measure']:
90 | if (measure == dspl2.GetUrl(sliceMeasure) or
91 | urlparse(measure).fragment == urlparse(dspl2.GetUrl(sliceMeasure)).fragment):
92 | slices.append(slice)
93 | break
94 | return Response(json.dumps(slices, indent=2),
95 | mimetype='application/json')
96 | except Exception as e:
97 | app.logger.warn(e)
98 | return Response("Unable to find requested dataset", status=404)
99 |
100 |
101 | @lru_cache(maxsize=10)
102 | def _ExpandDataset(dataset):
103 | getter = dspl2.HybridFileGetter(dataset)
104 | expander = dspl2.Dspl2JsonLdExpander(getter)
105 | return expander.Expand()
106 |
107 |
108 | def _ParseDate(text, date_pattern):
109 | df = SimpleDateFormat(date_pattern)
110 | ts = df.parse(text)
111 | return datetime.datetime.utcfromtimestamp(ts)
112 |
113 |
114 | @lru_cache(maxsize=100)
115 | def _GetDataSeries(dataset, slice, measure, dimension_value):
116 | dim_val_dict = dict([dim_val.split(':')
117 | for dim_val in dimension_value.split(',')])
118 | ds = _ExpandDataset(dataset)
119 | # Identify the time dimension's date format
120 | dateFormat = "yyyy-MM-dd" # default
121 | for dimension in ds['dimension']:
122 | if dimension['@type'] == 'TimeDimension':
123 | dateFormat = dimension.get('dateFormat')
124 | break
125 |
126 | for dsSlice in ds['slice']:
127 | if urlparse(dsSlice['@id']).fragment == urlparse(slice).fragment:
128 | ret = []
129 | for observation in dsSlice['data']:
130 | val = {}
131 | # Slice should have exactly the requested dims + a time dim:
132 | if len(observation['dimensionValues']) != len(dim_val_dict) + 1:
133 | continue
134 | # All the non-time dims should match the filter:
135 | matched_dims = 0
136 | for dim_val in observation['dimensionValues']:
137 | dim_id = urlparse(dim_val['dimension']).fragment
138 | if f'#{dim_id}' in dim_val_dict:
139 | if dim_val.get('codeValue') == dim_val_dict[f'#{dim_id}']:
140 | val[dim_id] = dim_val.get('codeValue')
141 | matched_dims += 1
142 | elif dim_val.get('value'):
143 | val[dim_id] = _ParseDate(dim_val.get('value'), dateFormat)
144 | if matched_dims != len(dim_val_dict):
145 | continue
146 | for meas_val in observation['measureValues']:
147 | if urlparse(meas_val['measure']).fragment == urlparse(measure).fragment:
148 | val[urlparse(measure).fragment] = meas_val['value']
149 | ret.append(val)
150 | return ret
151 |
152 | @app.route('/api/series')
153 | def api_series():
154 | dataset = request.args.get('dataset')
155 | if dataset is None:
156 | return Response("Dataset not specified", status=400)
157 | slice = request.args.get('slice')
158 | if slice is None:
159 | return Response("Slice not specified", status=400)
160 | measure = request.args.get('measure')
161 | if measure is None:
162 | return Response("Measure not specified", status=400)
163 | dimension_values = request.args.get('dimension_value')
164 | if dimension_values is None:
165 | return Response("Dimension values not specified", status=400)
166 | ret = _GetDataSeries(dataset, slice, measure, dimension_values)
167 | if ret is not None:
168 | out = StringIO()
169 | pd.DataFrame(ret).to_csv(out)
170 | return Response(out.getvalue(), mimetype="text/csv")
171 | return Response("Unable to find series for requested dimensions",
172 | status=404)
173 |
--------------------------------------------------------------------------------
/samples/us_census/retail_sales/census-retail-sales.xml:
--------------------------------------------------------------------------------
1 |
2 |
32 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 | Retail Sales in the U.S.
48 |
49 |
50 | Monthly Retail Trade and Food Services report
51 | for the United States. This dataset was prepared by Google based
52 | on data downloaded from the U.S. Census Bureau.
53 |
54 |
55 | http://www.census.gov/retail/
56 |
57 |
58 |
59 |
60 |
61 | U.S. Census Bureau
62 |
63 |
64 | U.S. Census Bureau
65 |
66 |
67 | http://www.census.gov/retail/
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 | Industry
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 | Type of business
86 |
87 |
88 | The principal kind of business being conducted at an establishment
89 |
90 |
91 | Types of business
92 |
93 |
94 |
95 |
96 |
97 | TABLE
98 |
99 |
100 |
101 |
102 | Parent Business
103 |
104 |
105 |
106 | 44x72
107 |
108 |
109 |
110 |
111 |
112 |
113 | Seasonality
114 |
115 |
116 | Are values seasonally adjusted or not
117 |
118 |
119 | Seasonalities
120 |
121 |
122 |
123 | Seasonally Adjusted
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 | Retail Sales Volume
132 |
133 |
134 | Sales include merchandise sold by establishments primarily engaged in retail trade.
135 |
136 |
137 |
138 |
139 |
140 | USD
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 | businesses.csv
165 |
166 |
167 |
168 |
169 |
170 |
171 | seasonalities.csv
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 | retail_sales_business.csv
182 |
183 |
184 |
185 |
186 |
187 |
--------------------------------------------------------------------------------
/samples/google/canonical/time.xml:
--------------------------------------------------------------------------------
1 |
2 |
32 |
34 |
35 |
36 |
37 | Google date and time
38 |
39 |
40 | Google date and time dataset
41 |
42 |
43 | http://code.google.com/apis/publicdata/docs/canonical/time.html
44 |
45 |
46 |
47 | Google Inc.
48 | Google Inc.
49 | http://www.google.com
50 |
51 |
52 |
53 |
54 |
55 | Point in time
56 | Point in time, with a given granularity.
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 | Year date
65 |
66 |
67 | A date with yearly granularity.
68 |
69 | The year concept is usually used directly in a slice definition to define a dimension that
70 | contains year. For example, a slice for yearly population by country would be defined
71 | as follows:
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 | ]]>
81 |
82 | The table definition for this slice would be defined as follows:
83 |
84 |
85 | ...
86 |
87 |
88 |
89 |
90 |
91 | country_slice.csv
92 |
93 |
94 | ...
95 | ]]>
96 |
97 |
98 | And the data contained in the CSV file for this table would look like:
99 |
100 | country, year, population
101 | AF, 1960, 9616353
102 | AF, 1961, 9799379
103 | AF, 1962, 9989846
104 | AF, 1963, 10188299
105 | ...
106 |
107 |
108 |
109 |
110 | YEARLY
111 |
112 |
113 |
114 |
115 |
116 | Quarter date
117 |
118 |
119 | A date with quarterly granularity.
120 | See the example for the year concept above.
121 |
122 |
123 |
124 |
125 | QUARTERLY
126 |
127 |
128 |
129 |
130 |
131 |
132 | Month date
133 |
134 |
135 | A date with monthly granularity.
136 | See the example for the year concept above.
137 |
138 |
139 |
140 |
141 | MONTHLY
142 |
143 |
144 |
145 |
146 |
147 | Week date
148 |
149 |
150 | A date with weekly granularity.
151 | See the example for the year concept above.
152 |
153 |
154 |
155 |
156 | WEEKLY
157 |
158 |
159 |
160 |
161 |
162 | Day date
163 |
164 |
165 | A date with daily granularity.
166 | See the example for the year concept above.
167 |
168 |
169 |
170 |
171 | DAILY
172 |
173 |
174 |
175 |
176 |
177 | Granularity
178 |
179 |
180 | Granularity of time, i.e., the uncertainty that
181 | a point in time may be anywhere within some time interval.
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 | granularity.csv
195 |
196 |
197 |
198 |
199 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/model/dspl_model_loader_test.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Tests of dspl_model_loader module."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | import os
15 | import os.path
16 | import shutil
17 | import tempfile
18 | import unittest
19 |
20 | import dspl_model_loader
21 | import dspl_model_test
22 |
23 |
24 | _SLICE_CSV_DATA = (
25 | """col1,col2
26 | val1,1
27 | val2 , 2
28 | val3,3""")
29 |
30 |
31 | class DSPLModelLoaderTests(unittest.TestCase):
32 | """Basic test cases for dspl_model_loader module."""
33 |
34 | def setUp(self):
35 | self.input_dir = tempfile.mkdtemp()
36 | self.xml_file_path = os.path.join(self.input_dir, 'dataset.xml')
37 |
38 | xml_file = open(self.xml_file_path, 'w')
39 | xml_file.write(dspl_model_test.TEST_DSPL_XML)
40 | xml_file.close()
41 |
42 | slice_csv_file = open(os.path.join(self.input_dir, 'mydata.csv'), 'w')
43 | slice_csv_file.write(_SLICE_CSV_DATA)
44 | slice_csv_file.close()
45 |
46 | def tearDown(self):
47 | shutil.rmtree(self.input_dir)
48 |
49 | def testDSPLImportLoading(self):
50 | """Test that dataset is imported correctly."""
51 | dspl_dataset = dspl_model_loader.LoadDSPLFromFiles(self.xml_file_path)
52 |
53 | # Test basic info
54 | self.assertEqual(dspl_dataset.name, 'My Dataset')
55 | self.assertEqual(dspl_dataset.description, 'My Dataset Description')
56 | self.assertEqual(dspl_dataset.url, 'url1')
57 |
58 | self.assertEqual(dspl_dataset.provider_name, 'Googler')
59 | self.assertEqual(dspl_dataset.provider_url, 'url2')
60 |
61 | # Test imports
62 | self.assertEqual(len(dspl_dataset.imports), 2)
63 |
64 | self.assertEqual(dspl_dataset.imports[0].namespace_id,
65 | 'imported_namespace1')
66 | self.assertEqual(dspl_dataset.imports[0].namespace_url,
67 | 'http://imported_namespace1_url')
68 | self.assertEqual(dspl_dataset.imports[1].namespace_id,
69 | 'imported_namespace2')
70 | self.assertEqual(dspl_dataset.imports[1].namespace_url,
71 | 'http://imported_namespace2_url')
72 |
73 | # Test topics
74 | self.assertEqual(len(dspl_dataset.topics), 2)
75 |
76 | self.assertEqual(dspl_dataset.topics[0].topic_id, 'topic1')
77 | self.assertEqual(dspl_dataset.topics[0].topic_name, 'topic1_name')
78 | self.assertEqual(len(dspl_dataset.topics[0].children), 2)
79 |
80 | self.assertEqual(dspl_dataset.topics[0].children[0].topic_id, 'topic2')
81 | self.assertEqual(
82 | dspl_dataset.topics[0].children[0].topic_name, 'topic2_name')
83 | self.assertEqual(dspl_dataset.topics[0].children[1].topic_id, 'topic3')
84 | self.assertEqual(
85 | dspl_dataset.topics[0].children[1].topic_name, 'topic3_name')
86 |
87 | self.assertEqual(dspl_dataset.topics[1].topic_id, 'topic4')
88 | self.assertEqual(dspl_dataset.topics[1].topic_name, 'topic4_name')
89 | self.assertEqual(len(dspl_dataset.topics[1].children), 0)
90 |
91 | # Test concepts
92 | self.assertEqual(len(dspl_dataset.concepts), 3)
93 |
94 | self.assertEqual(dspl_dataset.concepts[0].concept_id, 'concept1')
95 | self.assertEqual(dspl_dataset.concepts[0].concept_extension_reference,
96 | 'entity:entity')
97 | self.assertEqual(dspl_dataset.concepts[0].concept_name, 'Concept 1')
98 | self.assertEqual(dspl_dataset.concepts[0].concept_description,
99 | 'Concept 1 Description')
100 | self.assertEqual(dspl_dataset.concepts[0].data_type, 'string')
101 | self.assertEqual(len(dspl_dataset.concepts[0].attributes), 1)
102 | self.assertEqual(
103 | dspl_dataset.concepts[0].attributes[0].concept_ref, 'attribute_concept')
104 | self.assertEqual(
105 | dspl_dataset.concepts[0].attributes[0].value, 'attribute_value')
106 | self.assertEqual(len(dspl_dataset.concepts[0].properties), 2)
107 | self.assertEqual(
108 | dspl_dataset.concepts[0].properties[0].concept_ref, 'property_concept')
109 | self.assertEqual(
110 | dspl_dataset.concepts[0].properties[0].is_parent, False)
111 | self.assertEqual(
112 | dspl_dataset.concepts[0].properties[1].concept_ref,
113 | 'another_property_concept')
114 | self.assertEqual(
115 | dspl_dataset.concepts[0].properties[1].is_parent, True)
116 | self.assertEqual(dspl_dataset.concepts[0].table_ref, 'table2')
117 |
118 | self.assertEqual(dspl_dataset.concepts[1].concept_id, 'concept2')
119 | self.assertEqual(dspl_dataset.concepts[1].concept_name, 'Concept 2')
120 | self.assertEqual(dspl_dataset.concepts[1].concept_description,
121 | 'Concept 2 Description')
122 | self.assertEqual(dspl_dataset.concepts[1].topic_references,
123 | ['topic1', 'topic2'])
124 | self.assertEqual(dspl_dataset.concepts[1].data_type, 'integer')
125 | self.assertEqual(len(dspl_dataset.concepts[1].attributes), 0)
126 | self.assertEqual(len(dspl_dataset.concepts[1].properties), 0)
127 |
128 | self.assertEqual(dspl_dataset.concepts[2].concept_id, 'geo:country')
129 | self.assertEqual(dspl_dataset.concepts[2].concept_reference, 'geo:country')
130 |
131 | # Test slices
132 | self.assertEqual(len(dspl_dataset.slices), 1)
133 |
134 | self.assertEqual(dspl_dataset.slices[0].slice_id, 'data_slice')
135 | self.assertEqual(dspl_dataset.slices[0].dimension_refs,
136 | ['concept1', 'geo:country'])
137 | self.assertEqual(dspl_dataset.slices[0].metric_refs, ['concept2'])
138 | self.assertEqual(dspl_dataset.slices[0].table_ref, 'table3')
139 | self.assertEqual(
140 | sorted(dspl_dataset.slices[0].dimension_map.items()),
141 | sorted([('concept1', 'concept_column1'),
142 | ('geo:country', 'concept_column3')]))
143 | self.assertEqual(
144 | dspl_dataset.slices[0].metric_map.items(),
145 | [('concept2', 'concept_column2')])
146 |
147 | # Test tables
148 | self.assertEqual(len(dspl_dataset.tables), 1)
149 |
150 | self.assertEqual(dspl_dataset.tables[0].table_id, 'table')
151 | self.assertEqual(dspl_dataset.tables[0].file_name, 'mydata.csv')
152 |
153 | self.assertEqual(len(dspl_dataset.tables[0].columns), 2)
154 | self.assertEqual(dspl_dataset.tables[0].columns[0].column_id, 'col1')
155 | self.assertEqual(dspl_dataset.tables[0].columns[0].data_type, 'string')
156 | self.assertEqual(dspl_dataset.tables[0].columns[1].column_id, 'col2')
157 | self.assertEqual(dspl_dataset.tables[0].columns[1].data_type, 'integer')
158 |
159 | expected_table_rows = _SLICE_CSV_DATA.splitlines()
160 | expected_table_data = []
161 |
162 | for row in expected_table_rows:
163 | split_row = row.split(',')
164 | cleaned_row = [r.strip() for r in split_row]
165 |
166 | expected_table_data.append(cleaned_row)
167 |
168 | self.assertEqual(dspl_dataset.tables[0].table_data, expected_table_data)
169 |
170 | def testBadFileReference(self):
171 | """Test case in which CSV file does not exist."""
172 | os.remove(os.path.join(self.input_dir, 'mydata.csv'))
173 |
174 | self.assertRaises(
175 | dspl_model_loader.DSPLModelLoaderError,
176 | dspl_model_loader.LoadDSPLFromFiles,
177 | self.xml_file_path)
178 |
179 | def testPartialFileLoading(self):
180 | """Test case in which load_all_data is set to False."""
181 | dspl_dataset = dspl_model_loader.LoadDSPLFromFiles(
182 | self.xml_file_path, load_all_data=False)
183 |
184 | expected_table_rows = _SLICE_CSV_DATA.splitlines()[0:2]
185 | expected_table_data = [r.split(',') for r in expected_table_rows]
186 |
187 | self.assertEqual(dspl_dataset.tables[0].table_data, expected_table_data)
188 |
189 |
190 | if __name__ == '__main__':
191 | unittest.main()
192 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/data_sources/csv_sources_test_suite.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """A set of tests useful for CSV data sources."""
10 |
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 | import StringIO
15 | import unittest
16 |
17 | import data_source
18 |
19 |
20 | _TEST_CSV_CONTENT = (
21 | """date[type=date;format=yyyy-MM-dd],category1,category2[concept=geo:us_state;parent=category3;total_val=total],category3,metric1[extends=quantity:ratio;slice_role=metric],metric2[aggregation=avg],metric3[aggregation=count]
22 | 1980-01-01,red,california,west,89,321,71.21
23 | 1981-01-01,red,california,west,99,231,391.2
24 | 1982-01-01,blue,maine's,east,293,32,2.31
25 | 1983-01-01,blue,california,west,293,12,10.3
26 | 1984-01-01,red,maine's,east,932,48,10.78
27 | 1984-01-01,red,oregon,west,32,33,-14.34
28 | 1985-01-01,red,total,east,21,98,87.0
29 | 1986-01-01,red,total,west,33,90,-10.1""")
30 |
31 |
32 | class CSVSourcesTests(unittest.TestCase):
33 | """Basic tests of a CSV DataSource object."""
34 |
35 | def setUp(self):
36 | self.csv_file = StringIO.StringIO(_TEST_CSV_CONTENT)
37 | self.data_source_obj = self.data_source_class(self.csv_file, verbose=False)
38 |
39 | def tearDown(self):
40 | self.data_source_obj.Close()
41 | self.csv_file.close()
42 |
43 | def testColumnBundle(self):
44 | """Test that column bundle is properly generated."""
45 | column_bundle = self.data_source_obj.GetColumnBundle()
46 |
47 | self.assertEqual(
48 | [c.column_id for c in column_bundle.GetColumnIterator()],
49 | ['date', 'category1', 'category2', 'category3',
50 | 'metric1', 'metric2', 'metric3'])
51 | self.assertEqual(
52 | [c.data_type for c in column_bundle.GetColumnIterator()],
53 | ['date', 'string', 'string', 'string', 'integer', 'integer', 'float'])
54 | self.assertEqual(
55 | [c.data_format for c in column_bundle.GetColumnIterator()],
56 | ['yyyy-MM-dd', '', '', '', '', '', ''])
57 | self.assertEqual(
58 | [c.concept_ref for c in column_bundle.GetColumnIterator()],
59 | ['time:day', '', 'geo:us_state', '', '', '', ''])
60 | self.assertEqual(
61 | [c.concept_extension for c in column_bundle.GetColumnIterator()],
62 | ['', '', '', '', 'quantity:ratio', '', ''])
63 | self.assertEqual(
64 | [c.slice_role for c in column_bundle.GetColumnIterator()],
65 | ['dimension', 'dimension', 'dimension', 'dimension', 'metric', 'metric',
66 | 'metric'])
67 | self.assertEqual(
68 | [c.rollup for c in column_bundle.GetColumnIterator()],
69 | [False, False, False, True, False, False, False])
70 | self.assertEqual(
71 | [c.parent_ref for c in column_bundle.GetColumnIterator()],
72 | ['', '', 'category3', '', '', '', ''])
73 | self.assertEqual(
74 | [c.total_val for c in column_bundle.GetColumnIterator()],
75 | ['', '', 'total', '', '', '', ''])
76 |
77 | def testEntityTableGeneration(self):
78 | """Test that single-concept tables are generated correctly."""
79 | table_data = self.data_source_obj.GetTableData(
80 | data_source.QueryParameters(
81 | data_source.QueryParameters.CONCEPT_QUERY, ['category2']))
82 |
83 | # Make sure quotes are properly escaped
84 | self.assertEqual(table_data.rows,
85 | [['california'], ['maine\'s'], ['oregon']])
86 |
87 | def testMultiEntityTableGeneration(self):
88 | """Test that multi-concept tables are generated correctly."""
89 | table_data = self.data_source_obj.GetTableData(
90 | data_source.QueryParameters(
91 | data_source.QueryParameters.CONCEPT_QUERY,
92 | ['category2', 'category3']))
93 |
94 | # Make sure quotes are properly escaped
95 | self.assertEqual(table_data.rows,
96 | [['california', 'west'], ['maine\'s', 'east'],
97 | ['oregon', 'west']])
98 |
99 | def testSliceTableGeneration(self):
100 | """Test that slice tables are generated correctly."""
101 | table_data = self.data_source_obj.GetTableData(
102 | data_source.QueryParameters(
103 | data_source.QueryParameters.SLICE_QUERY,
104 | ['metric3', 'category2', 'metric1', 'metric2']))
105 |
106 | self.assertEqual(
107 | table_data.rows,
108 | [[3, 'california', 89 + 99 + 293, (321.0 + 231.0 + 12.0) / 3.0],
109 | [2, 'maine\'s', 293 + 932, (32.0 + 48.0) / 2.0],
110 | [1, 'oregon', 32, 33]])
111 |
112 | def testTotalsSliceTableGeneration(self):
113 | """Test that slice tables are generated correctly with total values."""
114 | table_data = self.data_source_obj.GetTableData(
115 | data_source.QueryParameters(
116 | data_source.QueryParameters.SLICE_QUERY,
117 | ['category1', 'metric1', 'metric2', 'metric3']))
118 |
119 | self.assertEqual(
120 | table_data.rows,
121 | [['red', 21 + 33, (98.0 + 90.0) / 2.0, 2]])
122 |
123 |
124 | class CSVSourcesErrorTests(unittest.TestCase):
125 | """Tests of a CSV DataSource object for error cases."""
126 |
127 | def setUp(self):
128 | pass
129 |
130 | def testBadHeaderKey(self):
131 | """Test that unknown key in header generates error."""
132 | csv_file = StringIO.StringIO(
133 | 'date[unknown_key=unknown_value],metric\n1990,23232')
134 |
135 | self.assertRaises(
136 | data_source.DataSourceError,
137 | self.data_source_class,
138 | csv_file, False)
139 |
140 | csv_file.close()
141 |
142 | def testBadDataType(self):
143 | """Test that unknown type value generates error."""
144 | csv_file = StringIO.StringIO('date[type=unknown_type],metric\n1990,23232')
145 |
146 | self.assertRaises(
147 | data_source.DataSourceError,
148 | self.data_source_class,
149 | csv_file, False)
150 |
151 | csv_file.close()
152 |
153 | def testBadAggregation(self):
154 | """Test that unknown aggregation operator generates error."""
155 | csv_file = StringIO.StringIO(
156 | 'date[aggregation=unknown_aggregation],metric\n1990,23232')
157 |
158 | self.assertRaises(
159 | data_source.DataSourceError,
160 | self.data_source_class,
161 | csv_file, False)
162 |
163 | csv_file.close()
164 |
165 | def testBadSliceRoleKey(self):
166 | """Test that unknown value for slice_role generates error."""
167 | csv_file = StringIO.StringIO(
168 | 'date[slice_role=unknown_role],metric\n1990,23232')
169 |
170 | self.assertRaises(
171 | data_source.DataSourceError,
172 | self.data_source_class,
173 | csv_file, False)
174 |
175 | csv_file.close()
176 |
177 | def testBadColumnID(self):
178 | """Test that a badly formatted column ID generates error."""
179 | csv_file = StringIO.StringIO('my date[type=date],metric\n1990,23232')
180 |
181 | self.assertRaises(
182 | data_source.DataSourceError,
183 | self.data_source_class,
184 | csv_file, False)
185 |
186 | csv_file.close()
187 |
188 | def testBadDataRow(self):
189 | """Test that row with wrong number of entries causes error."""
190 | csv_file = StringIO.StringIO(
191 | 'date,column\n01/01/1990,abcd,1234')
192 |
193 | self.assertRaises(
194 | data_source.DataSourceError,
195 | self.data_source_class,
196 | csv_file, False)
197 |
198 | csv_file.close()
199 |
200 | def testBadParentReference(self):
201 | """Test that illegal parent reference causes error."""
202 | csv_file = StringIO.StringIO(
203 | 'date,column[parent=unknown_parent]\n01/01/1990,abcd')
204 |
205 | self.assertRaises(
206 | data_source.DataSourceError,
207 | self.data_source_class,
208 | csv_file, False)
209 |
210 | csv_file.close()
211 |
212 | def testMultipleParents(self):
213 | """Test that having multiple parent instances causes error."""
214 | csv_file = StringIO.StringIO(
215 | 'date,column1[parent=column2],column2,column3\n'
216 | '1/1/2001,val1,parent1,323\n1/2/2001,val1,parent2,123')
217 |
218 | self.assertRaises(
219 | data_source.DataSourceError,
220 | self.data_source_class,
221 | csv_file, False)
222 |
223 | csv_file.close()
224 |
--------------------------------------------------------------------------------
/tools/dspltools/packages/dspllib/data_sources/csv_utilities.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python2
2 | #
3 | # Copyright 2018 Google LLC
4 | #
5 | # Use of this source code is governed by a BSD-style
6 | # license that can be found in the LICENSE file or at
7 | # https://developers.google.com/open-source/licenses/bsd
8 |
9 | """Utility functions useful for CSV data sources."""
10 | from __future__ import print_function
11 |
12 | __author__ = 'Benjamin Yolken '
13 |
14 |
15 | import csv
16 | import re
17 | import string
18 | import warnings
19 |
20 | import data_source
21 |
22 |
23 | def _HeaderToColumn(header_string):
24 | """Parse the header string for a column.
25 |
26 | Args:
27 | header_string: The complete string for the column header
28 |
29 | Returns:
30 | A DataColumn object populated based on the header data
31 |
32 | Raises:
33 | DataSourceError: If there are any errors in parsing, e.g. if an unrecognized
34 | key is found.
35 | """
36 | # The column id must be at least one character long, and cannot contain the
37 | # characters '[', ']', ';', or whitespace
38 | parameters_match = re.match(
39 | '^([^\]\[;\s]+)(?:\[(.*)\]){0,1}$',
40 | header_string.strip().replace('"', ''))
41 |
42 | if not parameters_match:
43 | raise data_source.DataSourceError(
44 | 'Formatting error for header string: %s' % header_string)
45 |
46 | column_id = parameters_match.group(1)
47 | column = data_source.DataSourceColumn(column_id, internal_parameters={})
48 |
49 | if parameters_match.group(2):
50 | # Parse the column parameters
51 | key_value_pairs = parameters_match.group(2).split(';')
52 |
53 | for key_value_pair in key_value_pairs:
54 | try:
55 | [key, value] = key_value_pair.split('=')
56 | except ValueError:
57 | raise data_source.DataSourceError(
58 | 'Formatting error for header string: %s' % header_string)
59 |
60 | # Map the key to the appropriate field of the DataSourceColumn object
61 | if key == 'type':
62 | if value not in ['date', 'float', 'integer', 'string']:
63 | raise data_source.DataSourceError(
64 | 'Unknown data type for column %s: %s' %
65 | (column.column_id, value))
66 |
67 | column.data_type = value
68 | elif key == 'format':
69 | column.data_format = value
70 | elif key == 'concept':
71 | column.concept_ref = value
72 | elif key == 'extends':
73 | column.concept_extension = value
74 | elif key == 'parent':
75 | column.parent_ref = value
76 | elif key == 'slice_role':
77 | role_value = value.lower()
78 |
79 | if role_value not in ['dimension', 'metric']:
80 | raise data_source.DataSourceError(
81 | 'Unrecognized slice_roll in column %s: %s' %
82 | (column.column_id, value))
83 | else:
84 | column.slice_role = role_value
85 | elif key == 'rollup':
86 | if value.lower() == 'true':
87 | column.rollup = True
88 | elif value.lower() == 'false':
89 | column.rollup = False
90 | else:
91 | raise data_source.DataSourceError(
92 | 'Unrecognized boolean value in column %s: %s' %
93 | (column.column_id, value))
94 | elif key == 'total_val':
95 | column.total_val = value
96 | elif key == 'dropif':
97 | column.internal_parameters['dropif_val'] = value
98 | elif key == 'zeroif':
99 | column.internal_parameters['zeroif_val'] = value
100 | elif key == 'aggregation':
101 | if string.lower(value) not in ['sum', 'max', 'min', 'avg', 'count']:
102 | raise data_source.DataSourceError(
103 | 'Unknown aggregation for column %s: %s' %
104 | (column.column_id, value))
105 |
106 | column.internal_parameters['aggregation'] = value
107 | else:
108 | raise data_source.DataSourceError(
109 | 'Unknown parameter for column %s: %s' %
110 | (column.column_id, key))
111 | return column
112 |
113 |
114 | def ConstructColumnBundle(csv_file, verbose=True):
115 | """Construct a ColumnBundle from the header information in a CSV file.
116 |
117 | Args:
118 | csv_file: The complete string for the column header
119 | verbose: Print out extra information to stdout
120 |
121 | Returns:
122 | A data_source.ColumnBundle object populated based on the CSV header
123 |
124 | Raises:
125 | DataSourceError: If there are any parsing errors or data
126 | inconsistencies
127 | """
128 | # Get the first and second rows of the CSV
129 | header_csv_reader = csv.reader(csv_file, delimiter=',', quotechar='"')
130 | header_row_values = next(header_csv_reader)
131 | second_row_values = next(header_csv_reader)
132 | csv_file.seek(0)
133 |
134 | # Check that second row is properly formatted
135 | if len(header_row_values) != len(second_row_values):
136 | raise data_source.DataSourceError(
137 | 'Number of columns in row 2 (%d) does not match number '
138 | 'expected (%d)' % (len(second_row_values), len(header_row_values)))
139 |
140 | column_bundle = data_source.DataSourceColumnBundle()
141 |
142 | for header_element in header_row_values:
143 | column_bundle.AddColumn(_HeaderToColumn(header_element))
144 |
145 | num_date_columns = 0
146 | has_metric_column = False
147 | column_ids = [column.column_id for column in
148 | column_bundle.GetColumnIterator()]
149 |
150 | # Iterate through columns, populating and refining DataSourceColumn
151 | # parameters as necessary
152 | for c, column in enumerate(column_bundle.GetColumnIterator()):
153 | if verbose:
154 | print('\nEvaluating column %s' % column.column_id)
155 |
156 | # Check data type
157 | if not column.data_type:
158 | column.data_type = (
159 | data_source.GuessDataType(second_row_values[c], column.column_id))
160 |
161 | if verbose:
162 | print('Guessing that column %s is of type %s' % (
163 | column.column_id, column.data_type))
164 |
165 | # Check slice type
166 | if not column.slice_role:
167 | if column.data_type == 'integer' or column.data_type == 'float':
168 | column.slice_role = 'metric'
169 | else:
170 | column.slice_role = 'dimension'
171 |
172 | if verbose:
173 | print('Guessing that column %s is a %s' % (
174 | column.column_id, column.slice_role))
175 |
176 | # Check aggregation
177 | if column.slice_role == 'metric':
178 | has_metric_column = True
179 |
180 | if 'aggregation' not in column.internal_parameters:
181 | column.internal_parameters['aggregation'] = 'SUM'
182 |
183 | if verbose:
184 | print('Guessing that column %s should be aggregated by %s' % (
185 | column.column_id, column.internal_parameters['aggregation']))
186 |
187 | # Check parent
188 | if column.parent_ref:
189 | if column.parent_ref not in column_ids:
190 | raise data_source.DataSourceError(
191 | 'Column %s references a parent not defined in this dataset: %s' %
192 | (column.column_id, column.parent_ref))
193 |
194 | parent_column = column_bundle.GetColumnByID(column.parent_ref)
195 |
196 | if not parent_column.rollup:
197 | parent_column.rollup = True
198 |
199 | if verbose:
200 | print('Making column %s rollup since it is a parent to column %s'
201 | % (parent_column.column_id, column.column_id))
202 |
203 | # Check date format and concept
204 | if column.data_type == 'date':
205 | num_date_columns += 1
206 |
207 | if not column.data_format:
208 | column.data_format = (
209 | data_source.GuessDateFormat(second_row_values[c]))
210 |
211 | if not column.concept_ref:
212 | column.concept_ref = (
213 | data_source.GuessDateConcept(column.data_format))
214 |
215 | if verbose:
216 | print('Guessing that column %s is formatted as %s and '
217 | 'corresponds to %s' % (
218 | column.column_id, column.data_format, column.concept_ref))
219 |
220 | # Warn user if their file will not produce interesting DSPL visualizations
221 | if num_date_columns == 0:
222 | warnings.warn('Input file does not have a date column',
223 | data_source.DataSourceWarning)
224 |
225 | elif num_date_columns > 1:
226 | warnings.warn('Input file has more than one date column',
227 | data_source.DataSourceWarning)
228 |
229 | if not has_metric_column:
230 | warnings.warn('Input file does not have any metrics',
231 | data_source.DataSourceWarning)
232 |
233 | return column_bundle
234 |
--------------------------------------------------------------------------------
/samples/eurostat/unemployment/eurostat-unemployment-dspl-v1.json:
--------------------------------------------------------------------------------
1 | {
2 | "@context": [
3 | "http://schema.org",
4 | {
5 | "name": { "@container": "@language" },
6 | "description": { "@container": "@language" },
7 | "url": { "@container": "@language" }
8 | }
9 | ],
10 | "@type": "StatisticalDataset",
11 | "@id": "#eurostat-unemployment",
12 | "name": {
13 | "en": "Unemployment in Europe (monthly)",
14 | "de": "Arbeitslosigkeit in Europa (monatlich)",
15 | "fr": "Le Chômage en Europe (mensuel)"
16 | },
17 | "description": {
18 | "en": "Harmonized unemployment data for European countries. This dataset was prepared by Google based on data downloaded from Eurostat.",
19 | "de": "Harmonisierte Daten zur Arbeitslosigkeit für europäische Länder. Dieser Datensatz wurde von Google aufbereitet, basierend auf online Daten von Eurostat.",
20 | "fr": "Données harmonisées sur le chômage dans les pays européens. Ces données ont été préparées par Google sur la base de données téléchargées à partir d'Eurostat."
21 | },
22 | "url": {
23 | "en": "http://epp.eurostat.ec.europa.eu/portal/page/portal/lang-en/employment_unemployment_lfs/introduction",
24 | "de": "http://epp.eurostat.ec.europa.eu/portal/page/portal/lang-fr/employment_unemployment_lfs/introduction",
25 | "fr": "http://epp.eurostat.ec.europa.eu/portal/page/portal/lang-de/employment_unemployment_lfs/introduction"
26 | },
27 | "license": "https://ec.europa.eu/eurostat/about/policies/copyright",
28 | "creator":{
29 | "@type":"Organization",
30 | "url": "https://ec.europa.eu/eurostat",
31 | "name":"Eurostat",
32 | "contactPoint": [
33 | {
34 | "@type":"ContactPoint",
35 | "name": "Eurostat Multilingual User Support Network"
36 | "contactType": "Central Support",
37 | "telephone":"+352 4301 36789",
38 | },
39 | {
40 | "@type":"ContactPoint",
41 | "name": "Eurostat Multilingual User Support Network"
42 | "contactType": "Republic of Ireland",
43 | "availableLanguage": "en",
44 | "telephone":"+353 151 33080",
45 | }
46 | ]
47 | },
48 | "distribution":[
49 | {
50 | "@type":"DataDownload",
51 | "encodingFormat":"text/tab-separated-values",
52 | "contentUrl":"https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2Fei_lmhu_m.tsv.gz"
53 | },
54 | {
55 | "@type":"DataDownload",
56 | "encodingFormat":"application/vnd.sdmx.genericdata+xml;version=2.0",
57 | "contentUrl":"https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2Fei_lmhu_m.sdmx.zip"
58 | }
59 | ],
60 | "temporalCoverage":"1993-01/2010-12",
61 | "spatialCoverage":{
62 | "@type":"Place",
63 | "geo":{
64 | "@type":"GeoShape",
65 | "name": "European Union",
66 | "box":"34.633285 -10.468556 70.096054 34.597916"
67 | }
68 | },
69 | "measure": [
70 | {
71 | "@type": "StatisticalMeasure",
72 | "@id": "#unemployment",
73 | "sameAs": "https://www.wikidata.org/wiki/Q41171",
74 | "name": {
75 | "en": "Unemployment (monthly)",
76 | "de": "Arbeitslosigkeit (monatlich)",
77 | "fr": "Chômeurs (mensuel)"
78 | },
79 | "description": {
80 | "en": "The total number of people unemployed",
81 | "de": "Anzahl der Arbeitslosen",
82 | "fr":" Le nombre total de chômeurs"
83 | },
84 | "url": {
85 | "en": "http://ec.europa.eu/eurostat/product?code=une_nb_m&language=en",
86 | "de": "http://ec.europa.eu/eurostat/product?code=une_nb_m&language=de",
87 | "fr": "http://ec.europa.eu/eurostat/product?code=une_nb_m&language=fr"
88 | },
89 | "unitCode": "IE"
90 | },
91 | {
92 | "@type": "StatisticalMeasure",
93 | "@id": "#unemployment_rate",
94 | "sameAs": "https://www.wikidata.org/wiki/Q1787954",
95 | "name": {
96 | "en": "Unemployment rate (monthly)",
97 | "de": "Arbeitslosenquote (monatlich)",
98 | "fr": "Taux de chômage (mensuel)"
99 | },
100 | "description": {
101 | "en": "The unemployment rate represents unemployed persons as a percentage of the labour force. The labour force is the total number of people employed and unemployed.",
102 | "de": "Die Arbeitslosenquote ist definiert als der prozentuale Anteil der Arbeitslosen an den Erwerbspersonen. Die Erwerbspersonen umfassen die Erwerbstätigen und die Arbeitslosen.",
103 | "fr": "Le taux de chômage représente le pourcentage de chômeurs dans la population active. La population active représente le nombre total des personnes ayant un emploi ou étant au chômage."
104 | },
105 | "url": {
106 | "en": "http://ec.europa.eu/eurostat/product?code=une_rt_m&language=en",
107 | "de": "http://ec.europa.eu/eurostat/product?code=une_rt_m&language=de",
108 | "fr": "http://ec.europa.eu/eurostat/product?code=une_rt_m&language=fr"
109 | },
110 | "unitCode": "P1"
111 | }
112 | ],
113 | "dimension": [
114 | {
115 | "@type": "CategoricalDimension",
116 | "@id": "#country_group",
117 | "codeList": "country_groups.csv"
118 | },
119 | {
120 | "@type": "CategoricalDimension",
121 | "@id": "#country",
122 | "codeList": "countries.csv",
123 | "equivalentType": "Country"
124 | },
125 | {
126 | "@type": "CategoricalDimension",
127 | "@id": "#age_group",
128 | "codeList": "age_groups.csv"
129 | },
130 | {
131 | "@type": "CategoricalDimension",
132 | "@id": "#sex",
133 | "codeList": "sexes.csv"
134 | },
135 | {
136 | "@type": "CategoricalDimension",
137 | "@id": "#seasonality",
138 | "codeList": "seasonalities.csv"
139 | },
140 | {
141 | "@type": "TimeDimension",
142 | "@id": "#month",
143 | "equivalentType": "xsd:gYearMonth",
144 | "datePattern": "yyyy.MM"
145 | }
146 | ],
147 | "footnote": "footnotes.csv",
148 | "slice": [
149 | {
150 | "@type": "DataSlice",
151 | "@id": "#country_age",
152 | "dimension": [
153 | "#country",
154 | "#age_group",
155 | "#month"
156 | ],
157 | "measure": [
158 | "#unemployment",
159 | "#unemployment_rate"
160 | ],
161 | "observation": "country_age.csv"
162 | },
163 | {
164 | "@type": "DataSlice",
165 | "@id": "#country_group_age",
166 | "dimension": [
167 | "#country_group",
168 | "#age_group",
169 | "#month"
170 | ],
171 | "measure": [
172 | "#unemployment",
173 | "#unemployment_rate"
174 | ],
175 | "observation": "country_group_age.csv"
176 | },
177 | {
178 | "@type": "DataSlice",
179 | "@id": "#country_group_sex_age",
180 | "dimension": [
181 | "#country_group",
182 | "#sex",
183 | "#age_group",
184 | "#month"
185 | ],
186 | "measure": [
187 | "#unemployment",
188 | "#unemployment_rate"
189 | ],
190 | "observation": "country_group_sex_age.csv"
191 | },
192 | {
193 | "@type": "DataSlice",
194 | "@id": "#country_group_sex",
195 | "dimension": [
196 | "#country_group",
197 | "#sex",
198 | "#month"
199 | ],
200 | "measure": [
201 | "#unemployment",
202 | "#unemployment_rate"
203 | ],
204 | "observation": "country_group_sex.csv"
205 | },
206 | {
207 | "@type": "DataSlice",
208 | "@id": "#country_group_total",
209 | "dimension": [
210 | "#country_group",
211 | "#month"
212 | ],
213 | "measure": [
214 | "#unemployment",
215 | "#unemployment_rate"
216 | ],
217 | "observation": "country_group_total.csv"
218 | },
219 | {
220 | "@type": "DataSlice",
221 | "@id": "#country_sex_age",
222 | "dimension": [
223 | "#country",
224 | "#sex",
225 | "#age_group",
226 | "#month"
227 | ],
228 | "measure": [
229 | "#unemployment",
230 | "#unemployment_rate"
231 | ],
232 | "observation": "country_sex_age.csv"
233 | },
234 | {
235 | "@type": "DataSlice",
236 | "@id": "#country_sex",
237 | "dimension": [
238 | "#country",
239 | "#sex",
240 | "#month"
241 | ],
242 | "measure": [
243 | "#unemployment",
244 | "#unemployment_rate"
245 | ],
246 | "observation": "country_sex.csv"
247 | },
248 | {
249 | "@type": "DataSlice",
250 | "@id": "#country_total",
251 | "dimension": [
252 | "#country",
253 | "#month"
254 | ],
255 | "measure": [
256 | "#unemployment",
257 | "#unemployment_rate"
258 | ],
259 | "observation": "country_total.csv"
260 | }
261 | ]
262 | }
263 |
--------------------------------------------------------------------------------