Datasets with error {{ error_type }}
6 |-
7 | {% if datasets %}
8 | {% for dataset in datasets %}
9 |
-
10 | {{ dataset.metadata.name }} ({{ dataset.id }})
11 |12 |15 |{{ dataset.error_details.exception }}
13 |{{ dataset.error_details.traceback }}14 |
16 | {% endfor %}
17 | {% else %}
18 | - No dataset with that error 19 | {% endif %} 20 |
11 |
27 | );
28 | }
29 | }
30 |
31 | export {FilterContainer};
32 |
--------------------------------------------------------------------------------
/lib_materialize/README.rst:
--------------------------------------------------------------------------------
1 | Datamart materialization library
2 | ================================
3 |
4 | This library can materialize datasets from Auctus, NYU's dataset search engine. You can use it to materialize search results directly on your side without relying on the server. It is also used internally by the service to materialize datasets (the ``/download`` endpoint downloads the dataset using this library then sends it to you).
5 |
6 | See also:
7 |
8 | * `The datamart-rest library for search/augmentation
12 |
25 | {this.props.title}
13 | this.props.onClose()} 15 | className="d-inline text-muted ml-1" 16 | style={{cursor: 'pointer'}} 17 | title="Remove this filter" 18 | > 19 |{this.props.children}
26 | {props.children}
;
19 | }
20 |
21 | function LinkButton(
22 | props: React.PropsWithChildren<{href: string; message?: string}>
23 | ) {
24 | return (
25 |
13 | 
14 | Auctus Dataset Search
15 |
16 | );
17 | }
18 |
19 | function HorizontalLogo(props: {onClick?: () => void}) {
20 | const style = props.onClick ? {cursor: 'pointer'} : undefined;
21 | return (
22 |
27 |
28 | Auctus
29 |
30 | );
31 | }
32 |
33 | function CenteredHorizontalLogo(props: {onClick?: () => void}) {
34 | return (
35 |
36 |
38 | );
39 | }
40 |
41 | export {VerticalLogo, HorizontalLogo, CenteredHorizontalLogo};
42 |
--------------------------------------------------------------------------------
/frontend/src/components/Chip/Chip.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import * as Icon from 'react-feather';
3 | import './Chip.css';
4 |
5 | interface ChipProps {
6 | label: string;
7 | onClose?: () => void;
8 | onEdit?: () => void;
9 | icon?: Icon.Icon;
10 | }
11 |
12 | function Chip(props: ChipProps) {
13 | let classes = 'chip chip-outline';
14 | // chip-primary
15 | // chip-clickable
16 | if (props.onClose) {
17 | classes += ' chip-closeable';
18 | }
19 | return (
20 |
21 | {props.icon && (
22 |
43 | );
44 | }
45 |
46 | function ChipGroup(props: React.PropsWithChildren<{}>) {
47 | return
23 |
25 | )}
26 |
27 | {props.label}
28 |
29 | {props.onEdit ? (
30 |
33 | ) : (
34 | ''
35 | )}
36 |
37 | {props.onClose && (
38 |
39 |
41 | )}
42 | {props.children}
;
48 | }
49 |
50 | export {Chip, ChipGroup};
51 |
--------------------------------------------------------------------------------
/discovery/ckan/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'elasticsearch~=7.0',
10 | 'requests',
11 | 'datamart-core',
12 | ]
13 | setup(name='datamart-ckan-discovery-service',
14 | version='0.0',
15 | py_modules=['ckan_discovery'],
16 | install_requires=req,
17 | description="CKAN discovery service for Auctus",
18 | author="Remi Rampin",
19 | author_email='remi.rampin@nyu.edu',
20 | maintainer="Remi Rampin",
21 | maintainer_email='remi.rampin@nyu.edu',
22 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
23 | project_urls={
24 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
26 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
27 | },
28 | long_description="CKAN discovery service for Auctus",
29 | license='Apache-2.0',
30 | keywords=['auctus', 'datamart'],
31 | classifiers=[
32 | 'Development Status :: 4 - Beta',
33 | 'Intended Audience :: Science/Research',
34 | 'License :: OSI Approved :: Apache Software License',
35 | 'Operating System :: Unix',
36 | 'Programming Language :: Python :: 3 :: Only',
37 | 'Topic :: Scientific/Engineering :: Information Analysis'])
38 |
--------------------------------------------------------------------------------
/discovery/zenodo/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'elasticsearch~=7.0',
10 | 'requests',
11 | 'datamart-core',
12 | ]
13 | setup(name='datamart-zenodo-discovery-service',
14 | version='0.0',
15 | py_modules=['zenodo_discovery'],
16 | install_requires=req,
17 | description="Zenodo discovery service for Auctus",
18 | author="Remi Rampin",
19 | author_email='remi.rampin@nyu.edu',
20 | maintainer="Remi Rampin",
21 | maintainer_email='remi.rampin@nyu.edu',
22 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
23 | project_urls={
24 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
26 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
27 | },
28 | long_description="Zenodo discovery service for Auctus",
29 | license='Apache-2.0',
30 | keywords=['auctus', 'datamart'],
31 | classifiers=[
32 | 'Development Status :: 4 - Beta',
33 | 'Intended Audience :: Science/Research',
34 | 'License :: OSI Approved :: Apache Software License',
35 | 'Operating System :: Unix',
36 | 'Programming Language :: Python :: 3 :: Only',
37 | 'Topic :: Scientific/Engineering :: Information Analysis'])
38 |
--------------------------------------------------------------------------------
/lib_materialize/datamart_materialize/excel.py:
--------------------------------------------------------------------------------
1 | import contextlib
2 | import csv
3 | from datetime import datetime
4 | import openpyxl
5 |
6 | from .common import UnsupportedConversion
7 | from .utils import SimpleConverter
8 |
9 |
10 | def xlsx_to_csv(source_filename, dest_fileobj):
11 | with contextlib.ExitStack() as stack:
12 | fp = stack.enter_context(open(source_filename, 'rb'))
13 | workbook = stack.enter_context(contextlib.closing(
14 | openpyxl.load_workbook(fp, read_only=True)
15 | ))
16 |
17 | sheets = workbook.worksheets
18 | if len(sheets) != 1:
19 | raise UnsupportedConversion(
20 | "Excel workbook has %d sheets" % len(sheets)
21 | )
22 | sheet, = sheets
23 |
24 | writer = csv.writer(dest_fileobj)
25 | for values in sheet.iter_rows(values_only=True):
26 | values = [
27 | # Avoid forced decimal point on integers
28 | '{0:g}'.format(v) if isinstance(v, float)
29 | # Decode dates into ISO-8601 strings
30 | else v.isoformat() if isinstance(v, datetime)
31 | else v
32 | for v in values
33 | ]
34 |
35 | writer.writerow(values)
36 |
37 |
38 | class ExcelConverter(SimpleConverter):
39 | """Adapter converting Excel files to CSV.
40 | """
41 | transform = staticmethod(xlsx_to_csv)
42 |
--------------------------------------------------------------------------------
/discovery/isi/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'elasticsearch~=7.0',
10 | 'requests',
11 | 'datamart-core',
12 | ]
13 | setup(name='datamart-isi-discovery-service',
14 | version='0.0',
15 | py_modules=['isi_discovery'],
16 | install_requires=req,
17 | description="ISI Datamart discovery service for Auctus",
18 | author="Remi Rampin",
19 | author_email='remi.rampin@nyu.edu',
20 | maintainer="Remi Rampin",
21 | maintainer_email='remi.rampin@nyu.edu',
22 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
23 | project_urls={
24 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
26 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
27 | },
28 | long_description="ISI Datamart discovery service for Auctus",
29 | license='Apache-2.0',
30 | keywords=['auctus', 'datamart'],
31 | classifiers=[
32 | 'Development Status :: 4 - Beta',
33 | 'Intended Audience :: Science/Research',
34 | 'License :: OSI Approved :: Apache Software License',
35 | 'Operating System :: Unix',
36 | 'Programming Language :: Python :: 3 :: Only',
37 | 'Topic :: Scientific/Engineering :: Information Analysis'])
38 |
--------------------------------------------------------------------------------
/discovery/socrata/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'elasticsearch~=7.0',
10 | 'sodapy',
11 | 'datamart-core',
12 | ]
13 | setup(name='datamart-socrata-discovery-service',
14 | version='0.0',
15 | py_modules=['socrata_discovery'],
16 | install_requires=req,
17 | description="Socrata discovery service for Auctus",
18 | author="Remi Rampin",
19 | author_email='remi.rampin@nyu.edu',
20 | maintainer="Remi Rampin",
21 | maintainer_email='remi.rampin@nyu.edu',
22 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
23 | project_urls={
24 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
26 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
27 | },
28 | long_description="Socrata discovery service for Auctus",
29 | license='Apache-2.0',
30 | keywords=['auctus', 'datamart'],
31 | classifiers=[
32 | 'Development Status :: 4 - Beta',
33 | 'Intended Audience :: Science/Research',
34 | 'License :: OSI Approved :: Apache Software License',
35 | 'Operating System :: Unix',
36 | 'Programming Language :: Python :: 3 :: Only',
37 | 'Topic :: Scientific/Engineering :: Information Analysis'])
38 |
--------------------------------------------------------------------------------
/frontend/src/components/ui/DropdownMenu/DropdownMenu.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 |
3 | interface Props {
4 | children: (api: {onClick: () => void; active: boolean}) => JSX.Element;
5 | }
6 |
7 | interface State {
8 | active: boolean;
9 | }
10 |
11 | class DropdownMenu extends React.Component (this.ref = node)}>
44 | {this.props.children({
45 | onClick: this.toggleState,
46 | active: this.state.active,
47 | })}
48 |
49 | );
50 | }
51 | }
52 |
53 | export {DropdownMenu};
54 |
--------------------------------------------------------------------------------
/frontend/src/components/ui/Tabs/Tabs.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import './Tabs.css';
3 |
4 | class Tabs extends React.PureComponent {
5 | render() {
6 | return - {this.props.children}
{this.props.children}
;
31 | }
32 | }
33 |
34 | interface TabPaneProps {
35 | id: string;
36 | active: boolean;
37 | }
38 |
39 | class TabPane extends React.PureComponent
46 | {this.props.children}
47 |
48 | );
49 | }
50 | }
51 |
52 | export {Tabs, Tab, TabContent, TabPane};
53 |
--------------------------------------------------------------------------------
/contrib/k8s/secrets.jsonnet:
--------------------------------------------------------------------------------
1 | // Set 'private_app: true' in the config to password-protect frontend & API
2 | // You can create this file using the htpasswd tool
3 | local private_app_password = |||
4 | auctus:$apr1$ECD/OaHB$CMBSkoEdcA/2uX8gPZM3y1
5 | |||;
6 |
7 | local amqp_user = 'auctususer';
8 | local amqp_password = 'auctuspassword';
9 | local admin_password = 'auctuspassword';
10 | local s3_key = 'devkey';
11 | local s3_secret = 'devpassword';
12 | local gcs_creds = std.base64('');
13 |
14 | {
15 | 'secrets.yml': std.manifestYamlStream([
16 | {
17 | apiVersion: 'v1',
18 | kind: 'Secret',
19 | type: 'Opaque',
20 | metadata: {
21 | name: 'secrets',
22 | },
23 | local data = {
24 | 'amqp.user': amqp_user,
25 | 'amqp.password': amqp_password,
26 | 'admin.password': admin_password,
27 | 's3.key': s3_key,
28 | 's3.secret': s3_secret,
29 | 'gcs.creds': gcs_creds,
30 | 'smtp.user': 'auctususer',
31 | 'smtp.password': 'auctuspassword',
32 | },
33 | data: {
34 | [k]: std.base64(data[k])
35 | for k in std.objectFields(data)
36 | },
37 | },
38 | {
39 | apiVersion: 'v1',
40 | kind: 'Secret',
41 | type: 'Opaque',
42 | metadata: {
43 | name: 'basic-auth',
44 | },
45 | data: {
46 | auth: std.base64(private_app_password),
47 | },
48 | },
49 | ]),
50 | }
51 |
--------------------------------------------------------------------------------
/apiserver/apiserver/enhance_metadata.py:
--------------------------------------------------------------------------------
1 | from datamart_materialize.d3m import d3m_metadata
2 |
3 |
4 | def enhance_metadata(result):
5 | """Add more metadata (e.g. D3M) from the original metadata.
6 |
7 | :param result: A dict with 'id' and 'metadata' keys
8 | :type result: dict
9 | :return: A dict with the 'metadata' key and additional keys such as
10 | 'd3m-metadata'
11 | """
12 | # Generate metadata in D3M format
13 | result = dict(
14 | result,
15 | d3m_dataset_description=d3m_metadata(result['id'], result['metadata']),
16 | )
17 |
18 | # Add temporal coverage information to columns for compatibility
19 | if result['metadata'].get('temporal_coverage'):
20 | columns = list(result['metadata']['columns'])
21 | for temporal in result['metadata']['temporal_coverage']:
22 | # Only works for temporal coverage extracted from a single column
23 | if len(temporal['column_indexes']) == 1:
24 | idx = temporal['column_indexes'][0]
25 | columns[idx] = dict(
26 | columns[idx],
27 | coverage=temporal['ranges'],
28 | )
29 | if 'temporal_resolution' in temporal:
30 | columns[idx]['temporal_resolution'] = \
31 | temporal['temporal_resolution']
32 |
33 | result['metadata'] = dict(result['metadata'], columns=columns)
34 |
35 | return result
36 |
--------------------------------------------------------------------------------
/tests/data/hourly.csv:
--------------------------------------------------------------------------------
1 | aug_date,rain
2 | 2019-06-12T00:00:00,no
3 | 2019-06-12T01:00:00,no
4 | 2019-06-12T02:00:00,no
5 | 2019-06-12T03:00:00,yes
6 | 2019-06-12T04:00:00,no
7 | 2019-06-12T05:00:00,yes
8 | 2019-06-12T06:00:00,yes
9 | 2019-06-12T07:00:00,no
10 | 2019-06-12T08:00:00,no
11 | 2019-06-12T09:00:00,no
12 | 2019-06-12T10:00:00,yes
13 | 2019-06-12T11:00:00,yes
14 | 2019-06-12T12:00:00,yes
15 | 2019-06-12T13:00:00,yes
16 | 2019-06-12T14:00:00,yes
17 | 2019-06-12T15:00:00,no
18 | 2019-06-12T16:00:00,no
19 | 2019-06-12T17:00:00,yes
20 | 2019-06-12T18:00:00,yes
21 | 2019-06-12T19:00:00,yes
22 | 2019-06-12T20:00:00,yes
23 | 2019-06-12T21:00:00,yes
24 | 2019-06-12T22:00:00,no
25 | 2019-06-12T23:00:00,no
26 | 2019-06-13T00:00:00,no
27 | 2019-06-13T01:00:00,no
28 | 2019-06-13T02:00:00,no
29 | 2019-06-13T03:00:00,no
30 | 2019-06-13T04:00:00,yes
31 | 2019-06-13T05:00:00,no
32 | 2019-06-13T06:00:00,no
33 | 2019-06-13T07:00:00,yes
34 | 2019-06-13T08:00:00,no
35 | 2019-06-13T09:00:00,yes
36 | 2019-06-13T10:00:00,yes
37 | 2019-06-13T11:00:00,yes
38 | 2019-06-13T12:00:00,yes
39 | 2019-06-13T13:00:00,yes
40 | 2019-06-13T14:00:00,yes
41 | 2019-06-13T15:00:00,no
42 | 2019-06-13T16:00:00,yes
43 | 2019-06-13T17:00:00,yes
44 | 2019-06-13T18:00:00,no
45 | 2019-06-13T19:00:00,yes
46 | 2019-06-13T20:00:00,yes
47 | 2019-06-13T21:00:00,yes
48 | 2019-06-13T22:00:00,yes
49 | 2019-06-13T23:00:00,yes
50 | 2019-06-14T00:00:00,yes
51 | 2019-06-14T01:00:00,yes
52 | 2019-06-14T02:00:00,yes
53 | 2019-06-14T03:00:00,yes
54 |
--------------------------------------------------------------------------------
/discovery/noaa/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'elasticsearch~=7.0',
10 | 'requests',
11 | 'datamart-core',
12 | ]
13 | setup(name='datamart-noaa-discovery-service',
14 | version='0.0',
15 | packages=['noaa_discovery'],
16 | package_data={'noaa_discovery': [
17 | 'noaa_city_stations.csv',
18 | ]},
19 | install_requires=req,
20 | description="NOAA discovery service for Auctus",
21 | author="Remi Rampin",
22 | author_email='remi.rampin@nyu.edu',
23 | maintainer="Remi Rampin",
24 | maintainer_email='remi.rampin@nyu.edu',
25 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
26 | project_urls={
27 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
28 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
29 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
30 | },
31 | long_description="NOAA discovery service for Auctus",
32 | license='Apache-2.0',
33 | keywords=['auctus', 'datamart'],
34 | classifiers=[
35 | 'Development Status :: 4 - Beta',
36 | 'Intended Audience :: Science/Research',
37 | 'License :: OSI Approved :: Apache Software License',
38 | 'Operating System :: Unix',
39 | 'Programming Language :: Python :: 3 :: Only',
40 | 'Topic :: Scientific/Engineering :: Information Analysis'])
41 |
--------------------------------------------------------------------------------
/lib_materialize/datamart_materialize/excel97.py:
--------------------------------------------------------------------------------
1 | import csv
2 | import xlrd
3 | import xlrd.sheet
4 |
5 | from .common import UnsupportedConversion
6 | from .utils import SimpleConverter
7 |
8 |
9 | def xls_to_csv(source_filename, dest_fileobj):
10 | with xlrd.open_workbook(source_filename) as workbook:
11 | datemode = workbook.datemode
12 | sheets = workbook.sheets()
13 | if len(sheets) != 1:
14 | raise UnsupportedConversion(
15 | "Excel workbook has %d sheets" % len(sheets)
16 | )
17 | sheet, = sheets
18 |
19 | writer = csv.writer(dest_fileobj)
20 | for row_num in range(sheet.nrows):
21 | values = sheet.row_values(row_num)
22 |
23 | for col_num, cell_type in enumerate(sheet.row_types(row_num)):
24 | if cell_type == xlrd.sheet.XL_CELL_DATE:
25 | # Decode dates into ISO-8601 strings
26 | values[col_num] = xlrd.xldate_as_datetime(
27 | values[col_num],
28 | datemode,
29 | ).isoformat()
30 | elif cell_type == xlrd.sheet.XL_CELL_NUMBER:
31 | # Avoid forced decimal point on integers
32 | values[col_num] = '{0:g}'.format(values[col_num])
33 |
34 | writer.writerow(values)
35 |
36 |
37 | class Excel97Converter(SimpleConverter):
38 | """Adapter converting Excel files to CSV.
39 | """
40 | transform = staticmethod(xls_to_csv)
41 |
--------------------------------------------------------------------------------
/discovery/worldbank/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'elasticsearch~=7.0',
10 | 'beautifulsoup4[html5lib]',
11 | 'pandas',
12 | 'datamart-core',
13 | 'datamart-profiler',
14 | ]
15 | setup(name='datamart-worldbank-discovery-service',
16 | version='0.0',
17 | py_modules=['worldbank_discovery'],
18 | install_requires=req,
19 | description="World Bank indicator discovery service for Auctus",
20 | author="Remi Rampin",
21 | author_email='remi.rampin@nyu.edu',
22 | maintainer="Remi Rampin",
23 | maintainer_email='remi.rampin@nyu.edu',
24 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | project_urls={
26 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
27 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
28 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
29 | },
30 | long_description="World Bank indicator discovery service for Auctus",
31 | license='Apache-2.0',
32 | keywords=['auctus', 'datamart'],
33 | classifiers=[
34 | 'Development Status :: 4 - Beta',
35 | 'Intended Audience :: Science/Research',
36 | 'License :: OSI Approved :: Apache Software License',
37 | 'Operating System :: Unix',
38 | 'Programming Language :: Python :: 3 :: Only',
39 | 'Topic :: Scientific/Engineering :: Information Analysis'])
40 |
--------------------------------------------------------------------------------
/discovery/uaz_indicators/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'requests',
10 | 'datamart-core',
11 | ]
12 | setup(name='datamart-uaz-indicators-service',
13 | version='0.0',
14 | py_modules=['uaz_indicators'],
15 | install_requires=req,
16 | description="Auctus discovery service for indicators from the " +
17 | "University of Arizona",
18 | author="Remi Rampin",
19 | author_email='remi.rampin@nyu.edu',
20 | maintainer="Remi Rampin",
21 | maintainer_email='remi.rampin@nyu.edu',
22 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
23 | project_urls={
24 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
26 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
27 | },
28 | long_description="Auctus discovery service for indicators from the " +
29 | "University of Arizona",
30 | license='Apache-2.0',
31 | keywords=['auctus', 'datamart'],
32 | classifiers=[
33 | 'Development Status :: 4 - Beta',
34 | 'Intended Audience :: Science/Research',
35 | 'License :: OSI Approved :: Apache Software License',
36 | 'Operating System :: Unix',
37 | 'Programming Language :: Python :: 3 :: Only',
38 | 'Topic :: Scientific/Engineering :: Information Analysis'])
39 |
--------------------------------------------------------------------------------
/lib_augmentation/setup.py:
--------------------------------------------------------------------------------
1 | import io
2 | import os
3 | from setuptools import setup
4 |
5 |
6 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
7 |
8 |
9 | req = [
10 | 'pandas',
11 | 'numpy',
12 | 'datamart-materialize==0.11',
13 | 'datamart-profiler==0.11',
14 | ]
15 | with io.open('README.rst', encoding='utf-8') as fp:
16 | description = fp.read()
17 | setup(name='datamart-augmentation',
18 | version='0.10',
19 | packages=['datamart_augmentation'],
20 | install_requires=req,
21 | description="Data augmentation functions for Auctus",
22 | author="Remi Rampin",
23 | author_email='remi.rampin@nyu.edu',
24 | maintainer="Remi Rampin",
25 | maintainer_email='remi.rampin@nyu.edu',
26 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
27 | project_urls={
28 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
29 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
30 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
31 | },
32 | long_description=description,
33 | license='Apache-2.0',
34 | keywords=['auctus', 'datamart'],
35 | classifiers=[
36 | 'Development Status :: 4 - Beta',
37 | 'Intended Audience :: Science/Research',
38 | 'License :: OSI Approved :: Apache Software License',
39 | 'Operating System :: OS Independent',
40 | 'Programming Language :: Python :: 3 :: Only',
41 | 'Topic :: Scientific/Engineering :: Information Analysis'])
42 |
--------------------------------------------------------------------------------
/frontend/src/components/DateFilter/DateFilter.css:
--------------------------------------------------------------------------------
1 | .react-datepicker__input-container input {
2 | border: 1px solid #ced4da;
3 | padding: 6px 10px;
4 | }
5 |
6 |
7 | .react-datepicker__day--in-range:hover,
8 | .react-datepicker__day--in-selecting-range:hover,
9 | .react-datepicker__day--selected:hover,
10 | .react-datepicker__month-text--in-range:hover,
11 | .react-datepicker__month-text--in-selecting-range:hover,
12 | .react-datepicker__month-text--selected:hover,
13 | .react-datepicker__quarter-text--in-range:hover,
14 | .react-datepicker__quarter-text--in-selecting-range:hover,
15 | .react-datepicker__quarter-text--selected:hover {
16 | background-color: #2e1b59;
17 | }
18 |
19 | .react-datepicker__day--keyboard-selected:hover,
20 | .react-datepicker__month-text--keyboard-selected:hover,
21 | .react-datepicker__quarter-text--keyboard-selected:hover {
22 | background-color: #2e1b59;
23 | }
24 |
25 | .react-datepicker__day--in-range,
26 | .react-datepicker__day--in-selecting-range,
27 | .react-datepicker__day--selected,
28 | .react-datepicker__month-text--in-range,
29 | .react-datepicker__month-text--in-selecting-range,
30 | .react-datepicker__month-text--selected,
31 | .react-datepicker__quarter-text--in-range,
32 | .react-datepicker__quarter-text--in-selecting-range,
33 | .react-datepicker__quarter-text--selected {
34 | background-color: #63508b;
35 | }
36 |
37 | .react-datepicker__day--keyboard-selected,
38 | .react-datepicker__month-text--keyboard-selected,
39 | .react-datepicker__quarter-text--keyboard-selected {
40 | background-color: #63508b;
41 | }
42 |
--------------------------------------------------------------------------------
/lib_core/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'aio-pika',
10 | 'elasticsearch~=7.0',
11 | 'lazo-index-service==0.7.0',
12 | 's3fs',
13 | 'gcsfs',
14 | 'advocate>=1.0,<2',
15 | 'prometheus_client',
16 | 'prometheus-async',
17 | 'sentry-sdk',
18 | ]
19 | setup(name='datamart-core',
20 | version='0.0',
21 | packages=['datamart_core'],
22 | install_requires=req,
23 | description="Core library for Auctus services",
24 | author="Remi Rampin",
25 | author_email='remi.rampin@nyu.edu',
26 | maintainer="Remi Rampin",
27 | maintainer_email='remi.rampin@nyu.edu',
28 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
29 | project_urls={
30 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
31 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
32 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
33 | },
34 | long_description="Core library for Auctus services",
35 | license='Apache-2.0',
36 | keywords=['auctus', 'datamart'],
37 | classifiers=[
38 | 'Development Status :: 4 - Beta',
39 | 'Intended Audience :: Science/Research',
40 | 'License :: OSI Approved :: Apache Software License',
41 | 'Operating System :: OS Independent',
42 | 'Programming Language :: Python :: 3 :: Only',
43 | 'Topic :: Scientific/Engineering :: Information Analysis'])
44 |
--------------------------------------------------------------------------------
/snapshotter/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'prometheus_client',
10 | 'datamart-core',
11 | ]
12 | setup(name='datamart-snapshotter-service',
13 | version='0.0',
14 | packages=['snapshotter'],
15 | entry_points={
16 | 'console_scripts': [
17 | 'snapshotter = snapshotter.snapshot:main']},
18 | install_requires=req,
19 | description="Snapshotter service for Auctus",
20 | author="Remi Rampin",
21 | author_email='remi.rampin@nyu.edu',
22 | maintainer="Remi Rampin",
23 | maintainer_email='remi.rampin@nyu.edu',
24 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | project_urls={
26 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
27 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
28 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
29 | },
30 | long_description="Snapshotter service for Auctus",
31 | license='Apache-2.0',
32 | keywords=['auctus', 'datamart'],
33 | classifiers=[
34 | 'Development Status :: 4 - Beta',
35 | 'Intended Audience :: Science/Research',
36 | 'License :: OSI Approved :: Apache Software License',
37 | 'Operating System :: Unix',
38 | 'Programming Language :: JavaScript',
39 | 'Programming Language :: Python :: 3 :: Only',
40 | 'Topic :: Scientific/Engineering :: Information Analysis'])
41 |
--------------------------------------------------------------------------------
/cache_cleaner/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup
3 |
4 |
5 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
6 |
7 |
8 | req = [
9 | 'prometheus_client',
10 | 'datamart-core',
11 | ]
12 | setup(name='datamart-cache-cleaner-service',
13 | version='0.0',
14 | packages=['cache_cleaner'],
15 | entry_points={
16 | 'console_scripts': [
17 | 'cache_cleaner = cache_cleaner.cache:main']},
18 | install_requires=req,
19 | description="Cache Cleaner service for Auctus",
20 | author="Remi Rampin",
21 | author_email='remi.rampin@nyu.edu',
22 | maintainer="Remi Rampin",
23 | maintainer_email='remi.rampin@nyu.edu',
24 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
25 | project_urls={
26 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
27 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
28 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
29 | },
30 | long_description="Cache Cleaner service for Auctus",
31 | license='Apache-2.0',
32 | keywords=['auctus', 'datamart'],
33 | classifiers=[
34 | 'Development Status :: 4 - Beta',
35 | 'Intended Audience :: Science/Research',
36 | 'License :: OSI Approved :: Apache Software License',
37 | 'Operating System :: Unix',
38 | 'Programming Language :: JavaScript',
39 | 'Programming Language :: Python :: 3 :: Only',
40 | 'Topic :: Scientific/Engineering :: Information Analysis'])
41 |
--------------------------------------------------------------------------------
/scripts/purge_source.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """This script deletes all the datasets in the index from a specific source.
4 | """
5 |
6 | import lazo_index_service
7 | import logging
8 | import os
9 | import sys
10 |
11 | from datamart_core.common import PrefixedElasticsearch, \
12 | delete_dataset_from_index
13 |
14 |
15 | SIZE = 10000
16 |
17 |
18 | def clear(source):
19 | es = PrefixedElasticsearch()
20 | lazo_client = lazo_index_service.LazoIndexClient(
21 | host=os.environ['LAZO_SERVER_HOST'],
22 | port=int(os.environ['LAZO_SERVER_PORT'])
23 | )
24 | hits = es.scan(
25 | index='datasets,pending',
26 | query={
27 | 'query': {
28 | 'bool': {
29 | 'should': [
30 | {
31 | 'term': {
32 | 'materialize.identifier': source,
33 | },
34 | },
35 | {
36 | 'term': {
37 | 'source': source,
38 | },
39 | },
40 | ],
41 | 'minimum_should_match': 1,
42 | },
43 | },
44 | },
45 | _source=False,
46 | size=SIZE,
47 | )
48 | for h in hits:
49 | delete_dataset_from_index(es, h['_id'], lazo_client)
50 |
51 |
52 | if __name__ == '__main__':
53 | logging.basicConfig(level=logging.INFO)
54 |
55 | clear(sys.argv[1])
56 |
--------------------------------------------------------------------------------
/docker/prometheus.yml:
--------------------------------------------------------------------------------
1 | global:
2 | scrape_interval: 15s
3 |
4 | scrape_configs:
5 | - job_name: prometheus
6 | static_configs:
7 | - targets: ["localhost:9090"]
8 | - job_name: elasticsearch
9 | scrape_interval: 30s
10 | scrape_timeout: 10s
11 | static_configs:
12 | - targets: ["elasticsearch-exporter:9114"]
13 | - job_name: rabbitmq
14 | scrape_timeout: 5s
15 | metrics_path: /metrics
16 | static_configs:
17 | - targets: ["rabbitmq:15692"]
18 | - job_name: haproxy
19 | scrape_timeout: 5s
20 | metrics_path: /metrics
21 | static_configs:
22 | - targets: ['apilb:8000']
23 | - job_name: apiserver
24 | dns_sd_configs:
25 | - names:
26 | - apiserver
27 | type: "A"
28 | port: 8000
29 | refresh_interval: 60s
30 | - job_name: coordinator
31 | static_configs:
32 | - targets: ["coordinator:8000"]
33 | - job_name: cache-cleaner
34 | dns_sd_configs:
35 | - names:
36 | - cache-cleaner
37 | type: "A"
38 | port: 8000
39 | refresh_interval: 60s
40 | - job_name: profiler
41 | dns_sd_configs:
42 | - names:
43 | - profiler
44 | type: "A"
45 | port: 8000
46 | refresh_interval: 60s
47 | - job_name: lazo
48 | dns_sd_configs:
49 | - names:
50 | - lazo
51 | type: "A"
52 | port: 8000
53 | refresh_interval: 60s
54 | - job_name: nominatim
55 | scrape_timeout: 5s
56 | metrics_path: /metrics
57 | static_configs:
58 | - targets: ["nominatim"]
59 |
--------------------------------------------------------------------------------
/lib_profiler/setup.py:
--------------------------------------------------------------------------------
1 | import io
2 | import os
3 | from setuptools import setup
4 |
5 |
6 | os.chdir(os.path.abspath(os.path.dirname(__file__)))
7 |
8 |
9 | req = [
10 | 'numpy',
11 | 'opentelemetry-api',
12 | 'pandas',
13 | 'prometheus_client',
14 | 'python-dateutil',
15 | 'scikit-learn',
16 | 'regex',
17 | 'requests',
18 | 'datamart-geo>=0.2.3,<0.4',
19 | ]
20 | with io.open('README.rst', encoding='utf-8') as fp:
21 | description = fp.read()
22 | setup(name='datamart-profiler',
23 | version='0.11',
24 | packages=['datamart_profiler'],
25 | install_requires=req,
26 | description="Data profiling library for Auctus",
27 | author="Remi Rampin",
28 | author_email='remi.rampin@nyu.edu',
29 | maintainer="Remi Rampin",
30 | maintainer_email='remi.rampin@nyu.edu',
31 | url='https://gitlab.com/ViDA-NYU/auctus/auctus',
32 | project_urls={
33 | 'Homepage': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
34 | 'Source': 'https://gitlab.com/ViDA-NYU/auctus/auctus',
35 | 'Tracker': 'https://gitlab.com/ViDA-NYU/auctus/auctus/-/issues',
36 | },
37 | long_description=description,
38 | license='Apache-2.0',
39 | keywords=['auctus', 'datamart'],
40 | classifiers=[
41 | 'Development Status :: 4 - Beta',
42 | 'Intended Audience :: Science/Research',
43 | 'License :: OSI Approved :: Apache Software License',
44 | 'Operating System :: OS Independent',
45 | 'Programming Language :: Python :: 3 :: Only',
46 | 'Topic :: Scientific/Engineering :: Information Analysis'])
47 |
--------------------------------------------------------------------------------
/scripts/migrate-source-url.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """This scripts adds the source_url for Socrata datasets.
4 | """
5 |
6 | import json
7 | import os
8 | import shutil
9 | import sys
10 |
11 |
12 | def migrate(from_folder, to_folder):
13 | assert os.listdir(from_folder)
14 | assert not os.listdir(to_folder)
15 |
16 | datasets = []
17 | lazo = []
18 | for f in os.listdir(from_folder):
19 | if f.startswith('lazo.'):
20 | lazo.append(f)
21 | else:
22 | datasets.append(f)
23 |
24 | for i, dataset in enumerate(datasets):
25 | if i % 100 == 0:
26 | print("% 5d / %5d datasets processed" % (i, len(datasets)))
27 |
28 | with open(os.path.join(from_folder, dataset)) as fp:
29 | obj = json.load(fp)
30 |
31 | if obj['materialize']['identifier'] == 'datamart.socrata':
32 | if 'source_url' not in obj:
33 | obj['source_url'] = 'https://%s/_/_/%s' % (
34 | obj['materialize']['socrata_domain'],
35 | obj['materialize']['socrata_id'],
36 | )
37 |
38 | with open(os.path.join(to_folder, dataset), 'w') as fp:
39 | json.dump(obj, fp, sort_keys=True, indent=2)
40 |
41 | print("Copying lazo data...")
42 | for i, f in enumerate(lazo):
43 | if i % 1000 == 0:
44 | print("% 5d / %5d files copied" % (i, len(lazo)))
45 | shutil.copy2(
46 | os.path.join(from_folder, f),
47 | os.path.join(to_folder, f),
48 | )
49 |
50 |
51 | if __name__ == '__main__':
52 | migrate(sys.argv[1], sys.argv[2])
53 |
--------------------------------------------------------------------------------
/scripts/migrate-point-format.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """This scripts updates the index for !166.
4 |
5 | It adds the column "point_format" information (the default one, "long,lat").
6 | """
7 |
8 | import json
9 | import os
10 | import shutil
11 | import sys
12 |
13 |
14 | def migrate(from_folder, to_folder):
15 | assert os.listdir(from_folder)
16 | assert not os.listdir(to_folder)
17 |
18 | datasets = []
19 | lazo = []
20 | for f in os.listdir(from_folder):
21 | if f.startswith('lazo.'):
22 | lazo.append(f)
23 | else:
24 | datasets.append(f)
25 |
26 | for i, dataset in enumerate(datasets):
27 | if i % 100 == 0:
28 | print("% 5d / %5d datasets processed" % (i, len(datasets)))
29 |
30 | with open(os.path.join(from_folder, dataset)) as fp:
31 | obj = json.load(fp)
32 |
33 | for column in obj['columns']:
34 | if (
35 | column['structural_type'] == 'http://schema.org/GeoCoordinates'
36 | and 'point_format' not in column
37 | ):
38 | column['point_format'] = 'long,lat'
39 |
40 | with open(os.path.join(to_folder, dataset), 'w') as fp:
41 | json.dump(obj, fp, sort_keys=True, indent=2)
42 |
43 | print("Copying lazo data...")
44 | for i, f in enumerate(lazo):
45 | if i % 1000 == 0:
46 | print("% 5d / %5d files copied" % (i, len(lazo)))
47 | shutil.copy2(
48 | os.path.join(from_folder, f),
49 | os.path.join(to_folder, f),
50 | )
51 |
52 |
53 | if __name__ == '__main__':
54 | migrate(sys.argv[1], sys.argv[2])
55 |
--------------------------------------------------------------------------------
/tests/test_common.py:
--------------------------------------------------------------------------------
1 | import unittest
2 |
3 | from datamart_core import common
4 |
5 |
6 | class TestDatasetIdEncoding(unittest.TestCase):
7 | def test_encode(self):
8 | """Test encoding a dataset ID to a file name"""
9 | self.assertEqual(
10 | common.encode_dataset_id('datamart_contrived/dataset#id;'),
11 | 'datamart__contrived_2Fdataset_23id_3B',
12 | )
13 |
14 | def test_decode(self):
15 | """Test decoding a file name to a dataset ID"""
16 | self.assertEqual(
17 | common.decode_dataset_id('datamart__contrived_2Fdataset_23id_3B'),
18 | 'datamart_contrived/dataset#id;',
19 | )
20 |
21 |
22 | class TestStripHtml(unittest.TestCase):
23 | def test_strip(self):
24 | """Strip HTML from text"""
25 | self.assertEqual(
26 | common.strip_html(
27 | "Text & tags & HTML
", 29 | ), 30 | "Text & tags & HTML", 31 | ) 32 | 33 | def test_link(self): 34 | """Keep link targets""" 35 | self.assertEqual( 36 | common.strip_html( 37 | "Some links here: " 38 | + "google.com", 39 | ), 40 | "Some links (https://google.com/) here: google.com", 41 | ) 42 | 43 | def test_unknown(self): 44 | """Unknown tags should be preserved""" 45 | self.assertEqual( 46 | common.strip_html( 47 | "Run python
34 |
61 | );
62 | };
63 |
64 | export {FunctionBin};
65 |
--------------------------------------------------------------------------------
/tests/data/spatiotemporal.py:
--------------------------------------------------------------------------------
1 | from datetime import datetime, timedelta
2 | import random
3 | import os
4 |
5 |
6 | GRID_CELL_SIZE = 0.001
7 | COLORS = ['red', 'green', 'blue', 'yellow', 'orange']
8 |
9 |
10 | def main():
11 | lat = 43.237597
12 | lon = 6.072545
13 |
14 | data_dir = os.path.dirname(__file__)
15 |
16 | with open(os.path.join(data_dir, 'spatiotemporal.csv'), 'w') as f_data:
17 | print('date,latitude,longitude,color', file=f_data)
18 | rand = random.Random(1)
19 | for t in range(20):
20 | time = datetime(2006, 6, 20)
21 | time += timedelta(minutes=t * 30)
22 | for _ in range(10):
23 | print(
24 | '%s,%.3f,%.3f,%s' % (
25 | time.isoformat(),
26 | lat + GRID_CELL_SIZE * (rand.random() * 6 - 3),
27 | lon + GRID_CELL_SIZE * (rand.random() * 6 - 3),
28 | rand.choice(COLORS),
29 | ),
30 | file=f_data,
31 | )
32 |
33 | with open(os.path.join(data_dir, 'spatiotemporal_aug.csv'), 'w') as f_data:
34 | print('date,latitude,longitude', file=f_data)
35 | for t in range(3):
36 | time = datetime(2006, 6, 20, 6)
37 | time += timedelta(hours=t)
38 | for x in range(-1, 1):
39 | for y in range(-1, 1):
40 | print(
41 | '%s,%.3f,%.3f' % (
42 | time.isoformat(),
43 | lat + GRID_CELL_SIZE * y,
44 | lon + GRID_CELL_SIZE * x,
45 | ),
46 | file=f_data,
47 | )
48 |
49 |
50 | if __name__ == '__main__':
51 | main()
52 |
--------------------------------------------------------------------------------
/frontend/src/spatial-utils.ts:
--------------------------------------------------------------------------------
1 | import MapBrowserEvent from 'ol/MapBrowserEvent';
2 | import Geometry from 'ol/geom/Geometry';
3 | import {FeatureLike} from 'ol/Feature';
4 | import {Map} from 'ol';
5 | import {Extent} from 'ol/extent';
6 |
7 | //
8 | // Following types are a temporary workaround to a bug in typings from the
9 | // OpenLayers library (package @types/ol) This may be removed after upgrading
10 | // this library to a newer version
11 | //
12 | export interface MyMapBrowserEvent extends MapBrowserEvent {
13 | pointerEvent: PointerEvent;
14 | }
15 |
16 | interface MyGeometry extends Geometry {
17 | getCoordinates(): number[][][];
18 | }
19 |
20 | export function transformCoordinates(feature: FeatureLike) {
21 | const geometry = feature.getGeometry() as MyGeometry;
22 | const transformedGeometry = geometry
23 | .clone()
24 | .transform('EPSG:3857', 'EPSG:4326') as MyGeometry;
25 | const coordinates = transformedGeometry.getCoordinates()[0];
26 | return {
27 | topLeftLat: coordinates[0][1],
28 | topLeftLon: coordinates[0][0],
29 | bottomRightLat: coordinates[2][1],
30 | bottomRightLon: coordinates[2][0],
31 | };
32 | }
33 |
34 | export function centralizeMapToExtent(map: Map, extent: Extent) {
35 | map.getView().fit(extent);
36 | map.updateSize();
37 | }
38 |
39 | export function centralizeMapToFeature(map: Map, feature: FeatureLike) {
40 | const extent = feature.getGeometry()?.getExtent();
41 | if (extent) {
42 | centralizeMapToExtent(map, extent);
43 | }
44 | }
45 |
46 | export function wrapLongitude(x: number) {
47 | if (-180 <= x && x <= 180) {
48 | return x;
49 | }
50 | console.log(`> ${x} ...`);
51 | x += 180;
52 | x = x % 360;
53 | x = (x + 360) % 360; // second pass for negative remainders
54 | x -= 180;
55 | console.log(`<< ${x}`);
56 | return x;
57 | }
58 |
--------------------------------------------------------------------------------
/frontend/src/components/visus/Loading/Spinner.tsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import styled, {keyframes} from 'styled-components';
3 |
4 | export const keyFrameInfiniteSpin = keyframes`
5 | from {transform: rotate(0deg)}
6 | to {transform: rotate(360deg)}
7 | `;
8 |
9 | export const SpinningSvg = styled.svg`
10 | animation-name: ${keyFrameInfiniteSpin};
11 | transition-property: transform;
12 | animation-iteration-count: infinite;
13 | animation-timing-function: linear;
14 | `;
15 |
16 | interface SpinnerProps {
17 | color?: string;
18 | speed?: string;
19 | gap?: number;
20 | thickness?: number;
21 | size?: string;
22 | }
23 |
24 | class Spinner extends React.PureComponent
43 |
44 | {isActive ? (
45 | 'Release!'
46 | ) : (
47 | <>
48 | {label ? (
49 | {label}
50 | ) : (
51 | <>
52 | {fn.toUpperCase()}(
53 |
60 |
13 |
14 |
15 | (
18 | (
30 | (
42 | <>
43 |
53 |
54 |
55 | );
56 | }
57 | }
58 |
59 | export {App};
60 |
--------------------------------------------------------------------------------
/scripts/README.md:
--------------------------------------------------------------------------------
1 | Scripts
2 | =======
3 |
4 | This folder contains scripts that make it easy to perform certain tasks. Those scripts are meant to be run inside the Auctus environment; some of them are prefixed with `docker_`, those will run the corresponding script inside an Auctus container (provided the images have been built using the default docker-compose names).
5 |
6 | * setup.sh: Run this once to setup your local checkout. This sets up the permissions on the volumes for docker-compose.
7 | * docker_import_snapshot.sh: This downloads a dump of Elasticsearch from https://auctus.vida-nyu.org/snapshot/ and imports it using import_all.py
8 | * docker_import_all.sh / import_all.py: This can be used to load a dump of Elasticsearch as JSON files. Useful to restore a backup
9 | * import.py: Import a single dataset from a JSON file
10 | * reprocess_all.py: This loads a dump of Elasticsearch as JSON files, but reprocesses the datasets
11 | * freshen_old_index.py: This reprocesses datasets that were profiled by old versions
12 | * docker_export_all.sh / export_all.py: This can be used to do a backup of the index. It creates a dump of Elasticsearch as JSON files
13 | * docker-save_uploads.sh: This can be used to save the datasets that have been manually uploaded into Auctus (the data itself, not the indexed JSON documents)
14 | * delete_dataset.py: Removes a single dataset from the index
15 | * list_big_datasets.py: Lists the big datasets that have been indexed (by looking for the 'size' property above 50 MB)
16 | * list_sources.py: This lists the number of datasets in the index per source (this is now shown on the index page of the coordinator as well)
17 | * docker_purge_source.sh / purge_source.py: This removes all datasets from a given source
18 | * clear_caches.py / docker_clear_caches.sh: This safely clears the caches
19 | * upload_dataset.sh: This profiles and adds a dataset to the index
20 | * report-uploads.sh: Alerts when datasets are uploaded to the system
21 | * dataset_to_sup_index.py: This creates the supplementary column indices after 5507ab47
22 |
--------------------------------------------------------------------------------
/docker/grafana.dockerfile:
--------------------------------------------------------------------------------
1 | ARG GRAFANA_VERSION="latest"
2 |
3 | FROM python:3.8 AS tini
4 |
5 | ENV TINI_VERSION v0.18.0
6 | RUN curl -Lo /tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini && \
7 | chmod +x /tini
8 |
9 | FROM grafana/grafana:${GRAFANA_VERSION}
10 |
11 | USER root
12 |
13 | ARG GF_INSTALL_IMAGE_RENDERER_PLUGIN="false"
14 |
15 | ENV GF_PATHS_PLUGINS="/var/lib/grafana-plugins"
16 |
17 | RUN mkdir -p "$GF_PATHS_PLUGINS" && \
18 | chown -R grafana:root "$GF_PATHS_PLUGINS"
19 |
20 | RUN if [ $GF_INSTALL_IMAGE_RENDERER_PLUGIN != "false" ]; then \
21 | echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
22 | echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories && \
23 | echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
24 | apk --no-cache upgrade && \
25 | apk add --no-cache udev ttf-opensans chromium && \
26 | rm -rf /tmp/* && \
27 | rm -rf /usr/share/grafana/tools/phantomjs; \
28 | fi
29 |
30 | USER grafana
31 |
32 | ENV GF_RENDERER_PLUGIN_CHROME_BIN="/usr/bin/chromium-browser"
33 | ENV GF_PLUGIN_RENDERING_CHROME_BIN="/usr/bin/chromium-browser"
34 |
35 | RUN if [ $GF_INSTALL_IMAGE_RENDERER_PLUGIN != "false" ]; then \
36 | grafana-cli \
37 | --pluginsDir "$GF_PATHS_PLUGINS" \
38 | --pluginUrl https://github.com/grafana/grafana-image-renderer/releases/download/$GF_INSTALL_IMAGE_RENDERER_PLUGIN/plugin-linux-x64-glibc-no-chromium.zip \
39 | plugins install grafana-image-renderer; \
40 | fi
41 |
42 | ARG GF_INSTALL_PLUGINS=""
43 |
44 | RUN if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then \
45 | OLDIFS=$IFS; \
46 | IFS=','; \
47 | for plugin in ${GF_INSTALL_PLUGINS}; do \
48 | IFS=$OLDIFS; \
49 | grafana-cli --pluginsDir "$GF_PATHS_PLUGINS" plugins install ${plugin}; \
50 | done; \
51 | fi
52 |
53 | # Use tini so Chrome processes get reaped
54 | # https://github.com/grafana/grafana-image-renderer/issues/179
55 | COPY --from=tini /tini /tini
56 | ENTRYPOINT ["/tini", "--", "/run.sh"]
57 |
--------------------------------------------------------------------------------
/scripts/reprocess_all.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | """This script imports an exported index, but reprocesses the datasets.
4 |
5 | The name, description, and date are loaded from the old JSON, but the rest will
6 | be reprofiled instead.
7 | """
8 |
9 | import aio_pika
10 | import asyncio
11 | import json
12 | import logging
13 | import os
14 | import sys
15 |
16 | from datamart_core.common import json2msg, decode_dataset_id
17 |
18 |
19 | async def import_all(folder):
20 | amqp_conn = await aio_pika.connect_robust(
21 | host=os.environ['AMQP_HOST'],
22 | port=int(os.environ['AMQP_PORT']),
23 | login=os.environ['AMQP_USER'],
24 | password=os.environ['AMQP_PASSWORD'],
25 | )
26 | amqp_chan = await amqp_conn.channel()
27 | amqp_profile_exchange = await amqp_chan.declare_exchange(
28 | 'profile',
29 | aio_pika.ExchangeType.FANOUT,
30 | )
31 |
32 | for name in os.listdir(folder):
33 | if not name.startswith('lazo.'):
34 | dataset_id = decode_dataset_id(name)
35 | path = os.path.join(folder, name)
36 | with open(path, 'r') as fp:
37 | obj = json.load(fp)
38 | metadata = dict(name=obj['name'],
39 | materialize=obj['materialize'],
40 | source=obj.get('source', 'unknown'))
41 | if obj.get('description'):
42 | metadata['description'] = obj['description']
43 | if obj.get('date'):
44 | metadata['date'] = obj['date']
45 | if obj.get('manual_annotations'):
46 | metadata['manual_annotations'] = obj['manual_annotations']
47 | await amqp_profile_exchange.publish(
48 | json2msg(dict(id=dataset_id, metadata=metadata)),
49 | '',
50 | )
51 | print('.', end='', flush=True)
52 |
53 |
54 | if __name__ == '__main__':
55 | logging.basicConfig(level=logging.INFO)
56 |
57 | loop = asyncio.get_event_loop()
58 | loop.run_until_complete(loop.create_task(
59 | import_all(sys.argv[1])
60 | ))
61 |
--------------------------------------------------------------------------------
/frontend/src/components/MainMenu/MainMenu.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import * as Icon from 'react-feather';
3 | import './MainMenu.css';
4 | import {DropdownMenu} from '../ui/DropdownMenu/DropdownMenu';
5 | import {Link as RouterLink} from 'react-router-dom';
6 |
7 | function Link(props: {path: string; label: string; icon: Icon.Icon}) {
8 | const content = (
9 |
10 |
19 |
25 | )}
26 | />
27 |
31 |
37 | )}
38 | />
39 |
15 | {props.path.startsWith('http:') || props.path.startsWith('https:') ? (
16 | {content}
17 | ) : (
18 | {content}
19 | )}
20 |
21 | );
22 | }
23 |
24 | class MainMenu extends React.PureComponent {
25 | render() {
26 | return (
27 |
31 |
56 |
57 | )}
58 |
32 |
33 |
36 | {active && (
37 |
38 |
43 |
48 |
53 |
54 | )}
55 | 10 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 11 | 12 | The page will reload if you make edits.
13 | You will also see any lint errors in the console. 14 | 15 | ### `npm test` 16 | 17 | Launches the test runner in the interactive watch mode.
18 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 19 | 20 | ### `npm run build` 21 | 22 | Builds the app for production to the `build` folder.
23 | It correctly bundles React in production mode and optimizes the build for the best performance. 24 | 25 | The build is minified and the filenames include the hashes.
26 | Your app is ready to be deployed! 27 | 28 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 29 | 30 | ### `npm run eject` 31 | 32 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!** 33 | 34 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 35 | 36 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. 37 | 38 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. 39 | 40 | ## Learn More 41 | 42 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 43 | 44 | To learn React, check out the [React documentation](https://reactjs.org/). 45 | -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 16 | 17 | 18 | 19 | 23 | 24 | 25 | 34 |
35 | {% block contents %}{% endblock %}
36 |
37 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
--------------------------------------------------------------------------------
/lib_materialize/datamart_materialize/utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | import tempfile
3 |
4 |
5 | class SimpleConverterProxy(object):
6 | def __init__(self, writer, transform, name, temp_file, fp):
7 | self._writer = writer
8 | self._transform = transform
9 | self._name = name
10 | self._temp_file = temp_file
11 | self._fp = fp
12 |
13 | def close(self):
14 | self._fp.close()
15 | self._convert()
16 |
17 | def _convert(self):
18 | # Read back the file we wrote, and transform it to the final file
19 | with self._writer.open_file('w', self._name) as dst:
20 | self._transform(self._temp_file, dst)
21 |
22 | # Those methods forward to the actual file object
23 |
24 | def write(self, buffer):
25 | return self._fp.write(buffer)
26 |
27 | def flush(self):
28 | self._fp.flush()
29 |
30 | def __enter__(self):
31 | self._fp.__enter__()
32 | return self
33 |
34 | def __exit__(self, exc, value, tb):
35 | self._fp.__exit__(exc, value, tb)
36 | if exc is None:
37 | self._convert()
38 |
39 |
40 | class SimpleConverter(object):
41 | """Base class for converters simply transforming files through a function.
42 | """
43 | def __init__(self, writer):
44 | self.writer = writer
45 | self.dir = tempfile.TemporaryDirectory(prefix='datamart_excel_')
46 |
47 | def set_metadata(self, dataset_id, metadata):
48 | self.writer.set_metadata(dataset_id, metadata)
49 |
50 | def open_file(self, mode='wb', name=None):
51 | temp_file = os.path.join(self.dir.name, 'file.xls')
52 |
53 | # Return a proxy that will write to the destination when closed
54 | if mode == 'wb':
55 | fp = open(temp_file, mode)
56 | elif mode == 'w':
57 | fp = open(temp_file, mode, encoding='utf-8', newline='')
58 | else:
59 | raise ValueError("Invalid write mode %r" % mode)
60 | return SimpleConverterProxy(
61 | self.writer, self.transform,
62 | name,
63 | temp_file, fp,
64 | )
65 |
66 | def finish(self):
67 | self.dir.cleanup()
68 | self.dir = None
69 |
70 | @staticmethod
71 | def transform(source_filename, dest_fileobj):
72 | raise NotImplementedError
73 |
--------------------------------------------------------------------------------
/frontend/src/components/visus/PersistentComponent/PersistentComponent.tsx:
--------------------------------------------------------------------------------
1 | import {PureComponent} from 'react';
2 | import {shallowEqual} from '../../../utils';
3 |
4 | const cache = new Map
12 | extends React.Component
18 |
27 | );
28 | }
29 | }
30 |
31 | interface CardShadowProps {
32 | className?: string;
33 | height?: string;
34 | }
35 |
36 | class CardShadow extends React.PureComponent
19 | {this.props.title ? (
20 |
26 | {this.props.title}
21 | ) : (
22 | ''
23 | )}
24 | {this.props.children}
25 |
50 |
52 | );
53 | }
54 | }
55 |
56 | interface CardAttrFieldProps {
57 | textAlign?: string;
58 | width?: string;
59 | fontWeight?: string;
60 | padding?: string;
61 | }
62 |
63 | const CardAttrField = styled.div{this.props.children}
51 |