├── .asf.yaml ├── .gitignore ├── .readthedocs.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README ├── README.md ├── analysis ├── .gitignore ├── README.md ├── __init__.py ├── static │ └── index.html ├── test.py ├── tests │ ├── README.md │ ├── __init__.py │ ├── algorithms │ │ ├── StandardDeviationSearch_test.py │ │ ├── longitudelatitudemap_test.py │ │ ├── test_base_doms_handler.py │ │ └── test_subsetter.py │ ├── algorithms_spark │ │ ├── __init__.py │ │ ├── test_matchup.py │ │ └── test_timeseriesspark.py │ ├── config │ │ └── algorithms.ini │ ├── conftest.py │ ├── data │ │ └── edge_response.json │ ├── integration │ │ └── algorithms_spark │ │ │ └── test_matchup.py │ ├── redirect │ │ ├── __init__.py │ │ ├── collections-config.yaml │ │ └── test_RemoteSDAPCache.py │ └── webapp_test.py └── webservice │ ├── Filtering.py │ ├── GenerateImageMRF.py │ ├── LayerConfig.py │ ├── NexusHandler.py │ ├── NexusLivyHandler.py │ ├── WorkflowDriver.py │ ├── __init__.py │ ├── algorithms │ ├── Capabilities.py │ ├── ColorBarHandler.py │ ├── CorrelationMap.py │ ├── DailyDifferenceAverage.py │ ├── DataInBoundsSearch.py │ ├── DataSeriesList.py │ ├── DelayTest.py │ ├── ErrorTosserTest.py │ ├── Heartbeat.py │ ├── HofMoeller.py │ ├── LongitudeLatitudeMap.py │ ├── MapFetchHandler.py │ ├── NexusCalcHandler.py │ ├── StandardDeviationSearch.py │ ├── TestInitializer.py │ ├── TileSearch.py │ ├── TimeAvgMap.py │ ├── TimeSeries.py │ ├── TimeSeriesSolr.py │ ├── Tomogram.py │ ├── Tomogram3D.py │ ├── __init__.py │ ├── colortables.py │ └── doms │ │ ├── BaseDomsHandler.py │ │ ├── DatasetListQuery.py │ │ ├── DomsInitialization.py │ │ ├── ExecutionCancel.py │ │ ├── ExecutionStatus.py │ │ ├── MatchupQuery.py │ │ ├── MetadataQuery.py │ │ ├── ResultsRetrieval.py │ │ ├── ResultsStorage.py │ │ ├── StacCatalog.py │ │ ├── StatsQuery.py │ │ ├── ValuesQuery.py │ │ ├── __init__.py │ │ ├── config.py │ │ ├── datafetch.py │ │ ├── domsconfig.ini.default │ │ ├── fetchedgeimpl.py │ │ ├── geo.py │ │ ├── histogramplot.py │ │ ├── insitu.py │ │ ├── insitusubset.py │ │ ├── scatterplot.py │ │ ├── subsetter.py │ │ ├── values.py │ │ └── workerthread.py │ ├── algorithms_spark │ ├── ClimMapSpark.py │ ├── CorrMapSpark.py │ ├── DailyDifferenceAverageSpark.py │ ├── HofMoellerSpark.py │ ├── Matchup.py │ ├── MatchupDoms.py │ ├── MaximaMinimaSpark.py │ ├── NexusCalcSparkHandler.py │ ├── NexusCalcSparkTornadoHandler.py │ ├── TimeAvgMapSpark.py │ ├── TimeSeriesSpark.py │ ├── VarianceSpark.py │ ├── __init__.py │ └── utils.py │ ├── apidocs │ ├── __init__.py │ ├── dataset-populate.js │ ├── index.html │ └── openapi.yml │ ├── config │ ├── algorithms.ini │ ├── scheduler.xml │ └── web.ini │ ├── management │ ├── Datasets.py │ └── __init__.py │ ├── matserver.py │ ├── metrics │ ├── MetricsField.py │ ├── MetricsRecord.py │ └── __init__.py │ ├── nexus_tornado │ ├── __init__.py │ ├── app_builders │ │ ├── HandlerArgsBuilder.py │ │ ├── NexusAppBuilder.py │ │ ├── RedirectAppBuilder.py │ │ ├── SparkContextBuilder.py │ │ └── __init__.py │ └── request │ │ ├── __init__.py │ │ ├── handlers │ │ ├── NexusRequestHandler.py │ │ └── __init__.py │ │ └── renderers │ │ ├── NexusCSVRenderer.py │ │ ├── NexusGIFRenderer.py │ │ ├── NexusJSONRenderer.py │ │ ├── NexusNETCDFRenderer.py │ │ ├── NexusPNGRenderer.py │ │ ├── NexusRendererFactory.py │ │ ├── NexusZIPRenderer.py │ │ └── __init__.py │ ├── plotting.py │ ├── redirect │ ├── RedirectHandler.py │ ├── RemoteCollectionMatcher.py │ ├── RemoteSDAPCache.py │ └── __init__.py │ ├── resources │ ├── transparent.png │ └── wkt.txt │ ├── webapp.py │ ├── webapp_livy.py │ └── webmodel │ ├── CustomEncoder.py │ ├── Exceptions.py │ ├── NexusExecutionResults.py │ ├── NexusRequestObject.py │ ├── NexusRequestObjectTornadoFree.py │ ├── NexusResults.py │ ├── RequestParameters.py │ ├── StandardNexusErrors.py │ ├── StatsComputeOptions.py │ ├── __init__.py │ └── cached.py ├── data-access ├── .gitignore ├── README.md ├── config │ └── schemas │ │ ├── cassandra │ │ └── nexustiles.cql │ │ ├── solr-7.1.0 │ │ └── nexustiles │ │ │ └── conf │ │ │ ├── lang │ │ │ ├── contractions_ca.txt │ │ │ ├── contractions_fr.txt │ │ │ ├── contractions_ga.txt │ │ │ ├── contractions_it.txt │ │ │ ├── hyphenations_ga.txt │ │ │ ├── stemdict_nl.txt │ │ │ ├── stoptags_ja.txt │ │ │ ├── stopwords_ar.txt │ │ │ ├── stopwords_bg.txt │ │ │ ├── stopwords_ca.txt │ │ │ ├── stopwords_cz.txt │ │ │ ├── stopwords_da.txt │ │ │ ├── stopwords_de.txt │ │ │ ├── stopwords_el.txt │ │ │ ├── stopwords_en.txt │ │ │ ├── stopwords_es.txt │ │ │ ├── stopwords_eu.txt │ │ │ ├── stopwords_fa.txt │ │ │ ├── stopwords_fi.txt │ │ │ ├── stopwords_fr.txt │ │ │ ├── stopwords_ga.txt │ │ │ ├── stopwords_gl.txt │ │ │ ├── stopwords_hi.txt │ │ │ ├── stopwords_hu.txt │ │ │ ├── stopwords_hy.txt │ │ │ ├── stopwords_id.txt │ │ │ ├── stopwords_it.txt │ │ │ ├── stopwords_ja.txt │ │ │ ├── stopwords_lv.txt │ │ │ ├── stopwords_nl.txt │ │ │ ├── stopwords_no.txt │ │ │ ├── stopwords_pt.txt │ │ │ ├── stopwords_ro.txt │ │ │ ├── stopwords_ru.txt │ │ │ ├── stopwords_sv.txt │ │ │ ├── stopwords_th.txt │ │ │ ├── stopwords_tr.txt │ │ │ └── userdict_ja.txt │ │ │ ├── managed-schema │ │ │ ├── params.json │ │ │ ├── protwords.txt │ │ │ ├── solrconfig.xml │ │ │ ├── stopwords.txt │ │ │ └── synonyms.txt │ │ └── solr │ │ ├── dataset │ │ ├── conf │ │ │ ├── _rest_managed.json │ │ │ ├── currency.xml │ │ │ ├── lang │ │ │ │ └── stopwords_en.txt │ │ │ ├── protwords.txt │ │ │ ├── schema.xml │ │ │ ├── solrconfig.xml │ │ │ ├── stopwords.txt │ │ │ └── synonyms.txt │ │ └── core.properties │ │ └── nexustiles │ │ ├── conf │ │ ├── currency.xml │ │ ├── elevate.xml │ │ ├── lang │ │ │ ├── contractions_ca.txt │ │ │ ├── contractions_fr.txt │ │ │ ├── contractions_ga.txt │ │ │ ├── contractions_it.txt │ │ │ ├── hyphenations_ga.txt │ │ │ ├── stemdict_nl.txt │ │ │ ├── stoptags_ja.txt │ │ │ ├── stopwords_ar.txt │ │ │ ├── stopwords_bg.txt │ │ │ ├── stopwords_ca.txt │ │ │ ├── stopwords_cz.txt │ │ │ ├── stopwords_da.txt │ │ │ ├── stopwords_de.txt │ │ │ ├── stopwords_el.txt │ │ │ ├── stopwords_en.txt │ │ │ ├── stopwords_es.txt │ │ │ ├── stopwords_eu.txt │ │ │ ├── stopwords_fa.txt │ │ │ ├── stopwords_fi.txt │ │ │ ├── stopwords_fr.txt │ │ │ ├── stopwords_ga.txt │ │ │ ├── stopwords_gl.txt │ │ │ ├── stopwords_hi.txt │ │ │ ├── stopwords_hu.txt │ │ │ ├── stopwords_hy.txt │ │ │ ├── stopwords_id.txt │ │ │ ├── stopwords_it.txt │ │ │ ├── stopwords_ja.txt │ │ │ ├── stopwords_lv.txt │ │ │ ├── stopwords_nl.txt │ │ │ ├── stopwords_no.txt │ │ │ ├── stopwords_pt.txt │ │ │ ├── stopwords_ro.txt │ │ │ ├── stopwords_ru.txt │ │ │ ├── stopwords_sv.txt │ │ │ ├── stopwords_th.txt │ │ │ ├── stopwords_tr.txt │ │ │ └── userdict_ja.txt │ │ ├── managed-schema │ │ ├── params.json │ │ ├── protwords.txt │ │ ├── solrconfig.xml │ │ ├── stopwords.txt │ │ └── synonyms.txt │ │ └── core.properties ├── nexustiles │ ├── AbstractTileService.py │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── nexusproto │ │ │ ├── __init__.py │ │ │ ├── backend.py │ │ │ ├── config │ │ │ │ └── datastores.ini.default │ │ │ └── dao │ │ │ │ ├── CassandraProxy.py │ │ │ │ ├── DynamoProxy.py │ │ │ │ ├── ElasticsearchProxy.py │ │ │ │ ├── S3Proxy.py │ │ │ │ ├── SolrProxy.py │ │ │ │ └── __init__.py │ │ └── zarr │ │ │ ├── __init__.py │ │ │ └── backend.py │ ├── config │ │ └── datasets.ini.default │ ├── exception.py │ ├── model │ │ ├── __init__.py │ │ └── nexusmodel.py │ └── nexustiles.py └── tests │ ├── __init__.py │ ├── config │ └── datastores.ini │ ├── nexusmodel_test.py │ ├── nexustiles_test.py │ ├── sizefromcass.py │ ├── solr_update.py │ ├── test_elasticsearch_proxy.py │ └── test_solr_proxy.py ├── docker ├── jupyter │ ├── Dockerfile │ ├── Time Series Example.ipynb │ └── requirements.txt ├── nexus-webapp │ ├── Dockerfile │ ├── Readme.md │ ├── entrypoint.sh │ ├── install_nexusproto.sh │ └── install_python.sh └── solr │ ├── Dockerfile │ ├── Readme.rst │ └── cloud-init │ ├── Dockerfile │ ├── Readme.rst │ └── create-collection.py ├── docs ├── .gitignore ├── Makefile ├── build.rst ├── collections.rst ├── conf.py ├── dockerimages.rst ├── granule-download.sh ├── images │ ├── Jupyter_Home.png │ ├── Jupyter_Quickstart.png │ └── nexus-summary.png ├── index.rst ├── intro.rst ├── make.bat ├── quickstart.rst ├── release.rst ├── requirements.txt └── test.rst ├── helm ├── .gitignore ├── .helmignore ├── Chart.yaml ├── README.md ├── requirements.yaml ├── templates │ ├── _helpers.tpl │ ├── collection-manager.yml │ ├── collections-config-gitcfg.yml │ ├── config-operator-rbac.yml │ ├── config-operator.yml │ ├── domspurge.yml │ ├── gitbasedconfig-crd.yml │ ├── granule-ingester.yml │ ├── history-pvc.yml │ ├── ingress.yml │ ├── init-cassandra-configmap.yml │ ├── onearth.yml │ ├── root-webpage.yml │ ├── solr-create-collection.yml │ ├── spark-serviceaccount.yml │ └── webapp.yml └── values.yaml ├── integrations ├── esri │ ├── README.md │ ├── nexus_toolbox_embedded.tbx │ └── zipped_toolbox │ │ ├── daily_difference_average.py │ │ ├── lat_hof_moeller.py │ │ ├── lon_hof_moeller.py │ │ ├── matchup.py │ │ ├── nexus_toolbox.tbx │ │ ├── time_average_map.py │ │ └── time_series_script.py └── python-client │ ├── .gitignore │ ├── docs │ └── nexuscli │ │ ├── index.html │ │ ├── nexuscli.m.html │ │ ├── nexuscli_ow.m.html │ │ └── test │ │ ├── index.html │ │ └── nexuscli_test.m.html │ ├── nexuscli │ ├── __init__.py │ ├── nexuscli.py │ ├── nexuscli_ow.py │ └── test │ │ ├── __init__.py │ │ └── nexuscli_test.py │ └── setup.py ├── nexus-ingest └── Readme.md ├── poetry.lock ├── pyproject.toml ├── tests ├── .gitignore ├── README.md ├── cdms_reader.py ├── conftest.py ├── download_data.sh ├── requirements.txt ├── test_collections.yaml └── test_sdap.py └── tools ├── build ├── README.md ├── build.py └── requirements.txt ├── cdms ├── README.md ├── cdms_reader.py └── requirements.txt ├── deletebyquery ├── .gitignore ├── README.md ├── deletebyquery.py └── requirements.txt ├── doms-data-tools ├── update_doms_data_pk.py └── update_values_type.py ├── domspurge ├── Dockerfile ├── README.md ├── entrypoint.sh ├── purge.py └── requirements.txt └── rmqmonitor ├── README.md └── monitor.py /.asf.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | github: 18 | autolink_jira: 19 | - SDAP 20 | protected_branches: 21 | master: 22 | required_status_checks: 23 | strict: true # Require branches be up to date 24 | required_pull_request_reviews: 25 | dismiss_stale_reviews: true 26 | require_code_owner_reviews: true 27 | required_approving_review_count: 1 28 | develop: 29 | required_status_checks: 30 | strict: true # Require branches be up to date 31 | required_pull_request_reviews: 32 | dismiss_stale_reviews: true 33 | require_code_owner_reviews: true 34 | required_approving_review_count: 1 35 | del_branch_on_merge: true 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pytest_cache 2 | *.vscode 3 | *.code-workspace 4 | *.idea 5 | *.DS_Store 6 | analysis/webservice/algorithms/doms/domsconfig.ini 7 | data-access/nexustiles/backends/nexusproto/config/datastores.ini 8 | data-access/nexustiles/config/datasets.ini 9 | venv/ 10 | 11 | dist/ -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | version: 2 17 | 18 | build: 19 | os: ubuntu-22.04 20 | tools: 21 | python: "3.7" 22 | 23 | python: 24 | install: 25 | - requirements: docs/requirements.txt 26 | 27 | sphinx: 28 | configuration: docs/conf.py 29 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache SDAP 2 | Copyright 2017 - 2025 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Apache SDAP release 1.4.0 2 | 3 | This is a source distribution of Apache SDAP - NEXUS. 4 | 5 | Changes since the previous release are described in the 6 | CHANGELOG.md file. 7 | 8 | The LICENSE and NOTICE files contain license information. 9 | 10 | You can find instructions how to build the release at 11 | https://sdap-nexus.readthedocs.io/en/latest/build.html 12 | 13 | Further information about Apache SDAP is available at its web site, 14 | https://sdap.apache.org. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NEXUS 2 | 3 | The next generation cloud-based science data service platform. More information can be found here http://incubator-sdap-nexus.readthedocs.io/en/latest/index.html 4 | 5 | 6 | ## Building the Docs 7 | 8 | Ensure sphinx, sphinx-autobuild, and recommonmark are installed. We use the recommonmark module for parsing Markdown files. 9 | 10 | pip install sphinx sphinx-autobuild recommonmark 11 | 12 | Run sphinx-autobuild to view the docs locally. 13 | 14 | cd docs 15 | sphinx-autobuild . _build/html 16 | -------------------------------------------------------------------------------- /analysis/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /analysis/static/index.html: -------------------------------------------------------------------------------- 1 | 17 | 18 | 19 | 20 | It works! -------------------------------------------------------------------------------- /analysis/test.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import numpy as np 18 | 19 | A = np.arange(12).reshape(3, 4) 20 | b = np.arange(3).reshape(1, 3) 21 | 22 | # np.linalg.lstsq(A,b) 23 | # This gives "LinAlgError: Incompatible dimensions" exception 24 | 25 | print(np.linalg.lstsq(A, b.T)) 26 | -------------------------------------------------------------------------------- /analysis/tests/README.md: -------------------------------------------------------------------------------- 1 | # Apache SDAP Testing 2 | 3 | ## Unit Tests 4 | 5 | Unit tests don't contain any externally running dependencies (like Solr 6 | or Cassandra). Unit tests, unlike integration tests, do not contain an 7 | 'integration' pytest marker. To run all unit tests and skip integration 8 | tests, run the following command: 9 | 10 | ```shell script 11 | pytest -m "not integration" analysis/tests/ 12 | ``` 13 | 14 | ## Integration Tests 15 | 16 | To run integration tests, Cassandra and Solr must be running locally. 17 | 18 | Integration tests have been marked with 'integration' pytest markers. 19 | In order to run only integration tests, run the following command: 20 | 21 | ```shell script 22 | pytest -m "integration" analysis/tests/ 23 | ``` 24 | -------------------------------------------------------------------------------- /analysis/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | -------------------------------------------------------------------------------- /analysis/tests/algorithms_spark/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | -------------------------------------------------------------------------------- /analysis/tests/config/algorithms.ini: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [multiprocessing] 17 | maxprocesses=8 18 | 19 | [spark] 20 | maxconcurrentjobs=10 -------------------------------------------------------------------------------- /analysis/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import logging 17 | import pytest 18 | import os 19 | 20 | from pyspark import HiveContext 21 | from pyspark import SparkConf 22 | from pyspark import SparkContext 23 | from pyspark.streaming import StreamingContext 24 | 25 | def quiet_py4j(): 26 | logger = logging.getLogger('py4j') 27 | logger.setLevel(logging.WARN) 28 | 29 | @pytest.fixture(scope="session") 30 | def spark_context(request): 31 | conf = (SparkConf().setMaster("local[2]").setAppName("pytest-pyspark-local-testing")) 32 | request.addfinalizer(lambda: sc.stop()) 33 | 34 | sc = SparkContext(conf=conf) 35 | quiet_py4j() 36 | return sc 37 | 38 | @pytest.fixture(scope="session") 39 | def hive_context(spark_context): 40 | return HiveContext(spark_context) 41 | 42 | @pytest.fixture(scope="session") 43 | def streaming_context(spark_context): 44 | return StreamingContext(spark_context, 1) 45 | 46 | @pytest.fixture(scope="session") 47 | def setup_pyspark_env(): 48 | os.environ['PYSPARK_DRIVER_PYTHON'] = '/usr/local/anaconda3/envs/nexus-messages3/bin/python' 49 | os.environ['PYSPARK_PYTHON'] = '/usr/local/anaconda3/envs/nexus-messages3/bin/python' -------------------------------------------------------------------------------- /analysis/tests/data/edge_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "total": 3, 3 | "results": [ 4 | { 5 | "point": "Point(5.0 15.0)", 6 | "longitude": 5.0, 7 | "latitude": 15.0, 8 | "time": 1595954285, 9 | "sea_water_temperature": 10.0, 10 | "id": 1234, 11 | "platform": { 12 | "code": "30" 13 | } 14 | }, 15 | { 16 | "point": "Point(10.0 10.0)", 17 | "longitude": 10.0, 18 | "latitude": 10.0, 19 | "time": 1595954286, 20 | "sea_water_temperature": 20.0, 21 | "id": 1235, 22 | "platform": { 23 | "code": "30" 24 | } 25 | }, 26 | { 27 | "point": "Point(18.0 3.0)", 28 | "longitude": 18.0, 29 | "latitude": 3.0, 30 | "time": 1595954287, 31 | "sea_water_temperature": 30.0, 32 | "id": 1236, 33 | "platform": { 34 | "code": "30" 35 | } 36 | } 37 | ] 38 | } 39 | 40 | -------------------------------------------------------------------------------- /analysis/tests/redirect/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /analysis/tests/redirect/collections-config.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | collections: 17 | - id: gmu-pm25 18 | path: https://aq-sdap.stcenter.net/nexus/ 19 | remote-id: PM25 20 | - id: ECCO_v4_r4_EVELMASS_latlon 21 | path: /data/datasets/ecco-distribution-archive/nexus-ingest/EVELMASSv4r4/*.nc 22 | priority: 6 23 | forward-processing-priority: 7 24 | projection: Grid 25 | dimensionNames: 26 | latitude: latitude 27 | longitude: longitude 28 | depth: Z 29 | time: time 30 | variable: EVELMASS 31 | slices: 32 | time: 1 33 | i: 30 34 | j: 30 35 | k: 1 -------------------------------------------------------------------------------- /analysis/webservice/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/Capabilities.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import json 18 | 19 | from webservice.NexusHandler import nexus_handler, AVAILABLE_HANDLERS 20 | from webservice.algorithms.NexusCalcHandler import NexusCalcHandler 21 | from webservice.webmodel import NexusResults 22 | 23 | 24 | @nexus_handler 25 | class CapabilitiesListCalcHandlerImpl(NexusCalcHandler): 26 | name = "Capabilities" 27 | path = "/capabilities" 28 | description = "Lists the current capabilities of this Nexus system" 29 | params = {} 30 | singleton = True 31 | 32 | def calc(self, computeOptions, **args): 33 | capabilities = [] 34 | 35 | for capability in AVAILABLE_HANDLERS: 36 | capabilityDef = { 37 | "name": capability.name, 38 | "path": capability.path, 39 | "description": capability.description, 40 | "parameters": capability.params 41 | } 42 | capabilities.append(capabilityDef) 43 | 44 | return CapabilitiesResults(capabilities) 45 | 46 | 47 | class CapabilitiesResults(NexusResults): 48 | def __init__(self, capabilities): 49 | NexusResults.__init__(self) 50 | self.__capabilities = capabilities 51 | 52 | def toJson(self): 53 | return json.dumps(self.__capabilities, indent=4) 54 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/DelayTest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import time 18 | 19 | from webservice.algorithms.NexusCalcHandler import NexusCalcHandler 20 | from webservice.NexusHandler import nexus_handler 21 | 22 | 23 | @nexus_handler 24 | class DelayCalcHandlerImpl(NexusCalcHandler): 25 | name = "Delay" 26 | path = "/delay" 27 | description = "Waits a little while" 28 | params = {} 29 | singleton = True 30 | 31 | def calc(self, computeOptions, **args): 32 | time.sleep(10) 33 | 34 | class SimpleResult(object): 35 | def toJson(self): 36 | return "" 37 | 38 | return SimpleResult() 39 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/ErrorTosserTest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from webservice.NexusHandler import nexus_handler 18 | from webservice.algorithms.NexusCalcHandler import NexusCalcHandler 19 | 20 | 21 | @nexus_handler 22 | class ErrorTosserCalcHandler(NexusCalcHandler): 23 | name = "MakeError" 24 | path = "/makeerror" 25 | description = "Causes an error" 26 | params = {} 27 | singleton = True 28 | 29 | def calc(self, computeOptions, **args): 30 | a = 100 / 0.0 31 | # raise Exception("I'm Mad!") 32 | # raise NexusProcessingException.NexusProcessingException(NexusProcessingException.StandardNexusErrors.UNKNOWN, "I'm Mad!") 33 | return {}, None, None 34 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/Heartbeat.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import json 18 | 19 | from webservice.NexusHandler import nexus_handler 20 | from webservice.algorithms.NexusCalcHandler import NexusCalcHandler 21 | 22 | 23 | @nexus_handler 24 | class HeartbeatCalcHandlerImpl(NexusCalcHandler): 25 | name = "Backend Services Status" 26 | path = "/heartbeat" 27 | description = "Returns health status of Nexus backend services" 28 | params = {} 29 | singleton = True 30 | 31 | def calc(self, computeOptions, **args): 32 | status = self._get_tile_service().heartbeat() 33 | 34 | class SimpleResult(object): 35 | def __init__(self, result): 36 | self.result = result 37 | 38 | def toJson(self): 39 | return json.dumps(self.result) 40 | 41 | return SimpleResult(status) 42 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/TestInitializer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | 18 | from webservice.NexusHandler import nexus_initializer 19 | 20 | 21 | @nexus_initializer 22 | class TestInitializer: 23 | 24 | def __init__(self): 25 | pass 26 | 27 | def init(self, config): 28 | print("*** TEST INITIALIZATION ***") 29 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from . import Capabilities 18 | from . import CorrelationMap 19 | from . import DailyDifferenceAverage 20 | from . import DataInBoundsSearch 21 | from . import DataSeriesList 22 | from . import DelayTest 23 | from . import ErrorTosserTest 24 | from . import Heartbeat 25 | from . import HofMoeller 26 | from . import LongitudeLatitudeMap 27 | from . import StandardDeviationSearch 28 | from . import TestInitializer 29 | from . import TileSearch 30 | from . import TimeAvgMap 31 | from . import TimeSeries 32 | from . import TimeSeriesSolr 33 | from . import Tomogram 34 | from . import Tomogram3D 35 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/doms/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from . import BaseDomsHandler 18 | from . import ExecutionStatus 19 | from . import ExecutionCancel 20 | from . import DatasetListQuery 21 | from . import DomsInitialization 22 | from . import MatchupQuery 23 | from . import StacCatalog 24 | from . import MetadataQuery 25 | from . import ResultsRetrieval 26 | from . import ResultsStorage 27 | from . import StatsQuery 28 | from . import ValuesQuery 29 | from . import config 30 | from . import datafetch 31 | from . import fetchedgeimpl 32 | from . import geo 33 | from . import insitusubset 34 | from . import subsetter 35 | from . import values 36 | from . import workerthread 37 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/doms/datafetch.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from . import fetchedgeimpl 17 | 18 | 19 | def getCount(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms=None): 20 | return fetchedgeimpl.getCount(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms) 21 | 22 | 23 | def __fetchSingleDataSource(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms=None): 24 | return fetchedgeimpl.fetch(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms) 25 | 26 | 27 | def __fetchMultipleDataSource(endpoints, startTime, endTime, bbox, depth_min, depth_max, platforms=None): 28 | data = [] 29 | for endpoint in endpoints: 30 | dataSingleSource = __fetchSingleDataSource(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms) 31 | data = data + dataSingleSource 32 | return data 33 | 34 | 35 | def fetchData(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms=None): 36 | if type(endpoint) == list: 37 | return __fetchMultipleDataSource(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms) 38 | else: 39 | return __fetchSingleDataSource(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms) 40 | 41 | 42 | def getValues(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms=None, placeholders=False): 43 | return fetchedgeimpl.getValues(endpoint, startTime, endTime, bbox, depth_min, depth_max, platforms, placeholders) 44 | 45 | 46 | if __name__ == "__main__": 47 | pass 48 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/doms/domsconfig.ini.default: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [cassandra] 17 | host=localhost 18 | port=9042 19 | keyspace=doms 20 | local_datacenter=datacenter1 21 | protocol_version=4 22 | dc_policy=DCAwareRoundRobinPolicy 23 | username= 24 | password= 25 | 26 | 27 | [cassandraDD] 28 | host=128.149.115.178,128.149.115.173,128.149.115.176,128.149.115.175,128.149.115.172,128.149.115.174,128.149.115.177 29 | keyspace=doms 30 | local_datacenter=B600 31 | protocol_version=3 32 | 33 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms/doms/workerthread.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import threading 17 | 18 | 19 | class WorkerThread(threading.Thread): 20 | 21 | def __init__(self, method, params): 22 | threading.Thread.__init__(self) 23 | self.method = method 24 | self.params = params 25 | self.completed = False 26 | self.results = None 27 | 28 | def run(self): 29 | self.results = self.method(*self.params) 30 | self.completed = True 31 | 32 | 33 | def __areAllComplete(threads): 34 | for thread in threads: 35 | if not thread.completed: 36 | return False 37 | 38 | return True 39 | 40 | 41 | def wait(threads, startFirst=False, poll=0.5): 42 | if startFirst: 43 | for thread in threads: 44 | thread.start() 45 | 46 | while not __areAllComplete(threads): 47 | threading._sleep(poll) 48 | 49 | 50 | def foo(param1, param2): 51 | print(param1, param2) 52 | return "c" 53 | 54 | 55 | if __name__ == "__main__": 56 | 57 | thread = WorkerThread(foo, params=("a", "b")) 58 | thread.start() 59 | while not thread.completed: 60 | threading._sleep(0.5) 61 | print(thread.results) 62 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms_spark/NexusCalcSparkTornadoHandler.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import logging 17 | from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSparkHandler 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | class NexusCalcSparkTornadoHandler(NexusCalcSparkHandler): 23 | pass 24 | -------------------------------------------------------------------------------- /analysis/webservice/algorithms_spark/utils.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from datetime import datetime 17 | import pytz 18 | import calendar 19 | 20 | 21 | def normalize_date(time_in_seconds): 22 | dt = datetime.utcfromtimestamp(time_in_seconds) 23 | normalized_dt = dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0, tzinfo=pytz.utc) 24 | return int(calendar.timegm(normalized_dt.timetuple())) 25 | -------------------------------------------------------------------------------- /analysis/webservice/apidocs/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /analysis/webservice/config/algorithms.ini: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [multiprocessing] 17 | maxprocesses=8 18 | 19 | [spark] 20 | maxconcurrentjobs=10 -------------------------------------------------------------------------------- /analysis/webservice/config/scheduler.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 1000 21 | 1 22 | 23 | 24 | 1 25 | 26 | -------------------------------------------------------------------------------- /analysis/webservice/config/web.ini: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [global] 17 | server.socket_port=8083 18 | server.socket_host = '127.0.0.1' 19 | server.max_simultaneous_requests = 10 20 | 21 | [livy] 22 | livy_port = 8998 23 | livy_host = localhost 24 | server.socket_port = 8084 25 | server.socket_host = '127.0.0.1' 26 | 27 | [static] 28 | static_enabled=true 29 | static_dir=static 30 | 31 | [modules] 32 | module_dirs=webservice.algorithms,webservice.algorithms_spark,webservice.algorithms.doms,webservice.management -------------------------------------------------------------------------------- /analysis/webservice/management/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from webservice.management.Datasets import DatasetAdd -------------------------------------------------------------------------------- /analysis/webservice/matserver.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import matplotlib.pyplot as plt 18 | import mpld3 19 | import numpy as np 20 | from mpld3 import plugins 21 | 22 | fig, ax = plt.subplots() 23 | 24 | x = np.linspace(-2, 2, 20) 25 | y = x[:, None] 26 | X = np.zeros((20, 20, 4)) 27 | 28 | X[:, :, 0] = np.exp(- (x - 1) ** 2 - (y) ** 2) 29 | X[:, :, 1] = np.exp(- (x + 0.71) ** 2 - (y - 0.71) ** 2) 30 | X[:, :, 2] = np.exp(- (x + 0.71) ** 2 - (y + 0.71) ** 2) 31 | X[:, :, 3] = np.exp(-0.25 * (x ** 2 + y ** 2)) 32 | 33 | im = ax.imshow(X, extent=(10, 20, 10, 20), 34 | origin='lower', zorder=1, interpolation='nearest') 35 | fig.colorbar(im, ax=ax) 36 | 37 | ax.set_title('An Image', size=20) 38 | 39 | plugins.connect(fig, plugins.MousePosition(fontsize=14)) 40 | 41 | mpld3.show() 42 | -------------------------------------------------------------------------------- /analysis/webservice/metrics/MetricsField.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from abc import abstractmethod 17 | 18 | 19 | class MetricsField(object): 20 | def __init__(self, key, description, initial_value): 21 | self.key = key 22 | self.description = description 23 | self._value = initial_value 24 | 25 | @abstractmethod 26 | def add(self, addend): 27 | pass 28 | 29 | def value(self): 30 | return self._value 31 | 32 | 33 | class SparkAccumulatorMetricsField(MetricsField): 34 | def __init__(self, key, description, accumulator): 35 | super(SparkAccumulatorMetricsField, self).__init__(key, description, accumulator) 36 | 37 | def add(self, addend): 38 | self._value.add(addend) 39 | 40 | def value(self): 41 | return self._value.value 42 | 43 | 44 | class NumberMetricsField(MetricsField): 45 | def __init__(self, key, description, initial_value=0): 46 | super(NumberMetricsField, self).__init__(key, description, initial_value) 47 | 48 | def add(self, addend): 49 | self._value += addend 50 | -------------------------------------------------------------------------------- /analysis/webservice/metrics/MetricsRecord.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from collections import OrderedDict 17 | import logging 18 | 19 | metrics_logger = logging.getLogger(__name__) 20 | 21 | 22 | class MetricsRecord(object): 23 | def __init__(self, fields): 24 | self._fields = OrderedDict() 25 | for field in fields: 26 | self._fields[field.key] = field 27 | 28 | def record_metrics(self, **kwargs): 29 | for field_key, addend in list(kwargs.items()): 30 | if field_key in self._fields: 31 | self._fields[field_key].add(addend) 32 | 33 | def print_metrics(self, logger=None, include_zero_values=False): 34 | if not logger: 35 | logger = metrics_logger 36 | 37 | logging_lines = [] 38 | for field in list(self._fields.values()): 39 | value = field.value() 40 | if value > 0 or include_zero_values: 41 | line = "{description}: {value}".format(description=field.description, value=field.value()) 42 | logging_lines.append(line) 43 | 44 | logger.info('\n'.join(logging_lines)) 45 | 46 | def write_metrics(self): 47 | raise NotImplementedError 48 | -------------------------------------------------------------------------------- /analysis/webservice/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .MetricsField import MetricsField, SparkAccumulatorMetricsField, NumberMetricsField 17 | from .MetricsRecord import MetricsRecord 18 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/app_builders/RedirectAppBuilder.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from webservice.redirect import RedirectHandler 17 | from webservice.redirect import RemoteCollectionMatcher 18 | import tornado 19 | 20 | 21 | class RedirectAppBuilder: 22 | def __init__(self, remote_collection_matcher: RemoteCollectionMatcher): 23 | redirected_collections = remote_collection_matcher.get_remote_collections() 24 | self.redirect_handler = (r'/(.*)', RedirectHandler, {'redirected_collections': redirected_collections}) 25 | 26 | def build(self, host=None, debug=False): 27 | return tornado.web.Application( 28 | [self.redirect_handler], 29 | default_host=host, 30 | debug=debug 31 | ) 32 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/app_builders/SparkContextBuilder.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import pkg_resources 16 | 17 | 18 | class SparkContextBuilder: 19 | def __init__(self): 20 | pass 21 | 22 | spark_context = None 23 | 24 | @classmethod 25 | def get_spark_context(cls): 26 | if cls.spark_context is None: 27 | from pyspark.sql import SparkSession 28 | 29 | scheduler_path = pkg_resources.resource_filename('webservice', "config/scheduler.xml") 30 | 31 | spark = SparkSession.builder.appName("nexus-analysis").config( 32 | "spark.scheduler.allocation.file", scheduler_path 33 | ).config( 34 | "spark.scheduler.mode", "FAIR" 35 | ).getOrCreate() 36 | cls.spark_context = spark.sparkContext 37 | 38 | return cls.spark_context 39 | 40 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/app_builders/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .NexusAppBuilder import NexusAppBuilder 17 | from .RedirectAppBuilder import RedirectAppBuilder -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .NexusRequestHandler import NexusRequestHandler -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/NexusCSVRenderer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | import traceback 18 | from webservice.webmodel import NexusProcessingException 19 | 20 | 21 | class NexusCSVRenderer(object): 22 | def __init__(self, nexus_request): 23 | self._request = nexus_request 24 | 25 | def render(self, tornado_handler, result): 26 | filename = self._request.get_argument('filename') 27 | if filename is None: 28 | filename = result.filename() 29 | filename = f'{filename}.csv' 30 | 31 | tornado_handler.set_header("Content-Type", "text/csv") 32 | tornado_handler.set_header("Content-Disposition", f"filename=\"{filename}\"") 33 | try: 34 | tornado_handler.write(result.toCSV()) 35 | tornado_handler.finish() 36 | except: 37 | traceback.print_exc(file=sys.stdout) 38 | raise NexusProcessingException(reason="Unable to convert results to CSV.") -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/NexusGIFRenderer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | import traceback 18 | from webservice.webmodel import NexusProcessingException 19 | 20 | 21 | class NexusGIFRenderer(object): 22 | def __init__(self, nexus_request): 23 | self._request = nexus_request 24 | 25 | def render(self, tornado_handler, result): 26 | tornado_handler.set_header("Content-Type", "image/gif") 27 | try: 28 | tornado_handler.write(result.toGif()) 29 | tornado_handler.finish() 30 | except AttributeError: 31 | traceback.print_exc(file=sys.stdout) 32 | raise NexusProcessingException(reason="Unable to convert results to a GIF.") 33 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/NexusJSONRenderer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | import traceback 18 | import json 19 | import logging 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class NexusJSONRenderer(object): 25 | def __init__(self, nexus_request): 26 | self.request = nexus_request 27 | 28 | def render(self, tornado_handler, result): 29 | logger.info('Rendering JSON result') 30 | 31 | tornado_handler.set_header("Content-Type", "application/json") 32 | try: 33 | result_str = result.toJson() 34 | 35 | if isinstance(result_str, bytes): 36 | tornado_handler.set_header("Content-Type", "application/gzip") 37 | tornado_handler.set_header("Content-Disposition", 38 | "attachment; filename=\"%s\"" % self.request.get_argument('filename', 39 | "subset.gz")) 40 | logger.info('Writing result') 41 | 42 | tornado_handler.write(result_str) 43 | tornado_handler.finish() 44 | except AttributeError: 45 | traceback.print_exc(file=sys.stdout) 46 | tornado_handler.write(json.dumps(result, indent=4)) 47 | tornado_handler.finish() 48 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/NexusNETCDFRenderer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | import traceback 18 | from webservice.webmodel import NexusProcessingException 19 | 20 | 21 | class NexusNETCDFRenderer(object): 22 | def __init__(self, nexus_request): 23 | self._request = nexus_request 24 | 25 | def render(self, tornado_handler, result): 26 | filename = self._request.get_argument('filename') 27 | if filename is None: 28 | filename = result.filename() 29 | filename = f'{filename}.nc' 30 | 31 | tornado_handler.set_header("Content-Type", "application/x-netcdf") 32 | tornado_handler.set_header("Content-Disposition", f"attachment; filename=\"{filename}\"") 33 | try: 34 | tornado_handler.write(result.toNetCDF()) 35 | except: 36 | traceback.print_exc(file=sys.stdout) 37 | raise NexusProcessingException(reason="Unable to convert results to NetCDF.") 38 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/NexusPNGRenderer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | import traceback 18 | from webservice.webmodel import NexusProcessingException 19 | 20 | 21 | class NexusPNGRenderer(object): 22 | def __init__(self, nexus_request): 23 | self._request = nexus_request 24 | 25 | def render(self, tornado_handler, result): 26 | tornado_handler.set_header("Content-Type", "image/png") 27 | try: 28 | tornado_handler.write(result.toImage()) 29 | tornado_handler.finish() 30 | except AttributeError: 31 | traceback.print_exc(file=sys.stdout) 32 | raise NexusProcessingException(reason="Unable to convert results to an Image.") -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/NexusRendererFactory.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | class NexusRendererFactory(object): 17 | content_types = ["CSV", "JSON", "XML", "PNG", "NETCDF", "ZIP", "GIF"] 18 | module = __import__(__name__) 19 | 20 | @classmethod 21 | def get_renderer(cls, request): 22 | content_type = request.get_content_type().upper() 23 | if content_type in cls.content_types: 24 | renderer_name = 'Nexus' + content_type + 'Renderer' 25 | renderer = getattr(cls.module.nexus_tornado.request.renderers, renderer_name) 26 | return renderer(request) 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/NexusZIPRenderer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | import traceback 18 | from webservice.webmodel import NexusProcessingException 19 | 20 | 21 | class NexusZIPRenderer(object): 22 | def __init__(self, nexus_request): 23 | self._request = nexus_request 24 | 25 | def render(self, tornado_handler, result): 26 | tornado_handler.set_header("Content-Type", "application/zip") 27 | tornado_handler.set_header("Content-Disposition", "filename=\"%s\"" % self._request.get_argument('filename', "download.zip")) 28 | try: 29 | tornado_handler.write(result.toZip()) 30 | tornado_handler.finish() 31 | except: 32 | traceback.print_exc(file=sys.stdout) 33 | raise NexusProcessingException(reason="Unable to convert results to Zip.") 34 | -------------------------------------------------------------------------------- /analysis/webservice/nexus_tornado/request/renderers/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .NexusRendererFactory import NexusRendererFactory 17 | from .NexusJSONRenderer import NexusJSONRenderer 18 | from .NexusCSVRenderer import NexusCSVRenderer 19 | from .NexusNETCDFRenderer import NexusNETCDFRenderer 20 | from .NexusPNGRenderer import NexusPNGRenderer 21 | from .NexusGIFRenderer import NexusGIFRenderer 22 | from .NexusZIPRenderer import NexusZIPRenderer 23 | -------------------------------------------------------------------------------- /analysis/webservice/redirect/RedirectHandler.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import tornado.web 17 | import tornado.gen 18 | import logging 19 | from webservice.webmodel.RequestParameters import RequestParameters 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class RedirectHandler(tornado.web.RequestHandler): 25 | 26 | def initialize(self, redirected_collections=None): 27 | self._redirected_collections = redirected_collections 28 | 29 | @tornado.gen.coroutine 30 | def get(self, algo): 31 | collection_id = self.request.query_arguments[RequestParameters.DATASET][0].decode('utf-8') 32 | collection = self._redirected_collections[collection_id] 33 | full_url = self.request.full_url() 34 | 35 | #redirect to new URL 36 | base_url = full_url[:full_url.find(algo)].rstrip('/') 37 | new_base_url = collection['path'].rstrip('/') 38 | new_full_url = full_url.replace(base_url, new_base_url) 39 | 40 | # use remote collection id 41 | if 'remote_id' in collection: 42 | dataset_param = f"ds={collection_id}" 43 | new_dataset_param = f"ds={collection['remote_id']}" 44 | new_full_url = new_full_url.replace(dataset_param, new_dataset_param) 45 | logger.info("redirect request to ", new_full_url) 46 | self.redirect( 47 | new_full_url, 48 | permanent=True 49 | ) 50 | 51 | 52 | -------------------------------------------------------------------------------- /analysis/webservice/redirect/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .RedirectHandler import RedirectHandler 17 | from .RemoteCollectionMatcher import RemoteCollectionMatcher 18 | from .RemoteSDAPCache import RemoteSDAPCache 19 | from .RemoteSDAPCache import CollectionNotFound -------------------------------------------------------------------------------- /analysis/webservice/resources/transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sdap-nexus/e30109653ffaba0bf64c7814f96a5d9cd581be0f/analysis/webservice/resources/transparent.png -------------------------------------------------------------------------------- /analysis/webservice/resources/wkt.txt: -------------------------------------------------------------------------------- 1 | GEOGCS["WGS 84", 2 | DATUM["WGS_1984", 3 | SPHEROID["WGS 84",6378137,298.257223563, 4 | AUTHORITY["EPSG","7030"]], 5 | AUTHORITY["EPSG","6326"]], 6 | PRIMEM["Greenwich",0, 7 | AUTHORITY["EPSG","8901"]], 8 | UNIT["degree",0.01745329251994328, 9 | AUTHORITY["EPSG","9122"]], 10 | AUTHORITY["EPSG","4326"]] -------------------------------------------------------------------------------- /analysis/webservice/webmodel/Exceptions.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from webservice.webmodel.StandardNexusErrors import StandardNexusErrors 17 | 18 | 19 | class NexusProcessingException(Exception): 20 | def __init__(self, error=StandardNexusErrors.UNKNOWN, reason="", code=500): 21 | self.error = error 22 | self.reason = reason 23 | self.code = code 24 | Exception.__init__(self, reason) 25 | 26 | 27 | class NoDataException(NexusProcessingException): 28 | def __init__(self, reason="No data found for the selected timeframe"): 29 | NexusProcessingException.__init__(self, StandardNexusErrors.NO_DATA, reason, 400) 30 | 31 | 32 | class DatasetNotFoundException(NexusProcessingException): 33 | def __init__(self, reason="Dataset not found"): 34 | NexusProcessingException.__init__(self, StandardNexusErrors.DATASET_MISSING, reason, code=404) -------------------------------------------------------------------------------- /analysis/webservice/webmodel/RequestParameters.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | class RequestParameters(object): 17 | SEASONAL_CYCLE_FILTER = "seasonalFilter" 18 | MAX_LAT = "maxLat" 19 | MIN_LAT = "minLat" 20 | MAX_LON = "maxLon" 21 | MIN_LON = "minLon" 22 | DATASET = "ds" 23 | ENVIRONMENT = "env" 24 | OUTPUT = "output" 25 | START_TIME = "startTime" 26 | END_TIME = "endTime" 27 | START_YEAR = "startYear" 28 | END_YEAR = "endYear" 29 | CLIM_MONTH = "month" 30 | START_ROW = "start" 31 | ROW_COUNT = "numRows" 32 | APPLY_LOW_PASS = "lowPassFilter" 33 | LOW_CUT = "lowCut" 34 | ORDER = "lpOrder" 35 | PLOT_SERIES = "plotSeries" 36 | PLOT_TYPE = "plotType" 37 | NPARTS = "nparts" 38 | METADATA_FILTER = "metadataFilter" 39 | NORMALIZE_DATES = "normalizeDates" 40 | -------------------------------------------------------------------------------- /analysis/webservice/webmodel/StandardNexusErrors.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | class StandardNexusErrors: 17 | UNKNOWN = 1000 18 | NO_DATA = 1001 19 | DATASET_MISSING = 1002 -------------------------------------------------------------------------------- /analysis/webservice/webmodel/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from .NexusRequestObjectTornadoFree import NexusRequestObjectTornadoFree 17 | from .NexusRequestObject import NexusRequestObject 18 | from .RequestParameters import RequestParameters 19 | from .StatsComputeOptions import StatsComputeOptions 20 | from .cached import cached 21 | from .CustomEncoder import CustomEncoder 22 | from .Exceptions import NexusProcessingException, NoDataException, DatasetNotFoundException 23 | from .StandardNexusErrors import StandardNexusErrors 24 | from .CustomEncoder import CustomEncoder 25 | from .NexusResults import NexusResults 26 | -------------------------------------------------------------------------------- /analysis/webservice/webmodel/cached.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import hashlib 17 | import inspect 18 | import time 19 | 20 | __CACHE = {} 21 | 22 | def cached(ttl=60000): 23 | def _hash_function_signature(func): 24 | hash_object = hashlib.md5(str(inspect.getargspec(func)).encode("utf8") + str(func).encode("utf8")) 25 | return hash_object.hexdigest() 26 | 27 | def _now(): 28 | return int(round(time.time() * 1000)) 29 | 30 | def _expired(t): 31 | if t is None or _now() - t > ttl: 32 | return True 33 | else: 34 | return False 35 | 36 | def _cached_decorator(func): 37 | 38 | def func_wrapper(self, computeOptions, **args): 39 | hash = _hash_function_signature(func) 40 | force = computeOptions.get_boolean_arg("nocached", default=False) 41 | 42 | if force or hash not in __CACHE or (hash in __CACHE and _expired(__CACHE[hash]["time"])): 43 | result = func(self, computeOptions, **args) 44 | __CACHE[hash] = { 45 | "time": _now(), 46 | "result": result 47 | } 48 | 49 | return __CACHE[hash]["result"] 50 | 51 | return func_wrapper 52 | 53 | return _cached_decorator 54 | -------------------------------------------------------------------------------- /data-access/config/schemas/cassandra/nexustiles.cql: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one or more 3 | contributor license agreements. See the NOTICE file distributed with 4 | this work for additional information regarding copyright ownership. 5 | The ASF licenses this file to You under the Apache License, Version 2.0 6 | (the "License"); you may not use this file except in compliance with 7 | the License. You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | 18 | CREATE KEYSPACE IF NOT EXISTS nexustiles WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; 19 | 20 | CREATE TABLE IF NOT EXISTS nexustiles.sea_surface_temp ( 21 | tile_id uuid PRIMARY KEY, 22 | tile_blob blob 23 | ); 24 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/contractions_ca.txt: -------------------------------------------------------------------------------- 1 | # Set of Catalan contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | l 5 | m 6 | n 7 | s 8 | t 9 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/contractions_fr.txt: -------------------------------------------------------------------------------- 1 | # Set of French contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | l 4 | m 5 | t 6 | qu 7 | n 8 | s 9 | j 10 | d 11 | c 12 | jusqu 13 | quoiqu 14 | lorsqu 15 | puisqu 16 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/contractions_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | m 5 | b 6 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/contractions_it.txt: -------------------------------------------------------------------------------- 1 | # Set of Italian contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | c 4 | l 5 | all 6 | dall 7 | dell 8 | nell 9 | sull 10 | coll 11 | pell 12 | gl 13 | agl 14 | dagl 15 | degl 16 | negl 17 | sugl 18 | un 19 | m 20 | t 21 | s 22 | v 23 | d 24 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/hyphenations_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish hyphenations for StopFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | h 4 | n 5 | t 6 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stemdict_nl.txt: -------------------------------------------------------------------------------- 1 | # Set of overrides for the dutch stemmer 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | fiets fiets 4 | bromfiets bromfiets 5 | ei eier 6 | kind kinder 7 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_ar.txt: -------------------------------------------------------------------------------- 1 | # This file was created by Jacques Savoy and is distributed under the BSD license. 2 | # See http://members.unine.ch/jacques.savoy/clef/index.html. 3 | # Also see http://www.opensource.org/licenses/bsd-license.html 4 | # Cleaned on October 11, 2009 (not normalized, so use before normalization) 5 | # This means that when modifying this list, you might need to add some 6 | # redundant entries, for example containing forms with both أ and ا 7 | من 8 | ومن 9 | منها 10 | منه 11 | في 12 | وفي 13 | فيها 14 | فيه 15 | و 16 | ف 17 | ثم 18 | او 19 | أو 20 | ب 21 | بها 22 | به 23 | ا 24 | أ 25 | اى 26 | اي 27 | أي 28 | أى 29 | لا 30 | ولا 31 | الا 32 | ألا 33 | إلا 34 | لكن 35 | ما 36 | وما 37 | كما 38 | فما 39 | عن 40 | مع 41 | اذا 42 | إذا 43 | ان 44 | أن 45 | إن 46 | انها 47 | أنها 48 | إنها 49 | انه 50 | أنه 51 | إنه 52 | بان 53 | بأن 54 | فان 55 | فأن 56 | وان 57 | وأن 58 | وإن 59 | التى 60 | التي 61 | الذى 62 | الذي 63 | الذين 64 | الى 65 | الي 66 | إلى 67 | إلي 68 | على 69 | عليها 70 | عليه 71 | اما 72 | أما 73 | إما 74 | ايضا 75 | أيضا 76 | كل 77 | وكل 78 | لم 79 | ولم 80 | لن 81 | ولن 82 | هى 83 | هي 84 | هو 85 | وهى 86 | وهي 87 | وهو 88 | فهى 89 | فهي 90 | فهو 91 | انت 92 | أنت 93 | لك 94 | لها 95 | له 96 | هذه 97 | هذا 98 | تلك 99 | ذلك 100 | هناك 101 | كانت 102 | كان 103 | يكون 104 | تكون 105 | وكانت 106 | وكان 107 | غير 108 | بعض 109 | قد 110 | نحو 111 | بين 112 | بينما 113 | منذ 114 | ضمن 115 | حيث 116 | الان 117 | الآن 118 | خلال 119 | بعد 120 | قبل 121 | حتى 122 | عند 123 | عندما 124 | لدى 125 | جميع 126 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_bg.txt: -------------------------------------------------------------------------------- 1 | # This file was created by Jacques Savoy and is distributed under the BSD license. 2 | # See http://members.unine.ch/jacques.savoy/clef/index.html. 3 | # Also see http://www.opensource.org/licenses/bsd-license.html 4 | а 5 | аз 6 | ако 7 | ала 8 | бе 9 | без 10 | беше 11 | би 12 | бил 13 | била 14 | били 15 | било 16 | близо 17 | бъдат 18 | бъде 19 | бяха 20 | в 21 | вас 22 | ваш 23 | ваша 24 | вероятно 25 | вече 26 | взема 27 | ви 28 | вие 29 | винаги 30 | все 31 | всеки 32 | всички 33 | всичко 34 | всяка 35 | във 36 | въпреки 37 | върху 38 | г 39 | ги 40 | главно 41 | го 42 | д 43 | да 44 | дали 45 | до 46 | докато 47 | докога 48 | дори 49 | досега 50 | доста 51 | е 52 | едва 53 | един 54 | ето 55 | за 56 | зад 57 | заедно 58 | заради 59 | засега 60 | затова 61 | защо 62 | защото 63 | и 64 | из 65 | или 66 | им 67 | има 68 | имат 69 | иска 70 | й 71 | каза 72 | как 73 | каква 74 | какво 75 | както 76 | какъв 77 | като 78 | кога 79 | когато 80 | което 81 | които 82 | кой 83 | който 84 | колко 85 | която 86 | къде 87 | където 88 | към 89 | ли 90 | м 91 | ме 92 | между 93 | мен 94 | ми 95 | мнозина 96 | мога 97 | могат 98 | може 99 | моля 100 | момента 101 | му 102 | н 103 | на 104 | над 105 | назад 106 | най 107 | направи 108 | напред 109 | например 110 | нас 111 | не 112 | него 113 | нея 114 | ни 115 | ние 116 | никой 117 | нито 118 | но 119 | някои 120 | някой 121 | няма 122 | обаче 123 | около 124 | освен 125 | особено 126 | от 127 | отгоре 128 | отново 129 | още 130 | пак 131 | по 132 | повече 133 | повечето 134 | под 135 | поне 136 | поради 137 | после 138 | почти 139 | прави 140 | пред 141 | преди 142 | през 143 | при 144 | пък 145 | първо 146 | с 147 | са 148 | само 149 | се 150 | сега 151 | си 152 | скоро 153 | след 154 | сме 155 | според 156 | сред 157 | срещу 158 | сте 159 | съм 160 | със 161 | също 162 | т 163 | тази 164 | така 165 | такива 166 | такъв 167 | там 168 | твой 169 | те 170 | тези 171 | ти 172 | тн 173 | то 174 | това 175 | тогава 176 | този 177 | той 178 | толкова 179 | точно 180 | трябва 181 | тук 182 | тъй 183 | тя 184 | тях 185 | у 186 | харесва 187 | ч 188 | че 189 | често 190 | чрез 191 | ще 192 | щом 193 | я 194 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_cz.txt: -------------------------------------------------------------------------------- 1 | a 2 | s 3 | k 4 | o 5 | i 6 | u 7 | v 8 | z 9 | dnes 10 | cz 11 | tímto 12 | budeš 13 | budem 14 | byli 15 | jseš 16 | můj 17 | svým 18 | ta 19 | tomto 20 | tohle 21 | tuto 22 | tyto 23 | jej 24 | zda 25 | proč 26 | máte 27 | tato 28 | kam 29 | tohoto 30 | kdo 31 | kteří 32 | mi 33 | nám 34 | tom 35 | tomuto 36 | mít 37 | nic 38 | proto 39 | kterou 40 | byla 41 | toho 42 | protože 43 | asi 44 | ho 45 | naši 46 | napište 47 | re 48 | což 49 | tím 50 | takže 51 | svých 52 | její 53 | svými 54 | jste 55 | aj 56 | tu 57 | tedy 58 | teto 59 | bylo 60 | kde 61 | ke 62 | pravé 63 | ji 64 | nad 65 | nejsou 66 | či 67 | pod 68 | téma 69 | mezi 70 | přes 71 | ty 72 | pak 73 | vám 74 | ani 75 | když 76 | však 77 | neg 78 | jsem 79 | tento 80 | článku 81 | články 82 | aby 83 | jsme 84 | před 85 | pta 86 | jejich 87 | byl 88 | ještě 89 | až 90 | bez 91 | také 92 | pouze 93 | první 94 | vaše 95 | která 96 | nás 97 | nový 98 | tipy 99 | pokud 100 | může 101 | strana 102 | jeho 103 | své 104 | jiné 105 | zprávy 106 | nové 107 | není 108 | vás 109 | jen 110 | podle 111 | zde 112 | už 113 | být 114 | více 115 | bude 116 | již 117 | než 118 | který 119 | by 120 | které 121 | co 122 | nebo 123 | ten 124 | tak 125 | má 126 | při 127 | od 128 | po 129 | jsou 130 | jak 131 | další 132 | ale 133 | si 134 | se 135 | ve 136 | to 137 | jako 138 | za 139 | zpět 140 | ze 141 | do 142 | pro 143 | je 144 | na 145 | atd 146 | atp 147 | jakmile 148 | přičemž 149 | já 150 | on 151 | ona 152 | ono 153 | oni 154 | ony 155 | my 156 | vy 157 | jí 158 | ji 159 | mě 160 | mne 161 | jemu 162 | tomu 163 | těm 164 | těmu 165 | němu 166 | němuž 167 | jehož 168 | jíž 169 | jelikož 170 | jež 171 | jakož 172 | načež 173 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_el.txt: -------------------------------------------------------------------------------- 1 | # Lucene Greek Stopwords list 2 | # Note: by default this file is used after GreekLowerCaseFilter, 3 | # so when modifying this file use 'σ' instead of 'ς' 4 | ο 5 | η 6 | το 7 | οι 8 | τα 9 | του 10 | τησ 11 | των 12 | τον 13 | την 14 | και 15 | κι 16 | κ 17 | ειμαι 18 | εισαι 19 | ειναι 20 | ειμαστε 21 | ειστε 22 | στο 23 | στον 24 | στη 25 | στην 26 | μα 27 | αλλα 28 | απο 29 | για 30 | προσ 31 | με 32 | σε 33 | ωσ 34 | παρα 35 | αντι 36 | κατα 37 | μετα 38 | θα 39 | να 40 | δε 41 | δεν 42 | μη 43 | μην 44 | επι 45 | ενω 46 | εαν 47 | αν 48 | τοτε 49 | που 50 | πωσ 51 | ποιοσ 52 | ποια 53 | ποιο 54 | ποιοι 55 | ποιεσ 56 | ποιων 57 | ποιουσ 58 | αυτοσ 59 | αυτη 60 | αυτο 61 | αυτοι 62 | αυτων 63 | αυτουσ 64 | αυτεσ 65 | αυτα 66 | εκεινοσ 67 | εκεινη 68 | εκεινο 69 | εκεινοι 70 | εκεινεσ 71 | εκεινα 72 | εκεινων 73 | εκεινουσ 74 | οπωσ 75 | ομωσ 76 | ισωσ 77 | οσο 78 | οτι 79 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_en.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # a couple of test stopwords to test that the words are really being 17 | # configured from this file: 18 | stopworda 19 | stopwordb 20 | 21 | # Standard english stop words taken from Lucene's StopAnalyzer 22 | a 23 | an 24 | and 25 | are 26 | as 27 | at 28 | be 29 | but 30 | by 31 | for 32 | if 33 | in 34 | into 35 | is 36 | it 37 | no 38 | not 39 | of 40 | on 41 | or 42 | such 43 | that 44 | the 45 | their 46 | then 47 | there 48 | these 49 | they 50 | this 51 | to 52 | was 53 | will 54 | with 55 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_eu.txt: -------------------------------------------------------------------------------- 1 | # example set of basque stopwords 2 | al 3 | anitz 4 | arabera 5 | asko 6 | baina 7 | bat 8 | batean 9 | batek 10 | bati 11 | batzuei 12 | batzuek 13 | batzuetan 14 | batzuk 15 | bera 16 | beraiek 17 | berau 18 | berauek 19 | bere 20 | berori 21 | beroriek 22 | beste 23 | bezala 24 | da 25 | dago 26 | dira 27 | ditu 28 | du 29 | dute 30 | edo 31 | egin 32 | ere 33 | eta 34 | eurak 35 | ez 36 | gainera 37 | gu 38 | gutxi 39 | guzti 40 | haiei 41 | haiek 42 | haietan 43 | hainbeste 44 | hala 45 | han 46 | handik 47 | hango 48 | hara 49 | hari 50 | hark 51 | hartan 52 | hau 53 | hauei 54 | hauek 55 | hauetan 56 | hemen 57 | hemendik 58 | hemengo 59 | hi 60 | hona 61 | honek 62 | honela 63 | honetan 64 | honi 65 | hor 66 | hori 67 | horiei 68 | horiek 69 | horietan 70 | horko 71 | horra 72 | horrek 73 | horrela 74 | horretan 75 | horri 76 | hortik 77 | hura 78 | izan 79 | ni 80 | noiz 81 | nola 82 | non 83 | nondik 84 | nongo 85 | nor 86 | nora 87 | ze 88 | zein 89 | zen 90 | zenbait 91 | zenbat 92 | zer 93 | zergatik 94 | ziren 95 | zituen 96 | zu 97 | zuek 98 | zuen 99 | zuten 100 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_ga.txt: -------------------------------------------------------------------------------- 1 | 2 | a 3 | ach 4 | ag 5 | agus 6 | an 7 | aon 8 | ar 9 | arna 10 | as 11 | b' 12 | ba 13 | beirt 14 | bhúr 15 | caoga 16 | ceathair 17 | ceathrar 18 | chomh 19 | chtó 20 | chuig 21 | chun 22 | cois 23 | céad 24 | cúig 25 | cúigear 26 | d' 27 | daichead 28 | dar 29 | de 30 | deich 31 | deichniúr 32 | den 33 | dhá 34 | do 35 | don 36 | dtí 37 | dá 38 | dár 39 | dó 40 | faoi 41 | faoin 42 | faoina 43 | faoinár 44 | fara 45 | fiche 46 | gach 47 | gan 48 | go 49 | gur 50 | haon 51 | hocht 52 | i 53 | iad 54 | idir 55 | in 56 | ina 57 | ins 58 | inár 59 | is 60 | le 61 | leis 62 | lena 63 | lenár 64 | m' 65 | mar 66 | mo 67 | mé 68 | na 69 | nach 70 | naoi 71 | naonúr 72 | ná 73 | ní 74 | níor 75 | nó 76 | nócha 77 | ocht 78 | ochtar 79 | os 80 | roimh 81 | sa 82 | seacht 83 | seachtar 84 | seachtó 85 | seasca 86 | seisear 87 | siad 88 | sibh 89 | sinn 90 | sna 91 | sé 92 | sí 93 | tar 94 | thar 95 | thú 96 | triúr 97 | trí 98 | trína 99 | trínár 100 | tríocha 101 | tú 102 | um 103 | ár 104 | é 105 | éis 106 | í 107 | ó 108 | ón 109 | óna 110 | ónár 111 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_gl.txt: -------------------------------------------------------------------------------- 1 | # galican stopwords 2 | a 3 | aínda 4 | alí 5 | aquel 6 | aquela 7 | aquelas 8 | aqueles 9 | aquilo 10 | aquí 11 | ao 12 | aos 13 | as 14 | así 15 | á 16 | ben 17 | cando 18 | che 19 | co 20 | coa 21 | comigo 22 | con 23 | connosco 24 | contigo 25 | convosco 26 | coas 27 | cos 28 | cun 29 | cuns 30 | cunha 31 | cunhas 32 | da 33 | dalgunha 34 | dalgunhas 35 | dalgún 36 | dalgúns 37 | das 38 | de 39 | del 40 | dela 41 | delas 42 | deles 43 | desde 44 | deste 45 | do 46 | dos 47 | dun 48 | duns 49 | dunha 50 | dunhas 51 | e 52 | el 53 | ela 54 | elas 55 | eles 56 | en 57 | era 58 | eran 59 | esa 60 | esas 61 | ese 62 | eses 63 | esta 64 | estar 65 | estaba 66 | está 67 | están 68 | este 69 | estes 70 | estiven 71 | estou 72 | eu 73 | é 74 | facer 75 | foi 76 | foron 77 | fun 78 | había 79 | hai 80 | iso 81 | isto 82 | la 83 | las 84 | lle 85 | lles 86 | lo 87 | los 88 | mais 89 | me 90 | meu 91 | meus 92 | min 93 | miña 94 | miñas 95 | moi 96 | na 97 | nas 98 | neste 99 | nin 100 | no 101 | non 102 | nos 103 | nosa 104 | nosas 105 | noso 106 | nosos 107 | nós 108 | nun 109 | nunha 110 | nuns 111 | nunhas 112 | o 113 | os 114 | ou 115 | ó 116 | ós 117 | para 118 | pero 119 | pode 120 | pois 121 | pola 122 | polas 123 | polo 124 | polos 125 | por 126 | que 127 | se 128 | senón 129 | ser 130 | seu 131 | seus 132 | sexa 133 | sido 134 | sobre 135 | súa 136 | súas 137 | tamén 138 | tan 139 | te 140 | ten 141 | teñen 142 | teño 143 | ter 144 | teu 145 | teus 146 | ti 147 | tido 148 | tiña 149 | tiven 150 | túa 151 | túas 152 | un 153 | unha 154 | unhas 155 | uns 156 | vos 157 | vosa 158 | vosas 159 | voso 160 | vosos 161 | vós 162 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_hy.txt: -------------------------------------------------------------------------------- 1 | # example set of Armenian stopwords. 2 | այդ 3 | այլ 4 | այն 5 | այս 6 | դու 7 | դուք 8 | եմ 9 | են 10 | ենք 11 | ես 12 | եք 13 | է 14 | էի 15 | էին 16 | էինք 17 | էիր 18 | էիք 19 | էր 20 | ըստ 21 | թ 22 | ի 23 | ին 24 | իսկ 25 | իր 26 | կամ 27 | համար 28 | հետ 29 | հետո 30 | մենք 31 | մեջ 32 | մի 33 | ն 34 | նա 35 | նաև 36 | նրա 37 | նրանք 38 | որ 39 | որը 40 | որոնք 41 | որպես 42 | ու 43 | ում 44 | պիտի 45 | վրա 46 | և 47 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_ja.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file defines a stopword set for Japanese. 3 | # 4 | # This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. 5 | # Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 6 | # for frequency lists, etc. that can be useful for making your own set (if desired) 7 | # 8 | # Note that there is an overlap between these stopwords and the terms stopped when used 9 | # in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note 10 | # that comments are not allowed on the same line as stopwords. 11 | # 12 | # Also note that stopping is done in a case-insensitive manner. Change your StopFilter 13 | # configuration if you need case-sensitive stopping. Lastly, note that stopping is done 14 | # using the same character width as the entries in this file. Since this StopFilter is 15 | # normally done after a CJKWidthFilter in your chain, you would usually want your romaji 16 | # entries to be in half-width and your kana entries to be in full-width. 17 | # 18 | の 19 | に 20 | は 21 | を 22 | た 23 | が 24 | で 25 | て 26 | と 27 | し 28 | れ 29 | さ 30 | ある 31 | いる 32 | も 33 | する 34 | から 35 | な 36 | こと 37 | として 38 | い 39 | や 40 | れる 41 | など 42 | なっ 43 | ない 44 | この 45 | ため 46 | その 47 | あっ 48 | よう 49 | また 50 | もの 51 | という 52 | あり 53 | まで 54 | られ 55 | なる 56 | へ 57 | か 58 | だ 59 | これ 60 | によって 61 | により 62 | おり 63 | より 64 | による 65 | ず 66 | なり 67 | られる 68 | において 69 | ば 70 | なかっ 71 | なく 72 | しかし 73 | について 74 | せ 75 | だっ 76 | その後 77 | できる 78 | それ 79 | う 80 | ので 81 | なお 82 | のみ 83 | でき 84 | き 85 | つ 86 | における 87 | および 88 | いう 89 | さらに 90 | でも 91 | ら 92 | たり 93 | その他 94 | に関する 95 | たち 96 | ます 97 | ん 98 | なら 99 | に対して 100 | 特に 101 | せる 102 | 及び 103 | これら 104 | とき 105 | では 106 | にて 107 | ほか 108 | ながら 109 | うち 110 | そして 111 | とともに 112 | ただし 113 | かつて 114 | それぞれ 115 | または 116 | お 117 | ほど 118 | ものの 119 | に対する 120 | ほとんど 121 | と共に 122 | といった 123 | です 124 | とも 125 | ところ 126 | ここ 127 | ##### End of file 128 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_lv.txt: -------------------------------------------------------------------------------- 1 | # Set of Latvian stopwords from A Stemming Algorithm for Latvian, Karlis Kreslins 2 | # the original list of over 800 forms was refined: 3 | # pronouns, adverbs, interjections were removed 4 | # 5 | # prepositions 6 | aiz 7 | ap 8 | ar 9 | apakš 10 | ārpus 11 | augšpus 12 | bez 13 | caur 14 | dēļ 15 | gar 16 | iekš 17 | iz 18 | kopš 19 | labad 20 | lejpus 21 | līdz 22 | no 23 | otrpus 24 | pa 25 | par 26 | pār 27 | pēc 28 | pie 29 | pirms 30 | pret 31 | priekš 32 | starp 33 | šaipus 34 | uz 35 | viņpus 36 | virs 37 | virspus 38 | zem 39 | apakšpus 40 | # Conjunctions 41 | un 42 | bet 43 | jo 44 | ja 45 | ka 46 | lai 47 | tomēr 48 | tikko 49 | turpretī 50 | arī 51 | kaut 52 | gan 53 | tādēļ 54 | tā 55 | ne 56 | tikvien 57 | vien 58 | kā 59 | ir 60 | te 61 | vai 62 | kamēr 63 | # Particles 64 | ar 65 | diezin 66 | droši 67 | diemžēl 68 | nebūt 69 | ik 70 | it 71 | taču 72 | nu 73 | pat 74 | tiklab 75 | iekšpus 76 | nedz 77 | tik 78 | nevis 79 | turpretim 80 | jeb 81 | iekam 82 | iekām 83 | iekāms 84 | kolīdz 85 | līdzko 86 | tiklīdz 87 | jebšu 88 | tālab 89 | tāpēc 90 | nekā 91 | itin 92 | jā 93 | jau 94 | jel 95 | nē 96 | nezin 97 | tad 98 | tikai 99 | vis 100 | tak 101 | iekams 102 | vien 103 | # modal verbs 104 | būt 105 | biju 106 | biji 107 | bija 108 | bijām 109 | bijāt 110 | esmu 111 | esi 112 | esam 113 | esat 114 | būšu 115 | būsi 116 | būs 117 | būsim 118 | būsiet 119 | tikt 120 | tiku 121 | tiki 122 | tika 123 | tikām 124 | tikāt 125 | tieku 126 | tiec 127 | tiek 128 | tiekam 129 | tiekat 130 | tikšu 131 | tiks 132 | tiksim 133 | tiksiet 134 | tapt 135 | tapi 136 | tapāt 137 | topat 138 | tapšu 139 | tapsi 140 | taps 141 | tapsim 142 | tapsiet 143 | kļūt 144 | kļuvu 145 | kļuvi 146 | kļuva 147 | kļuvām 148 | kļuvāt 149 | kļūstu 150 | kļūsti 151 | kļūst 152 | kļūstam 153 | kļūstat 154 | kļūšu 155 | kļūsi 156 | kļūs 157 | kļūsim 158 | kļūsiet 159 | # verbs 160 | varēt 161 | varēju 162 | varējām 163 | varēšu 164 | varēsim 165 | var 166 | varēji 167 | varējāt 168 | varēsi 169 | varēsiet 170 | varat 171 | varēja 172 | varēs 173 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/stopwords_th.txt: -------------------------------------------------------------------------------- 1 | # Thai stopwords from: 2 | # "Opinion Detection in Thai Political News Columns 3 | # Based on Subjectivity Analysis" 4 | # Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak 5 | ไว้ 6 | ไม่ 7 | ไป 8 | ได้ 9 | ให้ 10 | ใน 11 | โดย 12 | แห่ง 13 | แล้ว 14 | และ 15 | แรก 16 | แบบ 17 | แต่ 18 | เอง 19 | เห็น 20 | เลย 21 | เริ่ม 22 | เรา 23 | เมื่อ 24 | เพื่อ 25 | เพราะ 26 | เป็นการ 27 | เป็น 28 | เปิดเผย 29 | เปิด 30 | เนื่องจาก 31 | เดียวกัน 32 | เดียว 33 | เช่น 34 | เฉพาะ 35 | เคย 36 | เข้า 37 | เขา 38 | อีก 39 | อาจ 40 | อะไร 41 | ออก 42 | อย่าง 43 | อยู่ 44 | อยาก 45 | หาก 46 | หลาย 47 | หลังจาก 48 | หลัง 49 | หรือ 50 | หนึ่ง 51 | ส่วน 52 | ส่ง 53 | สุด 54 | สําหรับ 55 | ว่า 56 | วัน 57 | ลง 58 | ร่วม 59 | ราย 60 | รับ 61 | ระหว่าง 62 | รวม 63 | ยัง 64 | มี 65 | มาก 66 | มา 67 | พร้อม 68 | พบ 69 | ผ่าน 70 | ผล 71 | บาง 72 | น่า 73 | นี้ 74 | นํา 75 | นั้น 76 | นัก 77 | นอกจาก 78 | ทุก 79 | ที่สุด 80 | ที่ 81 | ทําให้ 82 | ทํา 83 | ทาง 84 | ทั้งนี้ 85 | ทั้ง 86 | ถ้า 87 | ถูก 88 | ถึง 89 | ต้อง 90 | ต่างๆ 91 | ต่าง 92 | ต่อ 93 | ตาม 94 | ตั้งแต่ 95 | ตั้ง 96 | ด้าน 97 | ด้วย 98 | ดัง 99 | ซึ่ง 100 | ช่วง 101 | จึง 102 | จาก 103 | จัด 104 | จะ 105 | คือ 106 | ความ 107 | ครั้ง 108 | คง 109 | ขึ้น 110 | ของ 111 | ขอ 112 | ขณะ 113 | ก่อน 114 | ก็ 115 | การ 116 | กับ 117 | กัน 118 | กว่า 119 | กล่าว 120 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/lang/userdict_ja.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This is a sample user dictionary for Kuromoji (JapaneseTokenizer) 3 | # 4 | # Add entries to this file in order to override the statistical model in terms 5 | # of segmentation, readings and part-of-speech tags. Notice that entries do 6 | # not have weights since they are always used when found. This is by-design 7 | # in order to maximize ease-of-use. 8 | # 9 | # Entries are defined using the following CSV format: 10 | # , ... , ... , 11 | # 12 | # Notice that a single half-width space separates tokens and readings, and 13 | # that the number tokens and readings must match exactly. 14 | # 15 | # Also notice that multiple entries with the same is undefined. 16 | # 17 | # Whitespace only lines are ignored. Comments are not allowed on entry lines. 18 | # 19 | 20 | # Custom segmentation for kanji compounds 21 | 日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞 22 | 関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタム名詞 23 | 24 | # Custom segmentation for compound katakana 25 | トートバッグ,トート バッグ,トート バッグ,かずカナ名詞 26 | ショルダーバッグ,ショルダー バッグ,ショルダー バッグ,かずカナ名詞 27 | 28 | # Custom reading for former sumo wrestler 29 | 朝青龍,朝青龍,アサショウリュウ,カスタム人名 30 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/params.json: -------------------------------------------------------------------------------- 1 | {"params":{ 2 | "query":{ 3 | "defType":"edismax", 4 | "q.alt":"*:*", 5 | "rows":"10", 6 | "fl":"*,score", 7 | "":{"v":0} 8 | }, 9 | "facets":{ 10 | "facet":"on", 11 | "facet.mincount": "1", 12 | "":{"v":0} 13 | }, 14 | "velocity":{ 15 | "wt": "velocity", 16 | "v.template":"browse", 17 | "v.layout": "layout", 18 | "":{"v":0} 19 | } 20 | }} -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/protwords.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | # Use a protected word file to protect against the stemmer reducing two 15 | # unrelated words to the same base word. 16 | 17 | # Some non-words that normally won't be encountered, 18 | # just to test that they won't be stemmed. 19 | dontstems 20 | zwhacky 21 | 22 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/stopwords.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr-7.1.0/nexustiles/conf/synonyms.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | #some test synonym mappings unlikely to appear in real input text 15 | aaafoo => aaabar 16 | bbbfoo => bbbfoo bbbbar 17 | cccfoo => cccbar cccbaz 18 | fooaaa,baraaa,bazaaa 19 | 20 | # Some synonym groups specific to this example 21 | GB,gib,gigabyte,gigabytes 22 | MB,mib,megabyte,megabytes 23 | Television, Televisions, TV, TVs 24 | #notice we use "gib" instead of "GiB" so any WordDelimiterGraphFilter coming 25 | #after us won't split it into two words. 26 | 27 | # Synonym mappings can be used for spelling correction too 28 | pixima => pixma 29 | 30 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/dataset/conf/_rest_managed.json: -------------------------------------------------------------------------------- 1 | {"initArgs":{},"managedList":[]} 2 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/dataset/conf/lang/stopwords_en.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # a couple of test stopwords to test that the words are really being 17 | # configured from this file: 18 | stopworda 19 | stopwordb 20 | 21 | # Standard english stop words taken from Lucene's StopAnalyzer 22 | a 23 | an 24 | and 25 | are 26 | as 27 | at 28 | be 29 | but 30 | by 31 | for 32 | if 33 | in 34 | into 35 | is 36 | it 37 | no 38 | not 39 | of 40 | on 41 | or 42 | such 43 | that 44 | the 45 | their 46 | then 47 | there 48 | these 49 | they 50 | this 51 | to 52 | was 53 | will 54 | with 55 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/dataset/conf/protwords.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | # Use a protected word file to protect against the stemmer reducing two 15 | # unrelated words to the same base word. 16 | 17 | # Some non-words that normally won't be encountered, 18 | # just to test that they won't be stemmed. 19 | dontstems 20 | zwhacky 21 | 22 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/dataset/conf/stopwords.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/dataset/conf/synonyms.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | #some test synonym mappings unlikely to appear in real input text 15 | aaafoo => aaabar 16 | bbbfoo => bbbfoo bbbbar 17 | cccfoo => cccbar cccbaz 18 | fooaaa,baraaa,bazaaa 19 | 20 | # Some synonym groups specific to this example 21 | GB,gib,gigabyte,gigabytes 22 | MB,mib,megabyte,megabytes 23 | Television, Televisions, TV, TVs 24 | #notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming 25 | #after us won't split it into two words. 26 | 27 | # Synonym mappings can be used for spelling correction too 28 | pixima => pixma 29 | 30 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/dataset/core.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/elevate.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 26 | 27 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/contractions_ca.txt: -------------------------------------------------------------------------------- 1 | # Set of Catalan contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | l 5 | m 6 | n 7 | s 8 | t 9 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/contractions_fr.txt: -------------------------------------------------------------------------------- 1 | # Set of French contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | l 4 | m 5 | t 6 | qu 7 | n 8 | s 9 | j 10 | d 11 | c 12 | jusqu 13 | quoiqu 14 | lorsqu 15 | puisqu 16 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/contractions_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | d 4 | m 5 | b 6 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/contractions_it.txt: -------------------------------------------------------------------------------- 1 | # Set of Italian contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | c 4 | l 5 | all 6 | dall 7 | dell 8 | nell 9 | sull 10 | coll 11 | pell 12 | gl 13 | agl 14 | dagl 15 | degl 16 | negl 17 | sugl 18 | un 19 | m 20 | t 21 | s 22 | v 23 | d 24 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/hyphenations_ga.txt: -------------------------------------------------------------------------------- 1 | # Set of Irish hyphenations for StopFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | h 4 | n 5 | t 6 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stemdict_nl.txt: -------------------------------------------------------------------------------- 1 | # Set of overrides for the dutch stemmer 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | fiets fiets 4 | bromfiets bromfiets 5 | ei eier 6 | kind kinder 7 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_ar.txt: -------------------------------------------------------------------------------- 1 | # This file was created by Jacques Savoy and is distributed under the BSD license. 2 | # See http://members.unine.ch/jacques.savoy/clef/index.html. 3 | # Also see http://www.opensource.org/licenses/bsd-license.html 4 | # Cleaned on October 11, 2009 (not normalized, so use before normalization) 5 | # This means that when modifying this list, you might need to add some 6 | # redundant entries, for example containing forms with both أ and ا 7 | من 8 | ومن 9 | منها 10 | منه 11 | في 12 | وفي 13 | فيها 14 | فيه 15 | و 16 | ف 17 | ثم 18 | او 19 | أو 20 | ب 21 | بها 22 | به 23 | ا 24 | أ 25 | اى 26 | اي 27 | أي 28 | أى 29 | لا 30 | ولا 31 | الا 32 | ألا 33 | إلا 34 | لكن 35 | ما 36 | وما 37 | كما 38 | فما 39 | عن 40 | مع 41 | اذا 42 | إذا 43 | ان 44 | أن 45 | إن 46 | انها 47 | أنها 48 | إنها 49 | انه 50 | أنه 51 | إنه 52 | بان 53 | بأن 54 | فان 55 | فأن 56 | وان 57 | وأن 58 | وإن 59 | التى 60 | التي 61 | الذى 62 | الذي 63 | الذين 64 | الى 65 | الي 66 | إلى 67 | إلي 68 | على 69 | عليها 70 | عليه 71 | اما 72 | أما 73 | إما 74 | ايضا 75 | أيضا 76 | كل 77 | وكل 78 | لم 79 | ولم 80 | لن 81 | ولن 82 | هى 83 | هي 84 | هو 85 | وهى 86 | وهي 87 | وهو 88 | فهى 89 | فهي 90 | فهو 91 | انت 92 | أنت 93 | لك 94 | لها 95 | له 96 | هذه 97 | هذا 98 | تلك 99 | ذلك 100 | هناك 101 | كانت 102 | كان 103 | يكون 104 | تكون 105 | وكانت 106 | وكان 107 | غير 108 | بعض 109 | قد 110 | نحو 111 | بين 112 | بينما 113 | منذ 114 | ضمن 115 | حيث 116 | الان 117 | الآن 118 | خلال 119 | بعد 120 | قبل 121 | حتى 122 | عند 123 | عندما 124 | لدى 125 | جميع 126 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_bg.txt: -------------------------------------------------------------------------------- 1 | # This file was created by Jacques Savoy and is distributed under the BSD license. 2 | # See http://members.unine.ch/jacques.savoy/clef/index.html. 3 | # Also see http://www.opensource.org/licenses/bsd-license.html 4 | а 5 | аз 6 | ако 7 | ала 8 | бе 9 | без 10 | беше 11 | би 12 | бил 13 | била 14 | били 15 | било 16 | близо 17 | бъдат 18 | бъде 19 | бяха 20 | в 21 | вас 22 | ваш 23 | ваша 24 | вероятно 25 | вече 26 | взема 27 | ви 28 | вие 29 | винаги 30 | все 31 | всеки 32 | всички 33 | всичко 34 | всяка 35 | във 36 | въпреки 37 | върху 38 | г 39 | ги 40 | главно 41 | го 42 | д 43 | да 44 | дали 45 | до 46 | докато 47 | докога 48 | дори 49 | досега 50 | доста 51 | е 52 | едва 53 | един 54 | ето 55 | за 56 | зад 57 | заедно 58 | заради 59 | засега 60 | затова 61 | защо 62 | защото 63 | и 64 | из 65 | или 66 | им 67 | има 68 | имат 69 | иска 70 | й 71 | каза 72 | как 73 | каква 74 | какво 75 | както 76 | какъв 77 | като 78 | кога 79 | когато 80 | което 81 | които 82 | кой 83 | който 84 | колко 85 | която 86 | къде 87 | където 88 | към 89 | ли 90 | м 91 | ме 92 | между 93 | мен 94 | ми 95 | мнозина 96 | мога 97 | могат 98 | може 99 | моля 100 | момента 101 | му 102 | н 103 | на 104 | над 105 | назад 106 | най 107 | направи 108 | напред 109 | например 110 | нас 111 | не 112 | него 113 | нея 114 | ни 115 | ние 116 | никой 117 | нито 118 | но 119 | някои 120 | някой 121 | няма 122 | обаче 123 | около 124 | освен 125 | особено 126 | от 127 | отгоре 128 | отново 129 | още 130 | пак 131 | по 132 | повече 133 | повечето 134 | под 135 | поне 136 | поради 137 | после 138 | почти 139 | прави 140 | пред 141 | преди 142 | през 143 | при 144 | пък 145 | първо 146 | с 147 | са 148 | само 149 | се 150 | сега 151 | си 152 | скоро 153 | след 154 | сме 155 | според 156 | сред 157 | срещу 158 | сте 159 | съм 160 | със 161 | също 162 | т 163 | тази 164 | така 165 | такива 166 | такъв 167 | там 168 | твой 169 | те 170 | тези 171 | ти 172 | тн 173 | то 174 | това 175 | тогава 176 | този 177 | той 178 | толкова 179 | точно 180 | трябва 181 | тук 182 | тъй 183 | тя 184 | тях 185 | у 186 | харесва 187 | ч 188 | че 189 | често 190 | чрез 191 | ще 192 | щом 193 | я 194 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_cz.txt: -------------------------------------------------------------------------------- 1 | a 2 | s 3 | k 4 | o 5 | i 6 | u 7 | v 8 | z 9 | dnes 10 | cz 11 | tímto 12 | budeš 13 | budem 14 | byli 15 | jseš 16 | můj 17 | svým 18 | ta 19 | tomto 20 | tohle 21 | tuto 22 | tyto 23 | jej 24 | zda 25 | proč 26 | máte 27 | tato 28 | kam 29 | tohoto 30 | kdo 31 | kteří 32 | mi 33 | nám 34 | tom 35 | tomuto 36 | mít 37 | nic 38 | proto 39 | kterou 40 | byla 41 | toho 42 | protože 43 | asi 44 | ho 45 | naši 46 | napište 47 | re 48 | což 49 | tím 50 | takže 51 | svých 52 | její 53 | svými 54 | jste 55 | aj 56 | tu 57 | tedy 58 | teto 59 | bylo 60 | kde 61 | ke 62 | pravé 63 | ji 64 | nad 65 | nejsou 66 | či 67 | pod 68 | téma 69 | mezi 70 | přes 71 | ty 72 | pak 73 | vám 74 | ani 75 | když 76 | však 77 | neg 78 | jsem 79 | tento 80 | článku 81 | články 82 | aby 83 | jsme 84 | před 85 | pta 86 | jejich 87 | byl 88 | ještě 89 | až 90 | bez 91 | také 92 | pouze 93 | první 94 | vaše 95 | která 96 | nás 97 | nový 98 | tipy 99 | pokud 100 | může 101 | strana 102 | jeho 103 | své 104 | jiné 105 | zprávy 106 | nové 107 | není 108 | vás 109 | jen 110 | podle 111 | zde 112 | už 113 | být 114 | více 115 | bude 116 | již 117 | než 118 | který 119 | by 120 | které 121 | co 122 | nebo 123 | ten 124 | tak 125 | má 126 | při 127 | od 128 | po 129 | jsou 130 | jak 131 | další 132 | ale 133 | si 134 | se 135 | ve 136 | to 137 | jako 138 | za 139 | zpět 140 | ze 141 | do 142 | pro 143 | je 144 | na 145 | atd 146 | atp 147 | jakmile 148 | přičemž 149 | já 150 | on 151 | ona 152 | ono 153 | oni 154 | ony 155 | my 156 | vy 157 | jí 158 | ji 159 | mě 160 | mne 161 | jemu 162 | tomu 163 | těm 164 | těmu 165 | němu 166 | němuž 167 | jehož 168 | jíž 169 | jelikož 170 | jež 171 | jakož 172 | načež 173 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_el.txt: -------------------------------------------------------------------------------- 1 | # Lucene Greek Stopwords list 2 | # Note: by default this file is used after GreekLowerCaseFilter, 3 | # so when modifying this file use 'σ' instead of 'ς' 4 | ο 5 | η 6 | το 7 | οι 8 | τα 9 | του 10 | τησ 11 | των 12 | τον 13 | την 14 | και 15 | κι 16 | κ 17 | ειμαι 18 | εισαι 19 | ειναι 20 | ειμαστε 21 | ειστε 22 | στο 23 | στον 24 | στη 25 | στην 26 | μα 27 | αλλα 28 | απο 29 | για 30 | προσ 31 | με 32 | σε 33 | ωσ 34 | παρα 35 | αντι 36 | κατα 37 | μετα 38 | θα 39 | να 40 | δε 41 | δεν 42 | μη 43 | μην 44 | επι 45 | ενω 46 | εαν 47 | αν 48 | τοτε 49 | που 50 | πωσ 51 | ποιοσ 52 | ποια 53 | ποιο 54 | ποιοι 55 | ποιεσ 56 | ποιων 57 | ποιουσ 58 | αυτοσ 59 | αυτη 60 | αυτο 61 | αυτοι 62 | αυτων 63 | αυτουσ 64 | αυτεσ 65 | αυτα 66 | εκεινοσ 67 | εκεινη 68 | εκεινο 69 | εκεινοι 70 | εκεινεσ 71 | εκεινα 72 | εκεινων 73 | εκεινουσ 74 | οπωσ 75 | ομωσ 76 | ισωσ 77 | οσο 78 | οτι 79 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_en.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # a couple of test stopwords to test that the words are really being 17 | # configured from this file: 18 | stopworda 19 | stopwordb 20 | 21 | # Standard english stop words taken from Lucene's StopAnalyzer 22 | a 23 | an 24 | and 25 | are 26 | as 27 | at 28 | be 29 | but 30 | by 31 | for 32 | if 33 | in 34 | into 35 | is 36 | it 37 | no 38 | not 39 | of 40 | on 41 | or 42 | such 43 | that 44 | the 45 | their 46 | then 47 | there 48 | these 49 | they 50 | this 51 | to 52 | was 53 | will 54 | with 55 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_eu.txt: -------------------------------------------------------------------------------- 1 | # example set of basque stopwords 2 | al 3 | anitz 4 | arabera 5 | asko 6 | baina 7 | bat 8 | batean 9 | batek 10 | bati 11 | batzuei 12 | batzuek 13 | batzuetan 14 | batzuk 15 | bera 16 | beraiek 17 | berau 18 | berauek 19 | bere 20 | berori 21 | beroriek 22 | beste 23 | bezala 24 | da 25 | dago 26 | dira 27 | ditu 28 | du 29 | dute 30 | edo 31 | egin 32 | ere 33 | eta 34 | eurak 35 | ez 36 | gainera 37 | gu 38 | gutxi 39 | guzti 40 | haiei 41 | haiek 42 | haietan 43 | hainbeste 44 | hala 45 | han 46 | handik 47 | hango 48 | hara 49 | hari 50 | hark 51 | hartan 52 | hau 53 | hauei 54 | hauek 55 | hauetan 56 | hemen 57 | hemendik 58 | hemengo 59 | hi 60 | hona 61 | honek 62 | honela 63 | honetan 64 | honi 65 | hor 66 | hori 67 | horiei 68 | horiek 69 | horietan 70 | horko 71 | horra 72 | horrek 73 | horrela 74 | horretan 75 | horri 76 | hortik 77 | hura 78 | izan 79 | ni 80 | noiz 81 | nola 82 | non 83 | nondik 84 | nongo 85 | nor 86 | nora 87 | ze 88 | zein 89 | zen 90 | zenbait 91 | zenbat 92 | zer 93 | zergatik 94 | ziren 95 | zituen 96 | zu 97 | zuek 98 | zuen 99 | zuten 100 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_ga.txt: -------------------------------------------------------------------------------- 1 | 2 | a 3 | ach 4 | ag 5 | agus 6 | an 7 | aon 8 | ar 9 | arna 10 | as 11 | b' 12 | ba 13 | beirt 14 | bhúr 15 | caoga 16 | ceathair 17 | ceathrar 18 | chomh 19 | chtó 20 | chuig 21 | chun 22 | cois 23 | céad 24 | cúig 25 | cúigear 26 | d' 27 | daichead 28 | dar 29 | de 30 | deich 31 | deichniúr 32 | den 33 | dhá 34 | do 35 | don 36 | dtí 37 | dá 38 | dár 39 | dó 40 | faoi 41 | faoin 42 | faoina 43 | faoinár 44 | fara 45 | fiche 46 | gach 47 | gan 48 | go 49 | gur 50 | haon 51 | hocht 52 | i 53 | iad 54 | idir 55 | in 56 | ina 57 | ins 58 | inár 59 | is 60 | le 61 | leis 62 | lena 63 | lenár 64 | m' 65 | mar 66 | mo 67 | mé 68 | na 69 | nach 70 | naoi 71 | naonúr 72 | ná 73 | ní 74 | níor 75 | nó 76 | nócha 77 | ocht 78 | ochtar 79 | os 80 | roimh 81 | sa 82 | seacht 83 | seachtar 84 | seachtó 85 | seasca 86 | seisear 87 | siad 88 | sibh 89 | sinn 90 | sna 91 | sé 92 | sí 93 | tar 94 | thar 95 | thú 96 | triúr 97 | trí 98 | trína 99 | trínár 100 | tríocha 101 | tú 102 | um 103 | ár 104 | é 105 | éis 106 | í 107 | ó 108 | ón 109 | óna 110 | ónár 111 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_gl.txt: -------------------------------------------------------------------------------- 1 | # galican stopwords 2 | a 3 | aínda 4 | alí 5 | aquel 6 | aquela 7 | aquelas 8 | aqueles 9 | aquilo 10 | aquí 11 | ao 12 | aos 13 | as 14 | así 15 | á 16 | ben 17 | cando 18 | che 19 | co 20 | coa 21 | comigo 22 | con 23 | connosco 24 | contigo 25 | convosco 26 | coas 27 | cos 28 | cun 29 | cuns 30 | cunha 31 | cunhas 32 | da 33 | dalgunha 34 | dalgunhas 35 | dalgún 36 | dalgúns 37 | das 38 | de 39 | del 40 | dela 41 | delas 42 | deles 43 | desde 44 | deste 45 | do 46 | dos 47 | dun 48 | duns 49 | dunha 50 | dunhas 51 | e 52 | el 53 | ela 54 | elas 55 | eles 56 | en 57 | era 58 | eran 59 | esa 60 | esas 61 | ese 62 | eses 63 | esta 64 | estar 65 | estaba 66 | está 67 | están 68 | este 69 | estes 70 | estiven 71 | estou 72 | eu 73 | é 74 | facer 75 | foi 76 | foron 77 | fun 78 | había 79 | hai 80 | iso 81 | isto 82 | la 83 | las 84 | lle 85 | lles 86 | lo 87 | los 88 | mais 89 | me 90 | meu 91 | meus 92 | min 93 | miña 94 | miñas 95 | moi 96 | na 97 | nas 98 | neste 99 | nin 100 | no 101 | non 102 | nos 103 | nosa 104 | nosas 105 | noso 106 | nosos 107 | nós 108 | nun 109 | nunha 110 | nuns 111 | nunhas 112 | o 113 | os 114 | ou 115 | ó 116 | ós 117 | para 118 | pero 119 | pode 120 | pois 121 | pola 122 | polas 123 | polo 124 | polos 125 | por 126 | que 127 | se 128 | senón 129 | ser 130 | seu 131 | seus 132 | sexa 133 | sido 134 | sobre 135 | súa 136 | súas 137 | tamén 138 | tan 139 | te 140 | ten 141 | teñen 142 | teño 143 | ter 144 | teu 145 | teus 146 | ti 147 | tido 148 | tiña 149 | tiven 150 | túa 151 | túas 152 | un 153 | unha 154 | unhas 155 | uns 156 | vos 157 | vosa 158 | vosas 159 | voso 160 | vosos 161 | vós 162 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_hy.txt: -------------------------------------------------------------------------------- 1 | # example set of Armenian stopwords. 2 | այդ 3 | այլ 4 | այն 5 | այս 6 | դու 7 | դուք 8 | եմ 9 | են 10 | ենք 11 | ես 12 | եք 13 | է 14 | էի 15 | էին 16 | էինք 17 | էիր 18 | էիք 19 | էր 20 | ըստ 21 | թ 22 | ի 23 | ին 24 | իսկ 25 | իր 26 | կամ 27 | համար 28 | հետ 29 | հետո 30 | մենք 31 | մեջ 32 | մի 33 | ն 34 | նա 35 | նաև 36 | նրա 37 | նրանք 38 | որ 39 | որը 40 | որոնք 41 | որպես 42 | ու 43 | ում 44 | պիտի 45 | վրա 46 | և 47 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_ja.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file defines a stopword set for Japanese. 3 | # 4 | # This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia. 5 | # Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745 6 | # for frequency lists, etc. that can be useful for making your own set (if desired) 7 | # 8 | # Note that there is an overlap between these stopwords and the terms stopped when used 9 | # in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note 10 | # that comments are not allowed on the same line as stopwords. 11 | # 12 | # Also note that stopping is done in a case-insensitive manner. Change your StopFilter 13 | # configuration if you need case-sensitive stopping. Lastly, note that stopping is done 14 | # using the same character width as the entries in this file. Since this StopFilter is 15 | # normally done after a CJKWidthFilter in your chain, you would usually want your romaji 16 | # entries to be in half-width and your kana entries to be in full-width. 17 | # 18 | の 19 | に 20 | は 21 | を 22 | た 23 | が 24 | で 25 | て 26 | と 27 | し 28 | れ 29 | さ 30 | ある 31 | いる 32 | も 33 | する 34 | から 35 | な 36 | こと 37 | として 38 | い 39 | や 40 | れる 41 | など 42 | なっ 43 | ない 44 | この 45 | ため 46 | その 47 | あっ 48 | よう 49 | また 50 | もの 51 | という 52 | あり 53 | まで 54 | られ 55 | なる 56 | へ 57 | か 58 | だ 59 | これ 60 | によって 61 | により 62 | おり 63 | より 64 | による 65 | ず 66 | なり 67 | られる 68 | において 69 | ば 70 | なかっ 71 | なく 72 | しかし 73 | について 74 | せ 75 | だっ 76 | その後 77 | できる 78 | それ 79 | う 80 | ので 81 | なお 82 | のみ 83 | でき 84 | き 85 | つ 86 | における 87 | および 88 | いう 89 | さらに 90 | でも 91 | ら 92 | たり 93 | その他 94 | に関する 95 | たち 96 | ます 97 | ん 98 | なら 99 | に対して 100 | 特に 101 | せる 102 | 及び 103 | これら 104 | とき 105 | では 106 | にて 107 | ほか 108 | ながら 109 | うち 110 | そして 111 | とともに 112 | ただし 113 | かつて 114 | それぞれ 115 | または 116 | お 117 | ほど 118 | ものの 119 | に対する 120 | ほとんど 121 | と共に 122 | といった 123 | です 124 | とも 125 | ところ 126 | ここ 127 | ##### End of file 128 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_lv.txt: -------------------------------------------------------------------------------- 1 | # Set of Latvian stopwords from A Stemming Algorithm for Latvian, Karlis Kreslins 2 | # the original list of over 800 forms was refined: 3 | # pronouns, adverbs, interjections were removed 4 | # 5 | # prepositions 6 | aiz 7 | ap 8 | ar 9 | apakš 10 | ārpus 11 | augšpus 12 | bez 13 | caur 14 | dēļ 15 | gar 16 | iekš 17 | iz 18 | kopš 19 | labad 20 | lejpus 21 | līdz 22 | no 23 | otrpus 24 | pa 25 | par 26 | pār 27 | pēc 28 | pie 29 | pirms 30 | pret 31 | priekš 32 | starp 33 | šaipus 34 | uz 35 | viņpus 36 | virs 37 | virspus 38 | zem 39 | apakšpus 40 | # Conjunctions 41 | un 42 | bet 43 | jo 44 | ja 45 | ka 46 | lai 47 | tomēr 48 | tikko 49 | turpretī 50 | arī 51 | kaut 52 | gan 53 | tādēļ 54 | tā 55 | ne 56 | tikvien 57 | vien 58 | kā 59 | ir 60 | te 61 | vai 62 | kamēr 63 | # Particles 64 | ar 65 | diezin 66 | droši 67 | diemžēl 68 | nebūt 69 | ik 70 | it 71 | taču 72 | nu 73 | pat 74 | tiklab 75 | iekšpus 76 | nedz 77 | tik 78 | nevis 79 | turpretim 80 | jeb 81 | iekam 82 | iekām 83 | iekāms 84 | kolīdz 85 | līdzko 86 | tiklīdz 87 | jebšu 88 | tālab 89 | tāpēc 90 | nekā 91 | itin 92 | jā 93 | jau 94 | jel 95 | nē 96 | nezin 97 | tad 98 | tikai 99 | vis 100 | tak 101 | iekams 102 | vien 103 | # modal verbs 104 | būt 105 | biju 106 | biji 107 | bija 108 | bijām 109 | bijāt 110 | esmu 111 | esi 112 | esam 113 | esat 114 | būšu 115 | būsi 116 | būs 117 | būsim 118 | būsiet 119 | tikt 120 | tiku 121 | tiki 122 | tika 123 | tikām 124 | tikāt 125 | tieku 126 | tiec 127 | tiek 128 | tiekam 129 | tiekat 130 | tikšu 131 | tiks 132 | tiksim 133 | tiksiet 134 | tapt 135 | tapi 136 | tapāt 137 | topat 138 | tapšu 139 | tapsi 140 | taps 141 | tapsim 142 | tapsiet 143 | kļūt 144 | kļuvu 145 | kļuvi 146 | kļuva 147 | kļuvām 148 | kļuvāt 149 | kļūstu 150 | kļūsti 151 | kļūst 152 | kļūstam 153 | kļūstat 154 | kļūšu 155 | kļūsi 156 | kļūs 157 | kļūsim 158 | kļūsiet 159 | # verbs 160 | varēt 161 | varēju 162 | varējām 163 | varēšu 164 | varēsim 165 | var 166 | varēji 167 | varējāt 168 | varēsi 169 | varēsiet 170 | varat 171 | varēja 172 | varēs 173 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/stopwords_th.txt: -------------------------------------------------------------------------------- 1 | # Thai stopwords from: 2 | # "Opinion Detection in Thai Political News Columns 3 | # Based on Subjectivity Analysis" 4 | # Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak 5 | ไว้ 6 | ไม่ 7 | ไป 8 | ได้ 9 | ให้ 10 | ใน 11 | โดย 12 | แห่ง 13 | แล้ว 14 | และ 15 | แรก 16 | แบบ 17 | แต่ 18 | เอง 19 | เห็น 20 | เลย 21 | เริ่ม 22 | เรา 23 | เมื่อ 24 | เพื่อ 25 | เพราะ 26 | เป็นการ 27 | เป็น 28 | เปิดเผย 29 | เปิด 30 | เนื่องจาก 31 | เดียวกัน 32 | เดียว 33 | เช่น 34 | เฉพาะ 35 | เคย 36 | เข้า 37 | เขา 38 | อีก 39 | อาจ 40 | อะไร 41 | ออก 42 | อย่าง 43 | อยู่ 44 | อยาก 45 | หาก 46 | หลาย 47 | หลังจาก 48 | หลัง 49 | หรือ 50 | หนึ่ง 51 | ส่วน 52 | ส่ง 53 | สุด 54 | สําหรับ 55 | ว่า 56 | วัน 57 | ลง 58 | ร่วม 59 | ราย 60 | รับ 61 | ระหว่าง 62 | รวม 63 | ยัง 64 | มี 65 | มาก 66 | มา 67 | พร้อม 68 | พบ 69 | ผ่าน 70 | ผล 71 | บาง 72 | น่า 73 | นี้ 74 | นํา 75 | นั้น 76 | นัก 77 | นอกจาก 78 | ทุก 79 | ที่สุด 80 | ที่ 81 | ทําให้ 82 | ทํา 83 | ทาง 84 | ทั้งนี้ 85 | ทั้ง 86 | ถ้า 87 | ถูก 88 | ถึง 89 | ต้อง 90 | ต่างๆ 91 | ต่าง 92 | ต่อ 93 | ตาม 94 | ตั้งแต่ 95 | ตั้ง 96 | ด้าน 97 | ด้วย 98 | ดัง 99 | ซึ่ง 100 | ช่วง 101 | จึง 102 | จาก 103 | จัด 104 | จะ 105 | คือ 106 | ความ 107 | ครั้ง 108 | คง 109 | ขึ้น 110 | ของ 111 | ขอ 112 | ขณะ 113 | ก่อน 114 | ก็ 115 | การ 116 | กับ 117 | กัน 118 | กว่า 119 | กล่าว 120 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/lang/userdict_ja.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This is a sample user dictionary for Kuromoji (JapaneseTokenizer) 3 | # 4 | # Add entries to this file in order to override the statistical model in terms 5 | # of segmentation, readings and part-of-speech tags. Notice that entries do 6 | # not have weights since they are always used when found. This is by-design 7 | # in order to maximize ease-of-use. 8 | # 9 | # Entries are defined using the following CSV format: 10 | # , ... , ... , 11 | # 12 | # Notice that a single half-width space separates tokens and readings, and 13 | # that the number tokens and readings must match exactly. 14 | # 15 | # Also notice that multiple entries with the same is undefined. 16 | # 17 | # Whitespace only lines are ignored. Comments are not allowed on entry lines. 18 | # 19 | 20 | # Custom segmentation for kanji compounds 21 | 日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞 22 | 関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,カスタム名詞 23 | 24 | # Custom segmentation for compound katakana 25 | トートバッグ,トート バッグ,トート バッグ,かずカナ名詞 26 | ショルダーバッグ,ショルダー バッグ,ショルダー バッグ,かずカナ名詞 27 | 28 | # Custom reading for former sumo wrestler 29 | 朝青龍,朝青龍,アサショウリュウ,カスタム人名 30 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/params.json: -------------------------------------------------------------------------------- 1 | {"params":{ 2 | "query":{ 3 | "defType":"edismax", 4 | "q.alt":"*:*", 5 | "rows":"10", 6 | "fl":"*,score", 7 | "":{"v":0} 8 | }, 9 | "facets":{ 10 | "facet":"on", 11 | "facet.mincount": "1", 12 | "":{"v":0} 13 | }, 14 | "velocity":{ 15 | "wt": "velocity", 16 | "v.template":"browse", 17 | "v.layout": "layout", 18 | "":{"v":0} 19 | } 20 | }} -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/protwords.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | # Use a protected word file to protect against the stemmer reducing two 15 | # unrelated words to the same base word. 16 | 17 | # Some non-words that normally won't be encountered, 18 | # just to test that they won't be stemmed. 19 | dontstems 20 | zwhacky 21 | 22 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/stopwords.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/conf/synonyms.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | #some test synonym mappings unlikely to appear in real input text 15 | aaafoo => aaabar 16 | bbbfoo => bbbfoo bbbbar 17 | cccfoo => cccbar cccbaz 18 | fooaaa,baraaa,bazaaa 19 | 20 | # Some synonym groups specific to this example 21 | GB,gib,gigabyte,gigabytes 22 | MB,mib,megabyte,megabytes 23 | Television, Televisions, TV, TVs 24 | #notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming 25 | #after us won't split it into two words. 26 | 27 | # Synonym mappings can be used for spelling correction too 28 | pixima => pixma 29 | 30 | -------------------------------------------------------------------------------- /data-access/config/schemas/solr/nexustiles/core.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. -------------------------------------------------------------------------------- /data-access/nexustiles/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /data-access/nexustiles/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /data-access/nexustiles/backends/nexusproto/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /data-access/nexustiles/backends/nexusproto/config/datastores.ini.default: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [cassandra] 17 | host=localhost 18 | port=9042 19 | keyspace=nexustiles 20 | local_datacenter=datacenter1 21 | protocol_version=4 22 | dc_policy=DCAwareRoundRobinPolicy 23 | username= 24 | password= 25 | 26 | [s3] 27 | bucket=nexus-jpl 28 | region=us-west-2 29 | 30 | [dynamo] 31 | table=nexus-jpl-table 32 | region=us-west-2 33 | 34 | [solr] 35 | host=http://localhost:8983 36 | core=nexustiles 37 | 38 | [datastore] 39 | store=cassandra 40 | -------------------------------------------------------------------------------- /data-access/nexustiles/backends/nexusproto/dao/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /data-access/nexustiles/backends/zarr/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /data-access/nexustiles/config/datasets.ini.default: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [solr] 17 | host=http://localhost:8983 18 | core=nexusdatasets 19 | -------------------------------------------------------------------------------- /data-access/nexustiles/exception.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | class NexusTileServiceException(Exception): 17 | def __init__(self, reason): 18 | Exception.__init__(self, reason) 19 | -------------------------------------------------------------------------------- /data-access/nexustiles/model/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /data-access/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /data-access/tests/config/datastores.ini: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | [cassandra] 17 | host=127.0.0.1 18 | keyspace=nexustiles 19 | local_datacenter=datacenter1 20 | protocol_version=3 21 | 22 | [solr] 23 | host=localhost:8983 24 | core=nexustiles -------------------------------------------------------------------------------- /data-access/tests/sizefromcass.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import pyximport 18 | pyximport.install() 19 | 20 | import configparser 21 | 22 | import pkg_resources 23 | 24 | from nexustiles.dao.CassandraProxy import CassandraProxy 25 | 26 | config = configparser.RawConfigParser() 27 | 28 | config.readfp(pkg_resources.resource_stream(__name__, "config/datastores.ini"), filename='datastores.ini') 29 | 30 | cass = CassandraProxy(config) 31 | 32 | tiles = cass.fetch_nexus_tiles('d9b5afe3-bd7f-3824-ad8a-d8d3b364689c') 33 | print(len(tiles[0].tile_blob)) 34 | -------------------------------------------------------------------------------- /data-access/tests/solr_update.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import solr 18 | 19 | solrcon = solr.Solr('http://%s/solr/%s' % ('localhost:8983', 'nexustiles')) 20 | 21 | ds = 'MXLDEPTH_ECCO_version4_release1' 22 | 23 | # print solrcon.select(q='dataset_s:%s' % ds, sort='id', cursorMark='*').results 24 | 25 | 26 | params = {'q': 'dataset_s:%s' % ds, 'sort': 'id', 'cursorMark': '*', 'rows': 5000} 27 | done = False 28 | while not done: 29 | response = solrcon.select(**params) 30 | print(len(response.results)) 31 | if params['cursorMark'] == response.nextCursorMark: 32 | done = True 33 | 34 | params['cursorMark'] = response.nextCursorMark 35 | -------------------------------------------------------------------------------- /docker/jupyter/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | FROM jupyter/scipy-notebook 16 | 17 | MAINTAINER Apache SDAP "dev@sdap.apache.org" 18 | 19 | USER root 20 | RUN apt-get update && \ 21 | apt-get install -y git libgeos-dev 22 | USER jovyan 23 | 24 | COPY requirements.txt /tmp 25 | RUN pip install -r /tmp/requirements.txt && \ 26 | conda install -y basemap 27 | 28 | ENV CHOWN_HOME_OPTS='-R' 29 | ENV REBUILD_CODE=true 30 | 31 | ARG APACHE_NEXUS=https://github.com/apache/incubator-sdap-nexus.git 32 | ARG APACHE_NEXUS_BRANCH=master 33 | 34 | RUN mkdir -p /home/jovyan/Quickstart && \ 35 | mkdir -p /home/jovyan/nexuscli && \ 36 | cd /home/jovyan/nexuscli && \ 37 | git init && \ 38 | git remote add -f origin ${APACHE_NEXUS} && \ 39 | git config core.sparseCheckout true && \ 40 | echo "integrations/python-client" >> .git/info/sparse-checkout && \ 41 | git pull origin ${APACHE_NEXUS_BRANCH} && \ 42 | cd integrations/python-client && \ 43 | python setup.py install 44 | 45 | COPY ["Time Series Example.ipynb", "/home/jovyan/Quickstart/Time Series Example.ipynb"] -------------------------------------------------------------------------------- /docker/jupyter/requirements.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | shapely==1.6.4.post2 17 | requests==2.21.0 18 | numpy>=1.13.3 19 | -------------------------------------------------------------------------------- /docker/nexus-webapp/Readme.md: -------------------------------------------------------------------------------- 1 | 2 | # Build the image 3 | 4 | To build the nexus-webapp docker image, `cd` into the root directory of this project (`incubator-sdap-nexus/`) and run: 5 | 6 | docker build . -f docker/nexus-webapp/Dockerfile -t nexusjpl/nexus-webapp:distributed.${BUILD_VERSION} 7 | 8 | where `${BUILD_VERSION}` is the build version of SDAP. 9 | 10 | 11 | # Push the image 12 | 13 | Push the images to the `nexusjpl` organization on DockerHub 14 | 15 | docker push nexusjpl/nexus-webapp:distributed.${BUILD_VERSION} 16 | 17 | # Add a new tag to the Git repo 18 | 19 | When you push a new image that is intended for more than just private use, you should also `git tag` the commit from which 20 | the image was built. The tag name should be the docker image tag of the new image you pushed, i.e.: 21 | 22 | git tag distributed.0.3.0 23 | git push --tags 24 | 25 | You can see a list of all SDAP version tags on the [SDAP Git repo tags page](https://github.com/apache/incubator-sdap-nexus/tags). -------------------------------------------------------------------------------- /docker/nexus-webapp/install_nexusproto.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | if [ ! -z ${BUILD_NEXUSPROTO+x} ]; then 20 | echo 'Building nexusproto from source...' 21 | 22 | APACHE_NEXUSPROTO="https://github.com/apache/incubator-sdap-nexusproto.git" 23 | MASTER="master" 24 | 25 | GIT_REPO=${1:-$APACHE_NEXUSPROTO} 26 | GIT_BRANCH=${2:-$MASTER} 27 | 28 | mkdir nexusproto 29 | pushd nexusproto 30 | git init 31 | git pull ${GIT_REPO} ${GIT_BRANCH} 32 | 33 | ./gradlew pythonInstall --info 34 | 35 | ./gradlew install --info 36 | 37 | rm -rf /root/.gradle 38 | popd 39 | rm -rf nexusproto 40 | elif [ ! -z ${POETRY_BUILD+x} ]; then 41 | poetry add nexusproto 42 | echo "Deleting poetry cache" 43 | rm -rf $POETRY_CACHE_DIR 44 | else 45 | pip install nexusproto 46 | fi 47 | -------------------------------------------------------------------------------- /docker/nexus-webapp/install_python.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | echo "Installing build dependencies" 20 | 21 | apt-get update 22 | apt-get upgrade -y 23 | apt-get install --no-install-recommends -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev liblzma-dev tk-dev libffi-dev 24 | apt-get clean 25 | rm -rf /var/lib/apt/lists/* 26 | 27 | echo "Downloading & extracting source tarball" 28 | 29 | cd /tmp/ 30 | wget https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz 31 | tar xzf Python-3.9.7.tgz 32 | cd Python-3.9.7 33 | 34 | echo "Running install" 35 | 36 | ./configure --prefix=/opt/python/3.9.7/ --enable-optimizations --with-lto --with-computed-gotos --with-system-ffi 37 | make -j "$(nproc)" 38 | make altinstall 39 | 40 | echo "Cleaning up install dir" 41 | 42 | rm /tmp/Python-3.9.7.tgz 43 | cd /tmp/ 44 | rm -rf Python-3.9.7 45 | 46 | echo "Final steps..." 47 | 48 | /opt/python/3.9.7/bin/python3.9 -m pip install --upgrade pip setuptools wheel 49 | -------------------------------------------------------------------------------- /docker/solr/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | FROM bitnami/solr:8.11.1-debian-10-r0 16 | ADD https://repo1.maven.org/maven2/org/locationtech/jts/jts-core/1.15.0/jts-core-1.15.0.jar /opt/bitnami/solr/server/solr-webapp/webapp/WEB-INF/lib/jts-core-1.15.0.jar 17 | USER root 18 | RUN chmod a+r /opt/bitnami/solr/server/solr-webapp/webapp/WEB-INF/lib/jts-core-1.15.0.jar 19 | USER 1001 20 | -------------------------------------------------------------------------------- /docker/solr/Readme.rst: -------------------------------------------------------------------------------- 1 | .. _solr_images: 2 | 3 | Solr Images 4 | ===================== 5 | 6 | All docker builds for the Solr images should happen from this directory. For copy/paste ability, first export the environment variable ``BUILD_VERSION`` to the version number you would like to tag images as. 7 | 8 | Common Environment Variables 9 | ------------------------------ 10 | 11 | Any environment variable that can be passed to `solr.in.sh `_ and be passed as an environment variable to the docker container and it will be utilized. A few options are called out here: 12 | 13 | ``SOLR_HEAP`` 14 | *default: 512m* 15 | 16 | Increase Java Heap as needed to support your indexing / query needs 17 | 18 | ``SOLR_HOME`` 19 | *default /opt/solr/server/solr* 20 | 21 | Path to a directory for Solr to store cores and their data. This directory is exposed as a ``VOLUME`` that can be mounted. 22 | 23 | If you want to mount the ``SOLR_HOME`` directory to a directory on the host machine, you need to provide the container path to the docker run ``-v`` option. Doing this allows you to retain the index between start/stop of this container. 24 | 25 | sdap/solr 26 | --------- 27 | 28 | This is the base image used by both singlenode and cloud versions of the Solr image. 29 | 30 | How To Build 31 | ^^^^^^^^^^^^ 32 | 33 | This image can be built by: 34 | 35 | .. code-block:: bash 36 | 37 | docker build -t nexusjpl/solr:${BUILD_VERSION} . 38 | 39 | How to Run 40 | ^^^^^^^^^^ 41 | 42 | This image is not intended to be run directly. 43 | 44 | .. include:: ../docker/solr/singlenode/Readme.rst 45 | 46 | .. include:: ../docker/solr/cloud/Readme.rst 47 | 48 | .. include:: ../docker/solr/cloud-init/Readme.rst 49 | -------------------------------------------------------------------------------- /docker/solr/cloud-init/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | FROM python:3.11 18 | MAINTAINER Apache SDAP "dev@sdap.apache.org" 19 | 20 | ENV MINIMUM_NODES="1" \ 21 | SDAP_ZK_SOLR="localhost:2181/solr" \ 22 | SDAP_SOLR_URL="http://localhost:8983/solr/" \ 23 | ZK_LOCK_GUID="c4d193b1-7e47-4b32-a169-a596463da0f5" \ 24 | MAX_RETRIES="30" \ 25 | CREATE_COLLECTION_PARAMS="name=nexustiles&collection.configName=nexustiles&numShards=1" 26 | 27 | 28 | RUN pip install kazoo==2.6.0 'requests>=2.26.0' 29 | COPY ./cloud-init/create-collection.py /tmp/create-collection.py 30 | 31 | ENTRYPOINT ["/tmp/create-collection.py"] 32 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | _static 3 | _templates 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Minimal makefile for Sphinx documentation 17 | # 18 | 19 | # You can set these variables from the command line. 20 | SPHINXOPTS = 21 | SPHINXBUILD = sphinx-build 22 | SPHINXPROJ = sdap-nexus 23 | SOURCEDIR = . 24 | BUILDDIR = _build 25 | 26 | # Put it first so that "make" without argument is like "make help". 27 | help: 28 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 29 | 30 | .PHONY: help Makefile 31 | 32 | # Catch-all target: route all unknown targets to Sphinx using the new 33 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 34 | %: Makefile 35 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/dockerimages.rst: -------------------------------------------------------------------------------- 1 | .. _dockerimages: 2 | 3 | ************* 4 | Docker Images 5 | ************* 6 | 7 | incubator-sdap-nexus contains a number of different Docker images for download. All images are available from the `SDAP organization `_ on DockerHub. 8 | 9 | 10 | .. include:: ../docker/solr/Readme.rst 11 | -------------------------------------------------------------------------------- /docs/images/Jupyter_Home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sdap-nexus/e30109653ffaba0bf64c7814f96a5d9cd581be0f/docs/images/Jupyter_Home.png -------------------------------------------------------------------------------- /docs/images/Jupyter_Quickstart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sdap-nexus/e30109653ffaba0bf64c7814f96a5d9cd581be0f/docs/images/Jupyter_Quickstart.png -------------------------------------------------------------------------------- /docs/images/nexus-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sdap-nexus/e30109653ffaba0bf64c7814f96a5d9cd581be0f/docs/images/nexus-summary.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to the Apache SDAP project documentation! 2 | ================================================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | intro 9 | collections 10 | quickstart 11 | build 12 | dockerimages 13 | release 14 | test 15 | 16 | 17 | Check out the :ref:`quickstart guide`. 18 | -------------------------------------------------------------------------------- /docs/intro.rst: -------------------------------------------------------------------------------- 1 | .. _intro: 2 | 3 | *********** 4 | About NEXUS 5 | *********** 6 | 7 | NEXUS is a data-intensive analysis solution using a new approach for handling science data to enable large-scale data analysis. 8 | 9 | It supports a number of algorithms out of the box: 10 | 11 | * Time Series 12 | * Latitude/Time Hovmöller 13 | * Longitude/Time Hovmöller 14 | * Latitude/Longitude Time Average 15 | * Area Averaged Time Series 16 | * Time Averaged Map 17 | * Climatological Map 18 | * Correlation Map 19 | * Daily Difference Average 20 | 21 | .. image:: images/nexus-summary.png 22 | 23 | Check out the :ref:`quickstart`. 24 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=sdap-nexus 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx-copybutton 2 | sphinx-rtd-theme 3 | -------------------------------------------------------------------------------- /helm/.gitignore: -------------------------------------------------------------------------------- 1 | *.lock 2 | *.tgz 3 | .DS_Store -------------------------------------------------------------------------------- /helm/.helmignore: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Patterns to ignore when building packages. 17 | # This supports shell glob matching, relative path matching, and 18 | # negation (prefixed with !). Only one pattern per line. 19 | .DS_Store 20 | # Common VCS dirs 21 | .git/ 22 | .gitignore 23 | .bzr/ 24 | .bzrignore 25 | .hg/ 26 | .hgignore 27 | .svn/ 28 | # Common backup files 29 | *.swp 30 | *.bak 31 | *.tmp 32 | *~ 33 | # Various IDEs 34 | .project 35 | .idea/ 36 | *.tmproj 37 | .vscode/ 38 | -------------------------------------------------------------------------------- /helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | apiVersion: v1 17 | appVersion: "0.3.0" 18 | description: Science Data Analytics Platform 19 | name: nexus 20 | version: 1.4.0 21 | -------------------------------------------------------------------------------- /helm/requirements.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | dependencies: 17 | - name: ingress-nginx 18 | version: 4.12.0 19 | repository: https://kubernetes.github.io/ingress-nginx 20 | condition: ingress-nginx.enabled 21 | - name: rabbitmq 22 | version: 8.24.13 23 | repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami 24 | condition: rabbitmq.enabled 25 | - name: solr 26 | version: 2.1.7 27 | repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami 28 | condition: solr.enabled 29 | - name: cassandra 30 | version: 9.1.7 31 | repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami 32 | condition: cassandra.enabled 33 | 34 | -------------------------------------------------------------------------------- /helm/templates/collections-config-gitcfg.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if .Values.ingestion.enabled }} 17 | {{ if not .Values.ingestion.collections.configMap }} 18 | apiVersion: sdap.apache.org/v1 19 | kind: GitBasedConfig 20 | metadata: 21 | name: collections-config-gitcfg 22 | spec: 23 | {{ if .Values.ingestion.collections.git }} 24 | git-url: {{ .Values.ingestion.collections.git.url }} 25 | git-branch: {{ .Values.ingestion.collections.git.branch }} 26 | git-token: {{ .Values.ingestion.collections.token }} 27 | {{ else }} 28 | local-dir: {{ .Values.ingestion.collections.localDir }} 29 | {{ end }} 30 | config-map: {{ include "nexus.collectionsConfig.configmapName" . }} 31 | {{ end }} 32 | {{ end }} -------------------------------------------------------------------------------- /helm/templates/config-operator-rbac.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if .Values.ingestion.enabled }} 17 | {{ if not .Values.ingestion.collections.configMap }} 18 | apiVersion: v1 19 | kind: ServiceAccount 20 | metadata: 21 | name: config-operator 22 | 23 | --- 24 | 25 | apiVersion: rbac.authorization.k8s.io/v1 26 | kind: ClusterRoleBinding 27 | metadata: 28 | name: config-operator-role-binding 29 | roleRef: 30 | apiGroup: rbac.authorization.k8s.io 31 | kind: ClusterRole 32 | name: cluster-admin 33 | subjects: 34 | - kind: ServiceAccount 35 | name: config-operator 36 | namespace: {{ .Release.Namespace }} 37 | {{ end }} 38 | {{ end }} 39 | 40 | -------------------------------------------------------------------------------- /helm/templates/config-operator.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if .Values.ingestion.enabled }} 17 | {{ if not .Values.ingestion.collections.configMap }} 18 | apiVersion: apps/v1 19 | kind: Deployment 20 | metadata: 21 | name: config-operator 22 | labels: 23 | app: config-operator 24 | spec: 25 | replicas: 1 26 | selector: 27 | matchLabels: 28 | app: config-operator 29 | template: 30 | metadata: 31 | labels: 32 | app: config-operator 33 | spec: 34 | serviceAccountName: config-operator 35 | containers: 36 | - name: config-operator 37 | image: {{ .Values.ingestion.configOperator.image }} 38 | imagePullPolicy: Always 39 | {{ end }} 40 | {{ end }} 41 | -------------------------------------------------------------------------------- /helm/templates/domspurge.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if .Values.cronJobs.domsPurge.enabled }} 17 | 18 | apiVersion: batch/v1 19 | kind: CronJob 20 | metadata: 21 | name: doms-purge 22 | spec: 23 | schedule: {{ .Values.cronJobs.domsPurge.schedule }} 24 | concurrencyPolicy: {{ .Values.cronJobs.domsPurge.concurrency }} 25 | jobTemplate: 26 | spec: 27 | template: 28 | spec: 29 | containers: 30 | - name: doms-purge 31 | image: {{ .Values.cronJobs.domsPurge.image }} 32 | imagePullPolicy: Always 33 | env: 34 | - name: CASSANDRA_CONTACT_POINTS 35 | value: {{ include "nexus.urls.cassandra" . }} 36 | - name: CASSANDRA_USERNAME 37 | value: {{ include "nexus.credentials.cassandra.username" . }} 38 | - name: CASSANDRA_PASSWORD 39 | value: {{ include "nexus.credentials.cassandra.password" . }} 40 | - name: "YES" 41 | value: "true" 42 | - name: BEFORE_MONTHS 43 | value: {{ .Values.cronJobs.domsPurge.keepBeforeMonths }} 44 | {{ if .Values.cronJobs.domsPurge.keepUncompleted }} 45 | - name: KEEP_FAILED 46 | value: {{ .Values.cronJobs.domsPurge.keepUncompleted }} 47 | {{ end }} 48 | restartPolicy: {{ .Values.cronJobs.domsPurge.restartPolicy }} 49 | 50 | {{ end }} 51 | -------------------------------------------------------------------------------- /helm/templates/gitbasedconfig-crd.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if .Values.ingestion.collections.createCrd }} 17 | apiVersion: apiextensions.k8s.io/v1beta1 18 | kind: CustomResourceDefinition 19 | metadata: 20 | name: gitbasedconfigs.sdap.apache.org 21 | spec: 22 | group: sdap.apache.org 23 | versions: 24 | - name: v1 25 | served: true 26 | storage: true 27 | scope: Namespaced 28 | names: 29 | plural: gitbasedconfigs 30 | singular: gitbasedconfig 31 | kind: GitBasedConfig 32 | shortNames: 33 | - gitcfg 34 | validation: 35 | openAPIV3Schema: 36 | type: object 37 | properties: 38 | git-url: 39 | type: string 40 | git-branch: 41 | type: string 42 | git-token: 43 | type: string 44 | update-every-seconds: 45 | type: int 46 | local-dir: 47 | type: string 48 | config-map: 49 | type: string 50 | {{ end }} 51 | -------------------------------------------------------------------------------- /helm/templates/history-pvc.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{- if not .Values.ingestion.history.solrEnabled }} 17 | apiVersion: v1 18 | kind: PersistentVolumeClaim 19 | metadata: 20 | name: history-volume-claim 21 | annotations: 22 | helm.sh/resource-policy: "keep" 23 | spec: 24 | accessModes: 25 | - ReadWriteOnce 26 | resources: 27 | requests: 28 | storage: 1Gi 29 | storageClassName: {{ .Values.ingestion.history.storageClass }} 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /helm/templates/init-cassandra-configmap.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | apiVersion: v1 17 | data: 18 | init.cql: | 19 | CREATE KEYSPACE IF NOT EXISTS nexustiles WITH REPLICATION = { 'class': 'SimpleStrategy', 'replication_factor': 1 }; 20 | 21 | CREATE TABLE IF NOT EXISTS nexustiles.sea_surface_temp ( 22 | tile_id uuid PRIMARY KEY, 23 | tile_blob blob 24 | ); 25 | kind: ConfigMap 26 | metadata: 27 | name: init-cassandra 28 | namespace: {{ .Release.Namespace }} 29 | -------------------------------------------------------------------------------- /helm/templates/onearth.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if ne .Values.onEarthProxyIP "" }} 17 | 18 | kind: Service 19 | apiVersion: v1 20 | metadata: 21 | name: onearth 22 | spec: 23 | type: ExternalName 24 | externalName: {{ .Values.onEarthProxyIP }} 25 | ports: 26 | - port: 80 27 | 28 | {{ end}} -------------------------------------------------------------------------------- /helm/templates/root-webpage.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if .Values.rootWebpage.enabled }} 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: root-webpage 21 | spec: 22 | type: ClusterIP 23 | ports: 24 | - port: 8080 25 | name: http 26 | selector: 27 | app: root-webpage 28 | 29 | --- 30 | 31 | apiVersion: apps/v1 32 | kind: Deployment 33 | metadata: 34 | labels: 35 | app: root-webpage 36 | name: root-webpage 37 | spec: 38 | replicas: 1 39 | selector: 40 | matchLabels: 41 | app: root-webpage 42 | template: 43 | metadata: 44 | labels: 45 | app: root-webpage 46 | spec: 47 | containers: 48 | - image: busybox 49 | command: 50 | # Tiny HTTP server that simply returns 200 OK on port 8080 51 | - sh 52 | - -c 53 | - | 54 | while true; do 55 | { echo -e 'HTTP/1.1 200 OK\r\n'; } | nc -l -p 8080 56 | done 57 | name: root-webpage 58 | restartPolicy: Always 59 | {{ end }} -------------------------------------------------------------------------------- /helm/templates/solr-create-collection.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | {{ if .Values.solr.initPodEnabled }} 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: solr-create-collection 21 | spec: 22 | selector: 23 | matchLabels: 24 | app: solr-create-collection # has to match .spec.template.metadata.labels 25 | replicas: 1 26 | template: 27 | metadata: 28 | labels: 29 | app: solr-create-collection 30 | spec: 31 | containers: 32 | - name: solr-create-collection 33 | imagePullPolicy: Always 34 | image: {{ .Values.solr.initImage | default "apache/sdap-solr-cloud-init:1.4.0" }} 35 | resources: 36 | requests: 37 | memory: "0.5Gi" 38 | cpu: "0.25" 39 | env: 40 | - name: MINIMUM_NODES 41 | value: "{{ .Values.solr.replicaCount }}" 42 | - name: SDAP_SOLR_URL 43 | value: {{ include "nexus.urls.solr" . }}/solr/ 44 | - name: SDAP_ZK_SOLR 45 | value: {{ include "nexus.urls.zookeeper" . }}/solr 46 | - name: CREATE_COLLECTION_PARAMS 47 | value: "name=nexustiles&numShards=$(MINIMUM_NODES)&waitForFinalState=true" 48 | restartPolicy: Always 49 | {{ end }} -------------------------------------------------------------------------------- /helm/templates/spark-serviceaccount.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | apiVersion: v1 17 | kind: ServiceAccount 18 | metadata: 19 | {{ if .Values.webapp.iamRoleArn }} 20 | annotations: 21 | eks.amazonaws.com/role-arn: "{{ .Values.webapp.iamRoleArn }}" 22 | {{ end }} 23 | name: spark-serviceaccount 24 | automountServiceAccountToken: true 25 | --- 26 | apiVersion: rbac.authorization.k8s.io/v1 27 | kind: RoleBinding 28 | metadata: 29 | name: spark-serviceaccount-role-binding 30 | subjects: 31 | - kind: ServiceAccount 32 | name: spark-serviceaccount 33 | roleRef: 34 | kind: ClusterRole 35 | name: admin 36 | apiGroup: rbac.authorization.k8s.io -------------------------------------------------------------------------------- /integrations/esri/nexus_toolbox_embedded.tbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sdap-nexus/e30109653ffaba0bf64c7814f96a5d9cd581be0f/integrations/esri/nexus_toolbox_embedded.tbx -------------------------------------------------------------------------------- /integrations/esri/zipped_toolbox/nexus_toolbox.tbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sdap-nexus/e30109653ffaba0bf64c7814f96a5d9cd581be0f/integrations/esri/zipped_toolbox/nexus_toolbox.tbx -------------------------------------------------------------------------------- /integrations/python-client/nexuscli/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from nexuscli.nexuscli import TimeSeries 17 | from nexuscli.nexuscli import set_target 18 | from nexuscli.nexuscli import time_series 19 | from nexuscli.nexuscli import dataset_list 20 | from nexuscli.nexuscli import daily_difference_average 21 | from nexuscli.nexuscli import subset 22 | from nexuscli.nexuscli_ow import set_target 23 | from nexuscli.nexuscli_ow import run_file 24 | from nexuscli.nexuscli_ow import run_str 25 | -------------------------------------------------------------------------------- /integrations/python-client/nexuscli/test/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /integrations/python-client/setup.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from setuptools import setup, find_packages 18 | 19 | __version__ = '1.0' 20 | 21 | setup( 22 | name="nexuscli", 23 | version=__version__, 24 | packages=find_packages(), 25 | url="https://github.com/apache/incubator-sdap-nexus", 26 | 27 | author="Apache SDAP", 28 | author_email="dev@sdap.apache.org", 29 | 30 | description="NEXUS Client Module", 31 | long_description=open('README.md').read(), 32 | 33 | platforms='any', 34 | 35 | install_requires=[ 36 | 'requests', 37 | 'shapely', 38 | 'numpy', 39 | 'pytz' 40 | ], 41 | 42 | classifiers=[ 43 | 'Development Status :: 1 - Pre-Alpha', 44 | 'Intended Audience :: Developers', 45 | 'Operating System :: OS Independent', 46 | 'Programming Language :: Python :: 3.6', 47 | ] 48 | ) 49 | -------------------------------------------------------------------------------- /nexus-ingest/Readme.md: -------------------------------------------------------------------------------- 1 | NEXUS ingestion has been moved to https://github.com/apache/incubator-sdap-ningester 2 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | responses/ 2 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | ## SDAP regression tests 2 | 3 | ### To run: 4 | 5 | ```shell 6 | pytest test_sdap.py --with-integration [--force-subset] 7 | ``` 8 | 9 | ### Options 10 | 11 | - `--force-subset`: Force the test for the `/cdmssubset` endpoint to be run. It is currently skipped by default until Zarr functionality 12 | can be incorporated into the subsetter algorithm. 13 | 14 | 15 | ### Environment variables: 16 | 17 | | Name | Function | 18 | |-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| 19 | | TEST_HOST | Host for SDAP. Ex `http://localhost:8083/`. Default: `http://doms.jpl.nasa.gov` | 20 | | INSITU_ENDPOINT | Endpoint URL for in-situ API calls. Default: `http://doms.jpl.nasa.gov/insitu/1.0/query_data_doms_custom_pagination` | 21 | | INSITU_SWAGGER_ENDPOINT | Endpoint URL for in-situ SwaggerUI. Default: `http://doms.jpl.nasa.gov/insitu/1.0/insitu_query_swagger/` | 22 | 23 | NOTE: When the CDMS Reader tool is available, replace `cdms_reader.py` in this directory with a soft link to `/tools/cdms/cdms_reader.py` -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import pytest 17 | 18 | 19 | def pytest_addoption(parser): 20 | parser.addoption("--skip-matchup", action="store_true", 21 | help="Skip matchup_spark test. (Only for script testing purposes)") 22 | parser.addoption('--matchup-warn-on-miscount', action='store_false', 23 | help='Issue a warning for matchup tests if they return an unexpected number of matches; ' 24 | 'otherwise fail') 25 | 26 | 27 | def pytest_collection_modifyitems(config, items): 28 | skip_matchup = config.getoption("--skip-matchup") 29 | 30 | if skip_matchup: 31 | skip = pytest.mark.skip(reason="Manually skipped") 32 | for item in items: 33 | if "matchup_spark" in item.name: 34 | item.add_marker(skip) 35 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | pytest 3 | pytest-integration 4 | requests 5 | beautifulsoup4 6 | python-dateutil 7 | pytz 8 | shapely 9 | geopy 10 | netCDF4 11 | -------------------------------------------------------------------------------- /tests/test_collections.yaml: -------------------------------------------------------------------------------- 1 | collections: 2 | - id: MUR25-JPL-L4-GLOB-v04.2_test 3 | path: /data/granules/MUR25-JPL-L4-GLOB-v04.2_test/*.nc 4 | priority: 1 5 | forward-processing-priority: 6 6 | projection: Grid 7 | dimensionNames: 8 | latitude: lat 9 | longitude: lon 10 | time: time 11 | variable: analysed_sst 12 | slices: 13 | time: 1 14 | lat: 100 15 | lon: 100 16 | - id: ASCATB-L2-Coastal_test 17 | path: /data/granules/ASCATB-L2-Coastal_test/*.nc 18 | priority: 1 19 | projection: SwathMulti 20 | dimensionNames: 21 | latitude: lat 22 | longitude: lon 23 | variables: 24 | - wind_speed 25 | - wind_dir 26 | time: time 27 | slices: 28 | NUMROWS: 15 29 | NUMCELLS: 15 30 | - id: OISSS_L4_multimission_7day_v1_test 31 | path: /data/granules/OISSS_L4_multimission_7day_v1_test/*.nc 32 | priority: 1 33 | forward-processing-priority: 1 34 | projection: Grid 35 | dimensionNames: 36 | latitude: latitude 37 | longitude: longitude 38 | time: time 39 | variable: sss 40 | slices: 41 | time: 1 42 | latitude: 100 43 | longitude: 100 44 | - id: VIIRS_NPP-2018_Heatwave_test 45 | path: /data/granules/VIIRS_NPP-2018_Heatwave_test/*.nc 46 | priority: 1 47 | projection: Swath 48 | dimensionNames: 49 | latitude: lat 50 | longitude: lon 51 | time: time 52 | variable: sea_surface_temperature 53 | slices: 54 | ni: 30 55 | nj: 30 56 | preprocess: 57 | - name: squeeze 58 | dimensions: 59 | - time 60 | - id: SMAP_JPL_L3_SSS_CAP_8DAY-RUNNINGMEAN_V5_test 61 | path: /data/granules/SMAP_JPL_L3_SSS_CAP_8DAY-RUNNINGMEAN_V5_test/*.nc 62 | priority: 1 63 | forward-processing-priority: 1 64 | projection: Grid 65 | dimensionNames: 66 | latitude: latitude 67 | longitude: longitude 68 | time: time 69 | variable: smap_sss 70 | slices: 71 | time: 1 72 | latitude: 100 73 | longitude: 100 -------------------------------------------------------------------------------- /tools/build/README.md: -------------------------------------------------------------------------------- 1 | # SDAP Docker image build helper 2 | 3 | This script will help ease the process of building a full set of SDAP Docker images, whether building from an official ASF release, a release candidate, GitHub (any branch, official repo or forks), or 4 | the local filesystem (not yet implemented). Builds from ASF will also have their checksums and signatures checked, making this useful in the release process. You can also choose to not use build cache 5 | and can also push the images once they've been built. 6 | 7 | ## Images built 8 | 9 | - `sdap-nexus-webapp` 10 | - `sdap-collection-manager` 11 | - `sdap-granule-ingester` 12 | - `sdap-solr-cloud` 13 | - `sdap-solr-cloud-init` 14 | 15 | ## Requirements 16 | 17 | - Docker must be installed and running 18 | - gpg 19 | - git 20 | - tar 21 | - Python with the dependencies in `requirements.txt` installed 22 | 23 | ## Usage 24 | 25 | Basic usage is simple: 26 | 27 | ```shell 28 | python build.py 29 | ``` 30 | 31 | You will be prompted for all the information needed. There are, however, a couple extra options. Run `python build.py -h` to learn more on them. -------------------------------------------------------------------------------- /tools/build/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | beautifulsoup4 3 | tenacity 4 | 5 | -------------------------------------------------------------------------------- /tools/cdms/requirements.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | netCDF4==1.5.5.1 17 | -------------------------------------------------------------------------------- /tools/deletebyquery/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store -------------------------------------------------------------------------------- /tools/deletebyquery/README.md: -------------------------------------------------------------------------------- 1 | # Deleting Datasets from SDAP 2 | 3 | ## Prerequisites 4 | 5 | _If you are running the `deletebyquery.py` script from within the nexus-webapp-driver Docker image, the following prerequisites are not necessary._ 6 | 7 | * Run `python setup.py install` in `incubator-sdap-nexus/data-access` 8 | * Run `python setup.py install` in `incubator-sdap-nexus/analysis` 9 | * Run `pip install -r requirements.txt` in `incubator-sdap-nexus/tools/deletebyquery` 10 | 11 | ## Running the Script 12 | _Note: It is recommended to run this from within the nexus-webapp-driver Docker image._ 13 | 14 | ``` 15 | python deletebyquery.py --solr :8983 --cassandra --cassandraUsername --cassandraPassword --query '' 16 | ``` 17 | 18 | Run `python deletebyquery.py` without any arguments to see the full list of options. 19 | 20 | ## Usage Example 21 | To delete a dataset called `my_dataset`, with SDAP deployed using the Helm chart, run the following from within the nexus-webapp-driver Docker image: 22 | ``` 23 | cd /incubator-sdap-nexus/tools/deletebyquery 24 | python deletebyquery.py --solr sdap-solr-svc:8983 --cassandra sdap-cassandra --cassandraUsername cassandra --cassandraPassword cassandra --query 'dataset_s:"my_dataset"' 25 | ``` -------------------------------------------------------------------------------- /tools/deletebyquery/requirements.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | solrcloudpy==4.0.1 17 | -------------------------------------------------------------------------------- /tools/domspurge/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | FROM python:3.8.13 17 | 18 | COPY tools/domspurge/purge.py /domspurge/purge.py 19 | COPY tools/domspurge/requirements.txt /domspurge/requirements.txt 20 | COPY tools/domspurge/entrypoint.sh /domspurge/entrypoint.sh 21 | COPY analysis/webservice/algorithms/doms/DomsInitialization.py /domspurge/DomsInitialization.py 22 | 23 | # We just need the keyspace & table initialization. Remove references to other SDAP modules. 24 | # Doing this because if the schemas change, we want it to be reflected in this utility 25 | RUN sed '/nexus_initializer/d' -i /domspurge/DomsInitialization.py 26 | 27 | WORKDIR /domspurge 28 | 29 | RUN pip3 install -r requirements.txt 30 | 31 | ENTRYPOINT ["/bin/bash", "/domspurge/entrypoint.sh"] 32 | -------------------------------------------------------------------------------- /tools/domspurge/README.md: -------------------------------------------------------------------------------- 1 | # Purge DOMS/CDMS Execution Data 2 | 3 | ## Prerequisites 4 | 5 | _If you are running the `purge.py` script from within the nexus-webapp-driver Docker image, the following prerequisites are not necessary._ 6 | 7 | * Run `python setup.py install` in `incubator-sdap-nexus/analysis` 8 | * Run `pip install -r requirements.txt` in `incubator-sdap-nexus/tools/domspurge` 9 | 10 | ## Usage 11 | `python purge.py -u USERNAME -p PASSWORD [CASSANDRA ARGS...] (--before DATETIME | --before-months MONTHS | --keep-completed | --all) [--keep-failed] [--dry-run] [-y]` 12 | 13 | ### Options 14 | - `-u & -p`: Cassandra username and password 15 | - One of: 16 | - `--before`: Datetime (ie, '2023-04-11T19:50:00') before which all data will be purged. Should be entered in UTC and the timezone must not be specified. 17 | - `--before-months`: Number of months prior to current date before which all data will be purged. 18 | - `--keep-completed`: Keep all completed executions. (only purge uncompleted executions) 19 | - `--all`: Purge ALL data. (drops and re-creates keyspace) 20 | - Cassandra args (optional) 21 | - `--cassandra`: Cassandra hostname(s) or IP(s) Can provide a list of args or provide them separated by commas (eg: `host1 host2 host3` or `host1,host2,host3`). (Default: localhost) 22 | - `-k / --cassandraKeyspace`: Cassandra keyspace for storing DOMS data. (Default: doms) 23 | - `--cassandraPort`: Port used to connect to Cassandra. (Default: 9042) 24 | - `--cassandraProtocolVersion`: The version of the Cassandra protocol the driver should use. (Default: 3) 25 | - Additional args 26 | - `--keep-failed`: Do not purge uncompleted executions (by default all are purged). Incompatible with `--keep-completed` 27 | - `--dry-run`: Only print the execution ids to be deleted / DB operations to be performed and exit. Do not actually alter the DB 28 | - `-y / --yes`: Do not prompt user for confirmation. 29 | 30 | ## Build Docker Image 31 | 32 | You can build an image for this script to run it in a Kubernetes CronJob. 33 | 34 | ```shell 35 | cd /incubator-sdap-nexus 36 | docker build . -f tools/domspurge/Dockerfile -t sdap-local/DomsPurge: 37 | ``` 38 | -------------------------------------------------------------------------------- /tools/domspurge/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | python /domspurge/purge.py \ 18 | $([[ ! -z "$CASSANDRA_CONTACT_POINTS" ]] && echo --cassandra=$CASSANDRA_CONTACT_POINTS) \ 19 | $([[ ! -z "$CASSANDRA_PORT" ]] && echo --cassandraPort=$CASSANDRA_PORT) \ 20 | $([[ ! -z "$CASSANDRA_KEYSPACE" ]] && echo --cassandraKeyspace=$CASSANDRA_KEYSPACE) \ 21 | $([[ ! -z "$CASSANDRA_USERNAME" ]] && echo --cassandra-username=$CASSANDRA_USERNAME) \ 22 | $([[ ! -z "$CASSANDRA_PASSWORD" ]] && echo --cassandra-password=$CASSANDRA_PASSWORD) \ 23 | $([[ ! -z "$CASSANDRA_PROTOCOL" ]] && echo --cassandraProtocolVersion=$CASSANDRA_PROTOCOL) \ 24 | $([[ ! -z "$YES" ]] && echo -y) \ 25 | $([[ ! -z "$DRY_RUN" ]] && echo --dry-run) \ 26 | $([[ ! -z "$BEFORE" ]] && echo --before=$BEFORE) \ 27 | $([[ ! -z "$BEFORE_MONTHS" ]] && echo --before-months=$BEFORE_MONTHS) \ 28 | $([[ ! -z "$KEEP_COMPLETED" ]] && echo --keep-completed) \ 29 | $([[ ! -z "$KEEP_FAILED" ]] && echo --keep-failed) \ 30 | $([[ ! -z "$PURGE_ALL" ]] && echo --all) 31 | -------------------------------------------------------------------------------- /tools/domspurge/requirements.txt: -------------------------------------------------------------------------------- 1 | cassandra-driver==3.24.0 2 | python-dateutil==2.7.5 3 | six==1.16.0 4 | tqdm==4.64.1 -------------------------------------------------------------------------------- /tools/rmqmonitor/README.md: -------------------------------------------------------------------------------- 1 | # RMQ Monitor 2 | 3 | A simple script to monitor the RabbitMQ message queue used by the SDAP ingester. 4 | 5 | ## To run 6 | 7 | You first need to install the Python requests package 8 | 9 | `pip install requests` 10 | 11 | To run: 12 | 13 | `python monitor.py` --------------------------------------------------------------------------------