12 | {% endblock %}
13 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0042_remove_collectionasset_is_external.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-22 12:05
2 |
3 | from django.db import migrations
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('stac_api', '0041_alter_collection_external_asset_pattern'),
10 | ]
11 |
12 | operations = [
13 | migrations.RemoveField(
14 | model_name='collectionasset',
15 | name='is_external',
16 | ),
17 | ]
18 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0044_alter_collectionlink_unique_together.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-30 15:26
2 |
3 | from django.db import migrations
4 |
5 |
6 | class Migration(migrations.Migration):
7 | dependencies = [
8 | ("stac_api", "0043_remove_collection_external_asset_pattern_and_more"),
9 | ]
10 |
11 | operations = [
12 | migrations.AlterUniqueTogether(
13 | name="collectionlink",
14 | unique_together=set(),
15 | ),
16 | ]
17 |
--------------------------------------------------------------------------------
/app/stac_api/templates/style/admin/upload.css:
--------------------------------------------------------------------------------
1 | .box {
2 | padding: 5px 10px 9px;
3 | border: 2px solid;
4 | border-radius: 20px;
5 | margin: 15px 0px;
6 | }
7 |
8 | .info {
9 | border-color: #264b5d;
10 | }
11 |
12 | .error {
13 | border-color: rgb(137, 0, 0);
14 | }
15 |
16 | .statusbox {
17 | padding: 5px 10px 9px;
18 | margin: 15px 0px;
19 | }
20 |
21 | .fileUploadForm {
22 | margin: 20px 0 40px;
23 | }
24 |
25 | .fileUploadForm div {
26 | margin: 20px 0;
27 | }
28 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0010_auto_20210621_1453.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.10 on 2021-06-21 14:53
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0009_collection_published'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddIndex(
15 | model_name='collection',
16 | index=models.Index(fields=['published'], name='collection_published_idx'),
17 | ),
18 | ]
19 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0008_auto_20210621_0840.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.10 on 2021-06-21 08:40
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0007_auto_20210421_0701'),
11 | ]
12 |
13 | operations = [
14 | migrations.AlterField(
15 | model_name='provider',
16 | name='description',
17 | field=models.TextField(blank=True, default=None, null=True),
18 | ),
19 | ]
20 |
--------------------------------------------------------------------------------
/app/tests/runner.py:
--------------------------------------------------------------------------------
1 | from django.test.runner import DiscoverRunner
2 |
3 |
4 | class TestRunner(DiscoverRunner):
5 |
6 | # We run the tests with debug True
7 | # otherwise we run into issues with things
8 | # defined in settings_dev.py.
9 | # Other option would be a dedicated test settings
10 | # file, but this would require to set the ENV
11 | # variable DJANGO_SETTINGS_MODULE whenever running
12 | # tests (see https://stackoverflow.com/a/41349685/9896222)
13 | def __init__(self, *args, **kwargs):
14 | super().__init__(*args, **kwargs)
15 | self.debug_mode = True
16 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0004_auto_20210408_0659.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.7 on 2021-04-08 06:59
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0003_auto_20210325_1001'),
11 | ]
12 |
13 | operations = [
14 | migrations.AlterField(
15 | model_name='asset',
16 | name='checksum_multihash',
17 | field=models.CharField(
18 | blank=True, default=None, editable=False, max_length=255, null=True
19 | ),
20 | ),
21 | ]
22 |
--------------------------------------------------------------------------------
/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | pythonpath = app
3 |
4 | DJANGO_SETTINGS_MODULE= app.config.settings_test
5 |
6 | addopts = --ds=config.settings_test
7 |
8 | django_debug_mode = true
9 |
10 | python_files = test_*.py
11 |
12 | # we have some environment variables that are mandatory, make sure that the
13 | # code can read them
14 | # The settings-setup is a bit unideal. Even though we specify the aws settings
15 | # manually in settings_test, it'll still raise exceptions since settings_prod
16 | # wants to read the environment, and settings_prod is included in settings_test
17 | env_files =
18 | .env
19 | .env.local
20 | .env.default
21 |
22 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0012_auto_20210709_0734.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.10 on 2021-07-09 07:34
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 | import stac_api.validators
7 |
8 |
9 | class Migration(migrations.Migration):
10 |
11 | dependencies = [
12 | ('stac_api', '0011_auto_20210623_0521'),
13 | ]
14 |
15 | operations = [
16 | migrations.AlterField(
17 | model_name='asset',
18 | name='eo_gsd',
19 | field=models.FloatField(
20 | blank=True, null=True, validators=[stac_api.validators.validate_eo_gsd]
21 | ),
22 | ),
23 | ]
24 |
--------------------------------------------------------------------------------
/spec/components/headers.yaml:
--------------------------------------------------------------------------------
1 | openapi: 3.0.3
2 | components:
3 | headers:
4 | ETag:
5 | schema:
6 | type: string
7 | description: >-
8 | The RFC7232 ETag header field in a response provides the current entity-
9 | tag for the selected resource. An entity-tag is an opaque identifier for
10 | different versions of a resource over time, regardless whether multiple
11 | versions are valid at the same time. An entity-tag consists of an opaque
12 | quoted string, possibly prefixed by a weakness indicator.
13 | example: "d01af8b8ebbf899e30095be8754b377ddb0f0ed0f7fddbc33ac23b0d1969736b"
14 | required: true
15 |
--------------------------------------------------------------------------------
/app/tests/tests_09/utils.py:
--------------------------------------------------------------------------------
1 | from io import StringIO
2 |
3 | from django.core.management import call_command
4 | from django.urls import reverse
5 |
6 | from tests.tests_09.base_test import VERSION_SHORT
7 |
8 |
9 | def reverse_version(viewname, urlconf=None, args=None, kwargs=None, current_app=None):
10 | ns = VERSION_SHORT
11 | return reverse(ns + ':' + viewname, urlconf, args, kwargs, current_app=ns)
12 |
13 |
14 | def calculate_extent(*args, **kwargs):
15 | out = StringIO()
16 | call_command(
17 | "calculate_extent",
18 | *args,
19 | stdout=out,
20 | stderr=StringIO(),
21 | **kwargs,
22 | )
23 | return out.getvalue()
24 |
--------------------------------------------------------------------------------
/app/tests/tests_10/utils.py:
--------------------------------------------------------------------------------
1 | from io import StringIO
2 |
3 | from django.core.management import call_command
4 | from django.urls import reverse
5 |
6 | from tests.tests_10.base_test import VERSION_SHORT
7 |
8 |
9 | def reverse_version(viewname, urlconf=None, args=None, kwargs=None, current_app=None):
10 | ns = VERSION_SHORT
11 | return reverse(ns + ':' + viewname, urlconf, args, kwargs, current_app=ns)
12 |
13 |
14 | def calculate_extent(*args, **kwargs):
15 | out = StringIO()
16 | call_command(
17 | "calculate_extent",
18 | *args,
19 | stdout=out,
20 | stderr=StringIO(),
21 | **kwargs,
22 | )
23 | return out.getvalue()
24 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0066_collectionasset_is_external.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.11 on 2025-03-24 13:38
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0065_remove_asset_add_del_asset_item_update_interval_trigger_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='collectionasset',
16 | name='is_external',
17 | field=models.BooleanField(
18 | default=False, help_text='Whether this asset is hosted externally'
19 | ),
20 | ),
21 | ]
22 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0032_alter_asset_file.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.6 on 2024-06-27 09:43
2 |
3 | from django.db import migrations
4 |
5 | import stac_api.models.general
6 |
7 |
8 | class Migration(migrations.Migration):
9 |
10 | dependencies = [
11 | ('stac_api', '0031_alter_collection_extent_geometry_alter_item_geometry'),
12 | ]
13 |
14 | operations = [
15 | migrations.AlterField(
16 | model_name='asset',
17 | name='file',
18 | field=stac_api.models.general.DynamicStorageFileField(
19 | max_length=255, upload_to=stac_api.models.general.upload_asset_to_path_hook
20 | ),
21 | ),
22 | ]
23 |
--------------------------------------------------------------------------------
/adr/2020_10_01_authentication.md:
--------------------------------------------------------------------------------
1 | # Authentication
2 |
3 | > `Status: accepted`
4 |
5 | > `Date: 2020-10-01`
6 |
7 | ## Context
8 | `service-stac` will be accepting machine-to-machine communication and will have an admin interface for operations/debugging. Authentication methods for this two use cases need to be defined.
9 |
10 | ## Decision
11 | Machine-to-machine communication will be using token authentication, access to the admin interface will be granted with usernames/passwords managed in the Django admin interface. At a later stage, this might be changed to a more advanced authentication scheme.
12 |
13 | ## Consequences
14 | Superuser and regular user permissions will be handled within the application.
15 |
--------------------------------------------------------------------------------
/app/middleware/exception.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import sys
3 |
4 | logger = logging.getLogger(__name__)
5 |
6 |
7 | class ExceptionLoggingMiddleware:
8 |
9 | def __init__(self, get_response):
10 | self.get_response = get_response
11 |
12 | def __call__(self, request):
13 | return self.get_response(request)
14 |
15 | def process_exception(self, request, exception):
16 | # NOTE: this process_exception is not called for REST Framework endpoints. For those
17 | # the exceptions handling and logging is done within stac_api.apps.custom_exception_handler
18 | extra = {"request": request}
19 | logger.critical(repr(exception), extra=extra, exc_info=sys.exc_info())
20 |
--------------------------------------------------------------------------------
/app/tests/test_healthcheck.py:
--------------------------------------------------------------------------------
1 | # Deactivate healthcheck test as endpoint is also deactivated
2 |
3 | # from config.settings import HEALTHCHECK_ENDPOINT
4 | # from config.settings import STAC_BASE
5 |
6 | # from django.test import Client
7 | # from django.test import TestCase
8 |
9 | # class HealthCheckTestCase(TestCase):
10 |
11 | # def setUp(self):
12 | # self.client = Client()
13 |
14 | # def test_healthcheck(self):
15 | # response = self.client.get(f"/{STAC_BASE}/{HEALTHCHECK_ENDPOINT}")
16 | # self.assertEqual(response.status_code, 200)
17 | # self.assertEqual(response['Content-Type'], 'application/json')
18 | # self.assertIn('no-cache', response['Cache-control'])
19 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0038_collection_allow_external_assets.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-18 09:09
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0037_asset_is_external_collectionasset_is_external'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='collection',
16 | name='allow_external_assets',
17 | field=models.BooleanField(
18 | default=False,
19 | help_text='Whether this collection can have assets that are hosted externally'
20 | ),
21 | ),
22 | ]
23 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0040_collection_external_asset_pattern.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-19 12:44
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0038_collection_allow_external_assets'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='collection',
16 | name='external_asset_pattern',
17 | field=models.CharField(
18 | blank=True,
19 | help_text='The allowed regex pattern for external URLs',
20 | max_length=1024
21 | ),
22 | ),
23 | ]
24 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0041_alter_collection_external_asset_pattern.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-19 13:16
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0040_collection_external_asset_pattern'),
11 | ]
12 |
13 | operations = [
14 | migrations.AlterField(
15 | model_name='collection',
16 | name='external_asset_pattern',
17 | field=models.CharField(
18 | blank=True,
19 | help_text='The regex pattern for allowed external URLs',
20 | max_length=1024
21 | ),
22 | ),
23 | ]
24 |
--------------------------------------------------------------------------------
/app/stac_api/templates/style/hover.css:
--------------------------------------------------------------------------------
1 | .NameHighlights {
2 | position:relative;
3 | }
4 | .NameHighlights div.SearchUsage {
5 | display: none;
6 | }
7 | .NameHighlightsHover {
8 | position:relative;
9 | }
10 | .NameHighlightsHover div.SearchUsage {
11 | display:block;
12 | position:absolute;
13 | width: 70em;
14 | top:4em;
15 | left:70px;
16 | z-index:1000;
17 | color: black;
18 | background-color: #DDD;
19 | padding: 5px;
20 | border-radius: 4px;
21 | }
22 |
23 |
24 |
25 | .SearchUsage ul li {
26 | list-style-type: "- " !important;
27 | list-style-position: inside !important;
28 | margin-left: 1em !important;
29 | }
30 |
31 | .SearchUsage ul li i {
32 | font-family: monospace;
33 | }
34 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0026_add_asset_upload_content_encoding.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.0.10 on 2023-03-22 14:15
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0025_add_asset_media_type'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='assetupload',
16 | name='content_encoding',
17 | field=models.CharField(
18 | blank=True,
19 | choices=[(None, ''), ('gzip', 'Gzip'), ('br', 'Br')],
20 | default='',
21 | max_length=32
22 | ),
23 | ),
24 | ]
25 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0018_assetupload_md5_parts.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.13 on 2021-09-15 14:05
2 |
3 | import django.core.serializers.json
4 | from django.db import migrations
5 | from django.db import models
6 |
7 |
8 | class Migration(migrations.Migration):
9 |
10 | dependencies = [
11 | ('stac_api', '0017_data_collection_summaries_lang'),
12 | ]
13 |
14 | operations = [
15 | migrations.AddField(
16 | model_name='assetupload',
17 | name='md5_parts',
18 | field=models.JSONField(
19 | blank=True,
20 | default=list,
21 | editable=False,
22 | encoder=django.core.serializers.json.DjangoJSONEncoder
23 | ),
24 | ),
25 | ]
26 |
--------------------------------------------------------------------------------
/app/tests/test_checker.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | from django.test import Client
4 | from django.test import TestCase
5 | from django.test import override_settings
6 |
7 |
8 | class CheckerTestCase(TestCase):
9 |
10 | def setUp(self):
11 | self.client = Client()
12 |
13 | def test_checker(self):
14 | response = self.client.get('/checker')
15 | self.assertEqual(response.status_code, 200)
16 |
17 | @override_settings(CHECKER_DELAY=2)
18 | def test_checker_delayed(self):
19 | start = time.time()
20 | response = self.client.get('/checker')
21 | end = time.time()
22 | self.assertEqual(response.status_code, 200)
23 | self.assertGreater(end, start + 2, f'Expected at least 2s, only took {end - start}')
24 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0049_item_properties_expires.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2024-08-14 11:13
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0048_remove_item_update_item_collection_extent_trigger_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='item',
16 | name='properties_expires',
17 | field=models.DateTimeField(
18 | blank=True,
19 | help_text=
20 | 'Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format',
21 | null=True
22 | ),
23 | ),
24 | ]
25 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0056_alter_collection_total_data_size_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2024-11-28 16:08
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 | dependencies = [
9 | ("stac_api", "0055_alter_asset_file_size_alter_assetupload_file_size_and_more"),
10 | ]
11 |
12 | operations = [
13 | migrations.AlterField(
14 | model_name="collection",
15 | name="total_data_size",
16 | field=models.BigIntegerField(blank=True, default=0, null=True),
17 | ),
18 | migrations.AlterField(
19 | model_name="item",
20 | name="total_data_size",
21 | field=models.BigIntegerField(blank=True, default=0, null=True),
22 | ),
23 | ]
24 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0009_collection_published.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.10 on 2021-06-21 13:57
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0008_auto_20210621_0840'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='collection',
16 | name='published',
17 | field=models.BooleanField(
18 | default=True,
19 | help_text=
20 | "When not published the collection doesn't appear on the api/stac/v0.9/collections "
21 | "endpoint and its items are not listed in /search endpoint."
22 | "
NOTE: unpublished collections/items can still be accessed by their path.
"
23 | ),
24 | ),
25 | ]
26 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0035_asset_roles.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.6 on 2024-07-09 14:10
2 |
3 | import django.contrib.postgres.fields
4 | from django.db import migrations
5 | from django.db import models
6 |
7 |
8 | class Migration(migrations.Migration):
9 |
10 | dependencies = [
11 | ('stac_api', '0034_merge_0032_alter_asset_file_0033_auto_20240704_1157'),
12 | ]
13 |
14 | operations = [
15 | migrations.AddField(
16 | model_name='asset',
17 | name='roles',
18 | field=django.contrib.postgres.fields.ArrayField(
19 | base_field=models.CharField(max_length=255),
20 | blank=True,
21 | default=None,
22 | help_text='Comma-separated list of roles to describe the purpose of the asset',
23 | null=True,
24 | size=None
25 | ),
26 | ),
27 | ]
28 |
--------------------------------------------------------------------------------
/app/tests/tests_09/test_conformance.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 | from django.test import TestCase
3 |
4 | from tests.tests_09.base_test import STAC_BASE_V
5 |
6 |
7 | class ConformanceTestCase(TestCase):
8 |
9 | def setUp(self):
10 | self.client = Client()
11 |
12 | def test_conforms_to_page(self):
13 | response = self.client.get(f"/{STAC_BASE_V}/conformance")
14 | self.assertEqual(response.status_code, 200)
15 | self.assertEqual(response['Content-Type'], 'application/json')
16 | required_keys = ['conformsTo']
17 | self.assertEqual(
18 | set(required_keys).difference(response.json().keys()),
19 | set(),
20 | msg="missing required attribute in json answer"
21 | )
22 |
23 | self.assertGreater(
24 | len(response.json()['conformsTo']), 0, msg='there are no links defined in conformsTo'
25 | )
26 |
--------------------------------------------------------------------------------
/app/tests/tests_10/test_conformance.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 | from django.test import TestCase
3 |
4 | from tests.tests_10.base_test import STAC_BASE_V
5 |
6 |
7 | class ConformanceTestCase(TestCase):
8 |
9 | def setUp(self):
10 | self.client = Client()
11 |
12 | def test_conforms_to_page(self):
13 | response = self.client.get(f"/{STAC_BASE_V}/conformance")
14 | self.assertEqual(response.status_code, 200)
15 | self.assertEqual(response['Content-Type'], 'application/json')
16 | required_keys = ['conformsTo']
17 | self.assertEqual(
18 | set(required_keys).difference(response.json().keys()),
19 | set(),
20 | msg="missing required attribute in json answer"
21 | )
22 |
23 | self.assertGreater(
24 | len(response.json()['conformsTo']), 0, msg='there are no links defined in conformsTo'
25 | )
26 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0037_asset_is_external_collectionasset_is_external.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-16 15:31
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0036_collectionasset_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='asset',
16 | name='is_external',
17 | field=models.BooleanField(
18 | default=False, help_text='Whether this asset is hosted externally'
19 | ),
20 | ),
21 | migrations.AddField(
22 | model_name='collectionasset',
23 | name='is_external',
24 | field=models.BooleanField(
25 | default=False, help_text='Whether this asset is hosted externally'
26 | ),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/app/tests/tests_09/test_cf_forwarded_proto.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 | from django.test import TestCase
3 |
4 | from tests.tests_09.base_test import STAC_BASE_V
5 |
6 |
7 | class CFForwardedProtoTestCase(TestCase):
8 |
9 | def setUp(self): # pylint: disable=invalid-name
10 | self.client = Client()
11 |
12 | def test_http_access(self):
13 | response = self.client.get(f"/{STAC_BASE_V}", HTTP_ACCEPT='application/json', follow=True)
14 | for link in response.json().get('links', []):
15 | self.assertTrue(link['href'].startswith('http://'))
16 |
17 | def test_https_access(self):
18 | response = self.client.get(
19 | f"/{STAC_BASE_V}",
20 | HTTP_CLOUDFRONT_FORWARDED_PROTO='https',
21 | HTTP_ACCEPT='application/json',
22 | follow=True
23 | )
24 | for link in response.json().get('links', []):
25 | self.assertTrue(link['href'].startswith('https://'))
26 |
--------------------------------------------------------------------------------
/app/tests/tests_10/test_cf_forwarded_proto.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 | from django.test import TestCase
3 |
4 | from tests.tests_10.base_test import STAC_BASE_V
5 |
6 |
7 | class CFForwardedProtoTestCase(TestCase):
8 |
9 | def setUp(self): # pylint: disable=invalid-name
10 | self.client = Client()
11 |
12 | def test_http_access(self):
13 | response = self.client.get(f"/{STAC_BASE_V}", HTTP_ACCEPT='application/json', follow=True)
14 | for link in response.json().get('links', []):
15 | self.assertTrue(link['href'].startswith('http://'))
16 |
17 | def test_https_access(self):
18 | response = self.client.get(
19 | f"/{STAC_BASE_V}",
20 | HTTP_CLOUDFRONT_FORWARDED_PROTO='https',
21 | HTTP_ACCEPT='application/json',
22 | follow=True
23 | )
24 | for link in response.json().get('links', []):
25 | self.assertTrue(link['href'].startswith('https://'))
26 |
--------------------------------------------------------------------------------
/.github/release.yml:
--------------------------------------------------------------------------------
1 | # .github/release.yml
2 | # WARNING this file is managed by terraform and cannot be edited manually, see
3 | # geoadmin/infra-terraform-bgdi/tf/github/geoadmin/modules/service-repository-semver/release_config.tf
4 |
5 | changelog:
6 | exclude:
7 | labels:
8 | - ignore-for-release
9 | - skip-changelog
10 | - skip-rn
11 | - skip-release-note
12 | - no-release-note
13 | - no-rn
14 | - no-changelog
15 | - new-release
16 | authors:
17 | - terraform-bgdi
18 | categories:
19 | - title: Breaking Changes 🛠
20 | labels:
21 | - breaking-change
22 | - title: New Features
23 | labels:
24 | - feature
25 | - enhancement
26 | - title: Data Updates
27 | labels:
28 | - data
29 | - data-integration
30 | - title: Bug Fixes
31 | labels:
32 | - fix
33 | - bugfix
34 | - bug
35 | - title: Other Changes
36 | labels:
37 | - "*"
38 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0054_update_conformance_endpoint.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2024-10-09 11:45
2 |
3 | from django.db import migrations
4 |
5 |
6 | def add_landing_page_version(apps, schema_editor):
7 | LandingPage = apps.get_model("stac_api", "LandingPage")
8 | lp = LandingPage.objects.get(version='v1')
9 | lp.conformsTo.insert(2, 'https://api.stacspec.org/v1.0.0/ogcapi-features')
10 | lp.save()
11 |
12 |
13 | def reverse_landing_page_version(apps, schema_editor):
14 | # Remove the landing page v0.9
15 | LandingPage = apps.get_model("stac_api", "LandingPage")
16 | lp = LandingPage.objects.get(version='v1')
17 | lp.conformsTo.remove('https://api.stacspec.org/v1.0.0/ogcapi-features')
18 | lp.save()
19 |
20 |
21 | class Migration(migrations.Migration):
22 | dependencies = [
23 | ("stac_api", "0053_alter_asset_media_type_and_more"),
24 | ]
25 |
26 | operations = [migrations.RunPython(add_landing_page_version, reverse_landing_page_version)]
27 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0058_collectionlink_hreflang_itemlink_hreflang_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2024-12-05 10:23
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0057_item_forecast_duration_item_forecast_horizon_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='collectionlink',
16 | name='hreflang',
17 | field=models.CharField(blank=True, max_length=32, null=True),
18 | ),
19 | migrations.AddField(
20 | model_name='itemlink',
21 | name='hreflang',
22 | field=models.CharField(blank=True, max_length=32, null=True),
23 | ),
24 | migrations.AddField(
25 | model_name='landingpagelink',
26 | name='hreflang',
27 | field=models.CharField(blank=True, max_length=32, null=True),
28 | ),
29 | ]
30 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0063_collection_cache_control_header.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.11 on 2025-03-06 09:14
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 | import stac_api.validators
7 |
8 |
9 | class Migration(migrations.Migration):
10 |
11 | dependencies = [
12 | ('stac_api', '0062_item_item_fc_reference_datetime_idx_and_more'),
13 | ]
14 |
15 | operations = [
16 | migrations.AddField(
17 | model_name='collection',
18 | name='cache_control_header',
19 | field=models.CharField(
20 | blank=True,
21 | help_text=
22 | 'Cache-Control header value to use for this collection. When set it override the default cache control header value for all API call related to the collection as well as for the data download call.',
23 | max_length=255,
24 | null=True,
25 | validators=[stac_api.validators.validate_cache_control_header]
26 | ),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/app/stac_api/profiling.py:
--------------------------------------------------------------------------------
1 | import cProfile
2 | import functools
3 | import io
4 | import os
5 | import pstats
6 |
7 |
8 | def profiling(function_to_profile):
9 | ''''Profiling wrapper for debugging
10 |
11 | You can use this wrapper to get some profiling data on the function. The profiling data are
12 | printed in the console.
13 | '''
14 |
15 | @functools.wraps(function_to_profile)
16 | def wrapper(*args, **kwargs):
17 | profiler = cProfile.Profile()
18 |
19 | profiler.enable()
20 | return_value = function_to_profile(*args, **kwargs)
21 | profiler.disable()
22 |
23 | stream = io.StringIO()
24 | stats = pstats.Stats(profiler,
25 | stream=stream).sort_stats(os.getenv('PROFILING_SORT_KEY', 'cumtime'))
26 | stats_lines = os.getenv('PROFILING_STATS_LINES', None)
27 | if stats_lines:
28 | stats.print_stats(stats_lines)
29 | else:
30 | stats.print_stats()
31 |
32 | return return_value
33 |
34 | return wrapper
35 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0043_remove_collection_external_asset_pattern_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-23 08:19
2 |
3 | import django.contrib.postgres.fields
4 | from django.db import migrations
5 | from django.db import models
6 |
7 |
8 | class Migration(migrations.Migration):
9 |
10 | dependencies = [
11 | ('stac_api', '0042_remove_collectionasset_is_external'),
12 | ]
13 |
14 | operations = [
15 | migrations.RemoveField(
16 | model_name='collection',
17 | name='external_asset_pattern',
18 | ),
19 | migrations.AddField(
20 | model_name='collection',
21 | name='external_asset_whitelist',
22 | field=django.contrib.postgres.fields.ArrayField(
23 | base_field=models.CharField(max_length=255),
24 | blank=True,
25 | default=list,
26 | help_text=
27 | 'Provide a comma separated list of protocol://domain values for the external asset url validation',
28 | size=None
29 | ),
30 | ),
31 | ]
32 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0029_alter_collectionlink_href_alter_itemlink_href_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.0.10 on 2024-03-19 14:58
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0028_alter_asset_media_type'),
11 | ]
12 |
13 | operations = [
14 | migrations.AlterField(
15 | model_name='collectionlink',
16 | name='href',
17 | field=models.URLField(max_length=2048),
18 | ),
19 | migrations.AlterField(
20 | model_name='itemlink',
21 | name='href',
22 | field=models.URLField(max_length=2048),
23 | ),
24 | migrations.AlterField(
25 | model_name='landingpagelink',
26 | name='href',
27 | field=models.URLField(max_length=2048),
28 | ),
29 | migrations.AlterField(
30 | model_name='provider',
31 | name='url',
32 | field=models.URLField(blank=True, max_length=2048, null=True),
33 | ),
34 | ]
35 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0061_remove_item_forecast_mode_remove_item_forecast_param_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2025-01-15 14:23
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0060_item_forecast_perturbed_item_forecast_variable'),
11 | ]
12 |
13 | operations = [
14 | migrations.RemoveField(
15 | model_name='item',
16 | name='forecast_mode',
17 | ),
18 | migrations.RemoveField(
19 | model_name='item',
20 | name='forecast_param',
21 | ),
22 | migrations.AlterField(
23 | model_name='item',
24 | name='forecast_variable',
25 | field=models.CharField(
26 | blank=True,
27 | help_text=
28 | 'Name of the model variable that corresponds to the data. The variables should correspond to the CF Standard Names, e.g. `air_temperature` for the air temperature.',
29 | null=True
30 | ),
31 | ),
32 | ]
33 |
--------------------------------------------------------------------------------
/app/tests/tests_10/test_utils.py:
--------------------------------------------------------------------------------
1 | from unittest import TestCase
2 |
3 | from stac_api.utils import parse_cache_control_header
4 |
5 |
6 | class TestUtils(TestCase):
7 |
8 | def test_parse_cache_control_header(self):
9 | self.assertEqual(parse_cache_control_header('max-age=360'), {'max-age': '360'})
10 | self.assertEqual(parse_cache_control_header('max-age=360,'), {'max-age': '360'})
11 | self.assertEqual(parse_cache_control_header(' max-age=360 , '), {'max-age': '360'})
12 | self.assertEqual(
13 | parse_cache_control_header('max-age=360, public'), {
14 | 'max-age': '360', 'public': True
15 | }
16 | )
17 | self.assertEqual(
18 | parse_cache_control_header(' max-age = 360 , test = hello'), {
19 | 'max-age': '360', 'test': 'hello'
20 | }
21 | )
22 | self.assertEqual(parse_cache_control_header(''), {})
23 | self.assertEqual(parse_cache_control_header(','), {})
24 | self.assertEqual(parse_cache_control_header(' '), {})
25 | self.assertEqual(parse_cache_control_header(' , '), {})
26 |
--------------------------------------------------------------------------------
/app/middleware/api_gateway_authentication.py:
--------------------------------------------------------------------------------
1 | from middleware import api_gateway
2 |
3 | from django.conf import settings
4 |
5 | from rest_framework.authentication import RemoteUserAuthentication
6 |
7 |
8 | class ApiGatewayAuthentication(RemoteUserAuthentication):
9 | header = api_gateway.REMOTE_USER_HEADER
10 |
11 | def authenticate(self, request):
12 | if not settings.FEATURE_AUTH_ENABLE_APIGW:
13 | return None
14 |
15 | api_gateway.validate_username_header(request)
16 | return super().authenticate(request)
17 |
18 | def authenticate_header(self, request):
19 | # For this authentication method, users send a "Bearer" token via the
20 | # Authorization header. API Gateway looks up that token in Cognito and
21 | # sets the Geoadmin-Username and Geoadmin-Authenticated headers. In this
22 | # module we only care about the Geoadmin-* headers. But when
23 | # authentication fails with a 401 error we need to hint at the correct
24 | # authentication method from the point of view of the user, which is the
25 | # Authorization/Bearer scheme.
26 | return 'Bearer'
27 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0017_data_collection_summaries_lang.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.13 on 2021-09-02 17:31
2 |
3 | from django.db import migrations
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('stac_api', '0016_auto_20210902_1731'),
10 | ]
11 |
12 | operations = [
13 | migrations.RunSQL(
14 | '''
15 | WITH collection_summaries AS (
16 | SELECT
17 | item.collection_id AS collection_id,
18 | array_remove(array_agg(DISTINCT(asset.geoadmin_lang)), null) AS geoadmin_lang
19 | FROM stac_api_item AS item
20 | LEFT JOIN stac_api_asset AS asset ON (asset.item_id = item.id)
21 | GROUP BY item.collection_id
22 | )
23 | UPDATE stac_api_collection
24 | SET
25 | summaries_geoadmin_lang = collection_summaries.geoadmin_lang
26 | FROM collection_summaries
27 | WHERE collection_summaries.collection_id = stac_api_collection.id;
28 | ''',
29 | reverse_sql=migrations.RunSQL.noop
30 | )
31 | ]
32 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | notification-configuration.json
2 | packaged.yaml
3 |
4 | # Byte-compiled / optimized / DLL files
5 | __pycache__/
6 | *.py[cod]
7 | *$py.class
8 |
9 | # C extensions
10 | *.so
11 |
12 | # django developp sqlite db
13 | *db.sqlite3
14 |
15 | # Distribution / packaging
16 | .Python
17 | env/
18 | _env
19 | build/
20 | develop-eggs/
21 | dist/
22 | downloads/
23 | eggs/
24 | .eggs/
25 | lib/
26 | lib64/
27 | parts/
28 | sdist/
29 | var/
30 | wheels/
31 | *.egg-info/
32 | .installed.cfg
33 | *.egg
34 | .local/
35 |
36 | # virtualenv
37 | .venv
38 |
39 | # mypy
40 | .mypy_cache/
41 |
42 | # unittest report
43 | test-reports/
44 | tests/report/
45 | nose2-junit.xml
46 |
47 | # PyCharm generated files
48 | *.idea
49 | */bin/
50 | */lib64
51 | */pip-selfcheck.json
52 | */pyvenv.cfg
53 |
54 | #vim
55 | *.swp
56 |
57 | # git generate files
58 | *.orig
59 |
60 | # IDE config files
61 | *.sublime-*
62 | *.code*
63 |
64 | # Makefile
65 | .timestamps
66 |
67 | # ignore local files
68 | **/settings.py
69 | .env
70 | .env.*local
71 | .volumes
72 | **/logs
73 | /scripts/**/*.zip
74 |
75 | # Node modules
76 | node_modules/*
77 |
78 | # Coverage
79 | .coverage*
80 | coverage.xml
81 | htmlcov
82 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0031_alter_collection_extent_geometry_alter_item_geometry.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.6 on 2024-06-11 06:42
2 |
3 | import django.contrib.gis.db.models.fields
4 | from django.db import migrations
5 |
6 | import stac_api.validators
7 |
8 |
9 | class Migration(migrations.Migration):
10 |
11 | dependencies = [
12 | ('stac_api', '0030_alter_asset_media_type'),
13 | ]
14 |
15 | operations = [
16 | migrations.AlterField(
17 | model_name='collection',
18 | name='extent_geometry',
19 | field=django.contrib.gis.db.models.fields.GeometryField(
20 | blank=True, default=None, editable=False, null=True, srid=4326
21 | ),
22 | ),
23 | migrations.AlterField(
24 | model_name='item',
25 | name='geometry',
26 | field=django.contrib.gis.db.models.fields.GeometryField(
27 | default=
28 | 'SRID=4326;POLYGON ((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))',
29 | srid=4326,
30 | validators=[stac_api.validators.validate_geometry]
31 | ),
32 | ),
33 | ]
34 |
--------------------------------------------------------------------------------
/spec/transaction/tags.yaml:
--------------------------------------------------------------------------------
1 | openapi: 3.0.3
2 | tags:
3 | - name: Capabilities
4 | - name: Data
5 | - name: STAC
6 | - name: Data Management
7 | description: |
8 | The endpoints in this section create, update or delete STAC metadata.
9 |
10 | See the [Tech Docs](https://docs.geo.admin.ch/download-data/stac-api/overview.html) for more details
11 | on authentication and supported media types.
12 |
13 | - name: Asset Upload Management
14 | description: |
15 | The endpoints in this section manage the process of uploading assets to the STAC catalog.
16 |
17 | See the [Tech Docs](https://docs.geo.admin.ch/download-data/stac-api/overview.html) for examples and
18 | more details on authentication and compression.
19 |
20 | - name: Collection Asset Upload Management
21 | description: |
22 | The endpoints in this section manage the process of uploading collection assets to the STAC catalog.
23 | The flow of the requests is the same as for assets that belong to features.
24 |
25 | See the [Tech Docs](https://docs.geo.admin.ch/download-data/stac-api/overview.html) for examples and
26 | more details on authentication and compression.
27 |
--------------------------------------------------------------------------------
/doc/stac-search-endpoint-example.md:
--------------------------------------------------------------------------------
1 | # STAC POST /search endpoint example
2 |
3 | ## Disclaimer
4 |
5 | This Readme file and the according Jupyter notebook might not stay in the service-stac repository but probably will be moved into another repo in the mid-term.
6 |
7 | ## Jupyter notebook
8 |
9 | This [jupyter notebook](./assets/stac-search-endpoint-example.ipynb) serves as a simple example for using the STAC API's POST /search endpoint. For the sake of readability, some best practices, such as proper error handling, checking checksums of the downloaded files, and the like, are not (fully) implemented in this notebook.
10 |
11 | ## Preparation
12 |
13 | For this notebook to work, you need a recent Python version installed. You further need to install the following dependencies, e.g. by running `pip install folium geopandas jupyter mapclassify matplotlib pandas requests xyzservices` or by creating a virtual environment, where those dependencies are installed.
14 |
15 | ## Jupyter notebooks
16 |
17 | Please refer to the documentation on how to run Jupyter notebooks on your machine: https://docs.jupyter.org/en/latest/running.html
18 | Some IDEs support Jupyter notebooks, when the according extensions are installed.
19 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0055_alter_asset_file_size_alter_assetupload_file_size_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2024-11-28 16:06
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 | dependencies = [
9 | ("stac_api", "0054_update_conformance_endpoint"),
10 | ]
11 |
12 | operations = [
13 | migrations.AlterField(
14 | model_name="asset",
15 | name="file_size",
16 | field=models.BigIntegerField(blank=True, default=0, null=True),
17 | ),
18 | migrations.AlterField(
19 | model_name="assetupload",
20 | name="file_size",
21 | field=models.BigIntegerField(blank=True, default=0, null=True),
22 | ),
23 | migrations.AlterField(
24 | model_name="collectionasset",
25 | name="file_size",
26 | field=models.BigIntegerField(blank=True, default=0, null=True),
27 | ),
28 | migrations.AlterField(
29 | model_name="collectionassetupload",
30 | name="file_size",
31 | field=models.BigIntegerField(blank=True, default=0, null=True),
32 | ),
33 | ]
34 |
--------------------------------------------------------------------------------
/app/config/version.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import subprocess
3 |
4 | # Note that this file is overwritten during the build
5 | # process by either the git tag of the commit
6 | # (see Dockerfile for details)
7 |
8 | # By default we expect to find a leightweight tag in
9 | # the history
10 | # This has the form 'v[0-9]+\.[0-9]+\.[0-9]+-beta.[0-9]' if
11 | # the tag is directly related to the commit or has an additional
12 | # suffix 'v[0-9]+\.[0-9]+\.[0-9]+-beta.[0-9]-[0-9]+-gHASH' denoting
13 | # the 'distance' to the latest tag
14 | with subprocess.Popen(["git", "describe", "--tags"], stdout=subprocess.PIPE,
15 | stderr=subprocess.PIPE) as proc:
16 | stdout, stderr = proc.communicate()
17 | GIT_VERSION = stdout.decode('utf-8').strip()
18 | if GIT_VERSION == '':
19 | # If theres no git tag found in the history we simply use the short
20 | # version of the latest git commit hash
21 | with subprocess.Popen(["git", "rev-parse", "--short", "HEAD"],
22 | stdout=subprocess.PIPE,
23 | stderr=subprocess.PIPE) as proc:
24 | stdout, stderr = proc.communicate()
25 | APP_VERSION = f"v_{stdout.decode('utf-8').strip()}"
26 | else:
27 | APP_VERSION = GIT_VERSION
28 |
--------------------------------------------------------------------------------
/app/stac_api/exceptions.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | from django.utils.translation import gettext_lazy as _
4 |
5 | import rest_framework.exceptions
6 | from rest_framework import status
7 |
8 | logger = logging.getLogger(__name__)
9 |
10 |
11 | class StacAPIException(rest_framework.exceptions.APIException):
12 | '''STAC API custom exception
13 |
14 | These exception can add additional data to the HTTP response.
15 | '''
16 |
17 | def __init__(self, detail=None, code=None, data=None):
18 | super().__init__(detail, code)
19 | if isinstance(data, dict):
20 | self.data = data
21 | elif data:
22 | self.data = {'data': data}
23 |
24 |
25 | class UploadNotInProgressError(StacAPIException):
26 | status_code = status.HTTP_409_CONFLICT
27 | default_detail = _('No upload in progress')
28 | default_code = 'conflict'
29 |
30 |
31 | class UploadInProgressError(StacAPIException):
32 | status_code = status.HTTP_409_CONFLICT
33 | default_detail = _('Upload already in progress')
34 | default_code = 'conflict'
35 |
36 |
37 | class NotImplementedException(StacAPIException):
38 | status_code = status.HTTP_501_NOT_IMPLEMENTED
39 | default_detail = _('Not Implemented')
40 | default_code = 'not_implemented'
41 |
--------------------------------------------------------------------------------
/app/stac_api/management/commands/populate_testdb.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from django.conf import settings
4 |
5 | from stac_api.sample_data import importer
6 | from stac_api.utils import CustomBaseCommand
7 |
8 | # path definition relative to the directory that contains manage.py
9 | DATADIR = settings.BASE_DIR / 'app/stac_api/sample_data/'
10 |
11 |
12 | class Command(CustomBaseCommand):
13 | help = """Populates the local test database with sample data
14 |
15 | The sample data has to be located in stac_api/management/sample_data and
16 | structured as follows
17 | /
18 | |- items/
19 | |- .json
20 | |- .json
21 | |- collection.json
22 | """
23 |
24 | def handle(self, *args, **options):
25 | # loop over the collection directories inside sample_data
26 | for collection_dir in os.scandir(DATADIR):
27 | if collection_dir.is_dir() and not collection_dir.name.startswith('_'):
28 | self.print('Import collection %s', collection_dir.name, level=1)
29 | importer.import_collection(collection_dir)
30 | else:
31 | self.print('Ignore file %s', collection_dir.name, level=2)
32 | self.print_success('Done')
33 |
--------------------------------------------------------------------------------
/.style.yapf:
--------------------------------------------------------------------------------
1 | [style]
2 | based_on_style=google
3 | # Put closing brackets on a separate line, dedented, if the bracketed
4 | # expression can't fit in a single line. Applies to all kinds of brackets,
5 | # including function definitions and calls. For example:
6 | #
7 | # config = {
8 | # 'key1': 'value1',
9 | # 'key2': 'value2',
10 | # } # <--- this bracket is dedented and on a separate line
11 | #
12 | # time_series = self.remote_client.query_entity_counters(
13 | # entity='dev3246.region1',
14 | # key='dns.query_latency_tcp',
15 | # transform=Transformation.AVERAGE(window=timedelta(seconds=60)),
16 | # start_ts=now()-timedelta(days=3),
17 | # end_ts=now(),
18 | # ) # <--- this bracket is dedented and on a separate line
19 | dedent_closing_brackets=True
20 | coalesce_brackets=True
21 |
22 | # This avoid issues with complex dictionary
23 | # see https://github.com/google/yapf/issues/392#issuecomment-407958737
24 | indent_dictionary_value=True
25 | allow_split_before_dict_value=False
26 |
27 | # Split before arguments, but do not split all sub expressions recursively
28 | # (unless needed).
29 | split_all_top_level_comma_separated_values=True
30 |
31 | # Split lines longer than 100 characters (this only applies to code not to
32 | # comment and docstring)
33 | column_limit=100
34 |
--------------------------------------------------------------------------------
/Pipfile:
--------------------------------------------------------------------------------
1 | [[source]]
2 | name = "pypi"
3 | url = "https://pypi.org/simple"
4 | verify_ssl = true
5 |
6 | [dev-packages]
7 | yapf = "*"
8 | isort = "*"
9 | pylint = "*"
10 | pylint-django = "*"
11 | django-extensions = "*"
12 | django_debug_toolbar = "*"
13 | pip = "*"
14 | tblib = "*" # needed for traceback when running tests in parallel
15 | mock = "*"
16 | responses = "*"
17 | requests-mock = "*"
18 | moto = {extras = [ "s3",], version = "*"}
19 | debugpy = "*"
20 | parameterized = "*"
21 | stac-api-validator = "*"
22 | pytest = "*"
23 | pytest-django = "*"
24 | pytest-dotenv = "*"
25 | pytest-cov = "*"
26 | pytest-xdist = "*"
27 |
28 | [packages]
29 | gevent = "~=25.5"
30 | gunicorn = "~=23.0"
31 | psycopg = {extras = ["binary"], version = "~=3.2"}
32 | numpy = "~=2.3"
33 | python-dotenv = "~=1.1"
34 | djangorestframework = "~=3.16"
35 | Django = "~=5.2"
36 | PyYAML = "~=6.0"
37 | whitenoise = "~=6.9"
38 | djangorestframework-gis = "~=1.2"
39 | python-dateutil = "~=2.9"
40 | django-storages = "~=1.14"
41 | boto3 = "~=1.38"
42 | django-rest-framework-condition = "~=0.1"
43 | requests = "~=2.32"
44 | py-multihash = "~=2.0"
45 | django-prometheus = "~=2.3"
46 | django-admin-autocomplete-filter = "~=0.7"
47 | django-pgtrigger = "~=4.15"
48 | django-environ = "~=0.12"
49 | language-tags = "~=1.2"
50 | logging-utilities = "~=5.0"
51 |
52 | [requires]
53 | python_version = "3.12"
54 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0062_item_item_fc_reference_datetime_idx_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.11 on 2025-01-27 08:22
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0061_remove_item_forecast_mode_remove_item_forecast_param_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddIndex(
15 | model_name='item',
16 | index=models.Index(
17 | fields=['forecast_reference_datetime'], name='item_fc_reference_datetime_idx'
18 | ),
19 | ),
20 | migrations.AddIndex(
21 | model_name='item',
22 | index=models.Index(fields=['forecast_horizon'], name='item_fc_horizon_idx'),
23 | ),
24 | migrations.AddIndex(
25 | model_name='item',
26 | index=models.Index(fields=['forecast_duration'], name='item_fc_duration_idx'),
27 | ),
28 | migrations.AddIndex(
29 | model_name='item',
30 | index=models.Index(fields=['forecast_variable'], name='item_fc_variable_idx'),
31 | ),
32 | migrations.AddIndex(
33 | model_name='item',
34 | index=models.Index(fields=['forecast_perturbed'], name='item_fc_perturbed_idx'),
35 | ),
36 | ]
37 |
--------------------------------------------------------------------------------
/app/middleware/cors.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | from django.conf import settings
4 |
5 | logger = logging.getLogger(__name__)
6 |
7 | STAC_BASE = settings.STAC_BASE
8 |
9 |
10 | class CORSHeadersMiddleware:
11 | '''Middleware that adds appropriate CORS headers.
12 |
13 | CORS has only effect on browser applications (e.g. STAC browser),
14 | not on other systems. Therefore we only allow GET and HEAD requests
15 | on all endpoints except /search.
16 | '''
17 |
18 | def __init__(self, get_response):
19 | self.get_response = get_response
20 |
21 | def __call__(self, request):
22 | # Code to be executed for each request before
23 | # the view (and later middleware) are called.
24 |
25 | response = self.get_response(request)
26 |
27 | # Code to be executed for each request/response after
28 | # the view is called.
29 | response['Access-Control-Allow-Origin'] = '*'
30 | # Access-Control-Allow-Methods:
31 | allow_methods = ['GET', 'HEAD']
32 | # For /search we allow POST as well
33 | if request.path in (f'/{STAC_BASE}/v0.9/search', f'/{STAC_BASE}/v1/search'):
34 | allow_methods.append('POST')
35 | response['Access-Control-Allow-Methods'] = ','.join(allow_methods)
36 | response['Access-Control-Allow-Headers'] = 'Content-Type,Accept,Accept-Language'
37 |
38 | return response
39 |
--------------------------------------------------------------------------------
/scripts/create_dummy_asset_for_multipart_testing.py:
--------------------------------------------------------------------------------
1 | # This will create a dummy asset file for testing the multipart upload via the API.
2 |
3 | import argparse
4 | import gzip
5 | import os
6 |
7 |
8 | def get_args():
9 | parser = argparse.ArgumentParser(
10 | description="This script creates a dummy asset file for testing the multipart upload via " \
11 | "the API. File size of the original file is specified via argument. The file will be " \
12 | "zipped afterwards, hence the final file size is smaller than the specified size " \
13 | "of the unzipped dummy asset file."
14 | )
15 | parser.add_argument(
16 | "--size",
17 | type=int,
18 | default=20,
19 | help="Size of the unzipped dummy asset file in MB [Integer, default: 20 MB]"
20 | )
21 |
22 | args = parser.parse_args()
23 |
24 | return args
25 |
26 |
27 | def main():
28 | # parsing the args and defining variables
29 | args = get_args()
30 |
31 | FILE_SIZE = args.size * 1024**2
32 | with open("dummy_asset_for_multipart_testing.zip", "wb") as dummy_file:
33 | # compresslevel=1 will result in a low compression level, hence large .zip file size, which is
34 | # desired for testing here.
35 | dummy_file.write(gzip.compress(os.urandom(FILE_SIZE), compresslevel=1))
36 |
37 |
38 | if __name__ == '__main__':
39 | main()
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | services:
2 | db:
3 | image: kartoza/postgis:16
4 | environment:
5 | - POSTGRES_DB=${DB_NAME:-service_stac_local}
6 | - POSTGRES_USER=postgres
7 | - POSTGRES_PASSWORD=postgres
8 | - POSTGRES_MULTIPLE_EXTENSIONS=postgis,postgis_topology
9 | - EXTRA_CONF=log_min_messages = ${DB_LOG_LEVEL:-FATAL}
10 | user: ${UID}
11 | ports:
12 | - ${DB_PORT:-15432}:5432
13 | volumes:
14 | - type: bind
15 | source: ${PWD}/.volumes/postgresql
16 | target: /var/lib/postgresql
17 | s3:
18 | image: minio/minio
19 | env_file: ./minio.env
20 | user: ${UID}
21 | command: server /data --console-address ":9001"
22 | volumes:
23 | - type: bind
24 | source: ${PWD}/.volumes/minio
25 | target: /data
26 | ports:
27 | - 9090:${S3_PORT:-9000}
28 | - 9001:9001
29 | s3-client:
30 | image: minio/mc
31 | links:
32 | - s3
33 | env_file: ./minio.env
34 | restart: on-failure
35 | entrypoint: >
36 | /bin/sh -c "
37 | set +o history;
38 | while ! echo > /dev/tcp/s3/${S3_PORT:-9000};
39 | do
40 | echo waiting for minio;
41 | sleep 1;
42 | done;
43 | echo minio server is up;
44 | /usr/bin/mc alias set minio http://s3:${S3_PORT:-9000} minioadmin minioadmin;
45 | /usr/bin/mc policy set download minio/service-stac-local;
46 | exit 0;
47 | "
48 |
--------------------------------------------------------------------------------
/app/config/settings_test.py:
--------------------------------------------------------------------------------
1 | """
2 | The special test settings file ensures that the moto
3 | mock is imported before anything else and can mock the boto3
4 | stuff right from the beginning.
5 | Note: Don't change the order of the first three lines, nothing
6 | else must be imported before the s3mock is started!
7 |
8 | isort:skip_file
9 | """
10 | # pylint: disable=wildcard-import,unused-wildcard-import
11 | from config.settings import *
12 |
13 | AWS_SETTINGS = {
14 | 'legacy': {
15 | "access_type": "key",
16 | "ACCESS_KEY_ID": 'my-key',
17 | "SECRET_ACCESS_KEY": 'my-key',
18 | "DEFAULT_ACL": 'public-read',
19 | "S3_REGION_NAME": 'wonderland',
20 | "S3_ENDPOINT_URL": None,
21 | "S3_CUSTOM_DOMAIN": 'testserver',
22 | "S3_BUCKET_NAME": 'legacy',
23 | "S3_SIGNATURE_VERSION": "s3v4"
24 | },
25 | "managed": {
26 | "access_type": "service_account",
27 | "DEFAULT_ACL": 'public-read',
28 | "ROLE_ARN": 'Arnold',
29 | "S3_REGION_NAME": 'wonderland',
30 | "S3_ENDPOINT_URL": None,
31 | "S3_CUSTOM_DOMAIN": 'testserver',
32 | "S3_BUCKET_NAME": 'managed',
33 | "S3_SIGNATURE_VERSION": "s3v4"
34 | }
35 | }
36 |
37 | try:
38 | EXTERNAL_TEST_ASSET_URL = env('EXTERNAL_TEST_ASSET_URL')
39 | EXTERNAL_TEST_ASSET_URL_2 = env('EXTERNAL_TEST_ASSET_URL_2')
40 | except KeyError as err:
41 | raise KeyError('External asset URL must be set for unit testing') from err
42 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0015_data_collection_summaries.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.13 on 2021-08-18 06:51
2 |
3 | from django.db import migrations
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('stac_api', '0014_auto_20210715_1358'),
10 | ]
11 |
12 | operations = [
13 | migrations.RunSQL(
14 | '''
15 | WITH collection_summaries AS (
16 | SELECT
17 | item.collection_id AS collection_id,
18 | array_remove(array_agg(DISTINCT(asset.proj_epsg)), null) AS proj_epsg,
19 | array_remove(array_agg(DISTINCT(asset.geoadmin_variant)), null) AS geoadmin_variant,
20 | array_remove(array_agg(DISTINCT(asset.eo_gsd)), null) AS eo_gsd
21 | FROM stac_api_item AS item
22 | LEFT JOIN stac_api_asset AS asset ON (asset.item_id = item.id)
23 | GROUP BY item.collection_id
24 | )
25 | UPDATE stac_api_collection
26 | SET
27 | summaries_proj_epsg = collection_summaries.proj_epsg,
28 | summaries_geoadmin_variant = collection_summaries.geoadmin_variant,
29 | summaries_eo_gsd = collection_summaries.eo_gsd
30 | FROM collection_summaries
31 | WHERE collection_summaries.collection_id = stac_api_collection.id;
32 | ''',
33 | reverse_sql=migrations.RunSQL.noop
34 | )
35 | ]
36 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0060_item_forecast_perturbed_item_forecast_variable.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2025-01-15 13:08
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0059_alter_asset_media_type_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='item',
16 | name='forecast_perturbed',
17 | field=models.BooleanField(
18 | blank=True,
19 | default=None,
20 | help_text=
21 | 'Denotes whether the data corresponds to the control run (`false`) or perturbed runs (`true`). The property needs to be specified in both cases as no default value is specified and as such the meaning is "unknown" in case it\'s missing.',
22 | null=True
23 | ),
24 | ),
25 | migrations.AddField(
26 | model_name='item',
27 | name='forecast_variable',
28 | field=models.CharField(
29 | blank=True,
30 | help_text=
31 | 'Name of the model variable that corresponds to the data. The variables should correspond to the [CF Standard Names](https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html), e.g. `air_temperature` for the air temperature.',
32 | null=True
33 | ),
34 | ),
35 | ]
36 |
--------------------------------------------------------------------------------
/app/middleware/api_gateway.py:
--------------------------------------------------------------------------------
1 | REMOTE_USER_HEADER = "HTTP_GEOADMIN_USERNAME"
2 |
3 |
4 | def validate_username_header(request):
5 | """Drop the Geoadmin-Username header if it's invalid.
6 |
7 | This should be called before making any decision based on the value of the
8 | Geoadmin-Username header.
9 |
10 | API Gateway always sends the Geoadmin-Username header regardless of
11 | whether it was able to authenticate the user. If it could not
12 | authenticate the user, the value of the header as seen on the wire is a
13 | single whitespace. An hexdump looks like this:
14 |
15 | 47 65 6f 61 64 6d 69 6e 5f 75 73 65 72 6e 61 6d 65 3a 20 0d 0a
16 | Geoadmin-Username:...
17 |
18 | This doesn't seem possible to reproduce with curl. It is possible to
19 | reproduce with wget. It is unclear whether that technically counts as an
20 | empty value or a whitespace. It is also possible that AWS change their
21 | implementation later to send something slightly different. Regardless,
22 | we already have a separate signal to tell us whether that value is
23 | valid: Geoadmin-Authenticated. So we only consider Geoadmin-Username if
24 | Geoadmin-Authenticated is set to "true".
25 |
26 | Based on discussion in https://code.djangoproject.com/ticket/35971
27 | """
28 | apigw_auth = request.META.get("HTTP_GEOADMIN_AUTHENTICATED", "false").lower() == "true"
29 | if not apigw_auth and REMOTE_USER_HEADER in request.META:
30 | del request.META[REMOTE_USER_HEADER]
31 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0011_auto_20210623_0521.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.10 on 2021-06-23 05:21
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('stac_api', '0010_auto_20210621_1453'),
11 | ]
12 |
13 | operations = [
14 | migrations.AlterField(
15 | model_name='item',
16 | name='properties_datetime',
17 | field=models.DateTimeField(
18 | blank=True,
19 | help_text=
20 | 'Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format',
21 | null=True
22 | ),
23 | ),
24 | migrations.AlterField(
25 | model_name='item',
26 | name='properties_end_datetime',
27 | field=models.DateTimeField(
28 | blank=True,
29 | help_text=
30 | 'Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format',
31 | null=True
32 | ),
33 | ),
34 | migrations.AlterField(
35 | model_name='item',
36 | name='properties_start_datetime',
37 | field=models.DateTimeField(
38 | blank=True,
39 | help_text=
40 | 'Enter date in yyyy-mm-dd format, and time in UTC hh:mm:ss format',
41 | null=True
42 | ),
43 | ),
44 | ]
45 |
--------------------------------------------------------------------------------
/app/middleware/rest_framework_authentication.py:
--------------------------------------------------------------------------------
1 | from django.conf import settings
2 |
3 | from rest_framework.authentication import BasicAuthentication
4 | from rest_framework.authentication import SessionAuthentication
5 | from rest_framework.authentication import TokenAuthentication
6 |
7 |
8 | class RestrictedBasicAuthentication(BasicAuthentication):
9 | """ A Django rest framework basic authentication class that skips v1 of STAC API. """
10 |
11 | def authenticate(self, request):
12 | if settings.FEATURE_AUTH_RESTRICT_V1 and request.path.startswith(
13 | f'/{settings.STAC_BASE}/v1/'
14 | ):
15 | return None
16 |
17 | return super().authenticate(request)
18 |
19 |
20 | class RestrictedSessionAuthentication(SessionAuthentication):
21 | """ A Django rest framework session authentication class that skips v1 of STAC API. """
22 |
23 | def authenticate(self, request):
24 | if settings.FEATURE_AUTH_RESTRICT_V1 and request.path.startswith(
25 | f'/{settings.STAC_BASE}/v1/'
26 | ):
27 | return None
28 |
29 | return super().authenticate(request)
30 |
31 |
32 | class RestrictedTokenAuthentication(TokenAuthentication):
33 | """ A Django rest framework token authentication class that skips v1 of STAC API. """
34 |
35 | def authenticate(self, request):
36 | if settings.FEATURE_AUTH_RESTRICT_V1 and request.path.startswith(
37 | f'/{settings.STAC_BASE}/v1/'
38 | ):
39 | return None
40 |
41 | return super().authenticate(request)
42 |
--------------------------------------------------------------------------------
/adr/2020_10_21_static_asset.md:
--------------------------------------------------------------------------------
1 | # Static Assets
2 |
3 | > `Status: accepted`
4 |
5 | > `Date: 2020-10-21`
6 |
7 | > `Participants: Brice Schaffner, Christoph Böcklin`
8 |
9 | ## Context
10 |
11 | `service-stac` needs to serve some static assets for the admin pages (css, images, icons, ...). Django is not appropriate to serve static files on production environment. Currently Django is served directly by `gunicorn`. As a good practice to avoid issue with slow client and to avoid Denial of Service attacks, `gunicorn` should be served behind a Reversed proxy (e.g. Apache or Nginx).
12 |
13 | ## Decision
14 |
15 | Because it is to us not clear yet if a Reverse Proxy is really necessary for our Architecture (CloudFront with Kubernetes Ingress), we decided to use WhiteNoise for static assets. This middleware seems to performs well with CDN (like CloudFront) therefore we will use it to serve static files as it is very simple to uses and take care of compressing and settings corrects Headers for caching.
16 |
17 | ## Consequences
18 |
19 | We might have to reconsider in future using a Reverse Proxy for `gunicorn` and then uses this proxy (e.g. nginx) to serve static files instead of Whitenoise.
20 |
21 | ## References
22 |
23 | - [Static Files Made Easy — Django Compressor + Whitenoise + AWS CloudFront + Heroku](https://medium.com/technolingo/fastest-static-files-served-django-compressor-whitenoise-aws-cloudfront-ef777849090c)
24 | - [Isn’t serving static files from Python horribly inefficient?](https://whitenoise.readthedocs.io/en/stable/index.html#isn-t-serving-static-files-from-python-horribly-inefficient)
25 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0032_delete_conformancepage_landingpage_conformsto_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.6 on 2024-07-04 11:46
2 |
3 | import django.contrib.postgres.fields
4 | from django.db import migrations
5 | from django.db import models
6 |
7 | import stac_api.models.general
8 | import stac_api.validators
9 |
10 |
11 | class Migration(migrations.Migration):
12 |
13 | dependencies = [
14 | ('stac_api', '0031_alter_collection_extent_geometry_alter_item_geometry'),
15 | ]
16 |
17 | operations = [
18 | migrations.DeleteModel(name='ConformancePage',),
19 | migrations.AddField(
20 | model_name='landingpage',
21 | name='conformsTo',
22 | field=django.contrib.postgres.fields.ArrayField(
23 | base_field=models.URLField(),
24 | default=stac_api.models.general.get_conformance_default_links,
25 | help_text='Comma-separated list of URLs for the value conformsTo',
26 | size=None
27 | ),
28 | ),
29 | migrations.AddField(
30 | model_name='landingpage',
31 | name='version',
32 | field=models.CharField(default='v1', max_length=255),
33 | ),
34 | migrations.AlterField(
35 | model_name='landingpage',
36 | name='name',
37 | field=models.CharField(
38 | default='ch',
39 | max_length=255,
40 | validators=[stac_api.validators.validate_name],
41 | verbose_name='id'
42 | ),
43 | ),
44 | ]
45 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0051_asset_file_size_assetupload_file_size_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.8 on 2024-10-08 13:07
2 |
3 | from django.db import migrations
4 | from django.db import models
5 |
6 |
7 | class Migration(migrations.Migration):
8 | dependencies = [
9 | ("stac_api", "0050_collectionassetupload_and_more"),
10 | ]
11 |
12 | operations = [
13 | migrations.AddField(
14 | model_name="asset",
15 | name="file_size",
16 | field=models.IntegerField(blank=True, default=0, null=True),
17 | ),
18 | migrations.AddField(
19 | model_name="assetupload",
20 | name="file_size",
21 | field=models.IntegerField(blank=True, default=0, null=True),
22 | ),
23 | migrations.AddField(
24 | model_name="collection",
25 | name="total_data_size",
26 | field=models.IntegerField(blank=True, default=0, null=True),
27 | ),
28 | migrations.AddField(
29 | model_name="collectionasset",
30 | name="file_size",
31 | field=models.IntegerField(blank=True, default=0, null=True),
32 | ),
33 | migrations.AddField(
34 | model_name="collectionassetupload",
35 | name="file_size",
36 | field=models.IntegerField(blank=True, default=0, null=True),
37 | ),
38 | migrations.AddField(
39 | model_name="item",
40 | name="total_data_size",
41 | field=models.IntegerField(blank=True, default=0, null=True),
42 | ),
43 | ]
44 |
--------------------------------------------------------------------------------
/app/stac_api/views/test.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | from rest_framework import generics
4 |
5 | from stac_api.models.general import LandingPage
6 | from stac_api.views.collection import CollectionAssetDetail
7 | from stac_api.views.collection import CollectionDetail
8 | from stac_api.views.item import AssetDetail
9 | from stac_api.views.item import ItemDetail
10 |
11 | logger = logging.getLogger(__name__)
12 |
13 |
14 | class TestHttp500(generics.GenericAPIView):
15 | queryset = LandingPage.objects.all()
16 |
17 | def get(self, request, *args, **kwargs):
18 | logger.debug('Test request that raises an exception')
19 |
20 | raise AttributeError('test exception')
21 |
22 |
23 | class TestCollectionUpsertHttp500(CollectionDetail):
24 |
25 | def perform_upsert(self, serializer, lookup):
26 | super().perform_upsert(serializer, lookup)
27 | raise AttributeError('test exception')
28 |
29 |
30 | class TestItemUpsertHttp500(ItemDetail):
31 |
32 | def perform_upsert(self, serializer, lookup):
33 | super().perform_upsert(serializer, lookup)
34 |
35 | raise AttributeError('test exception')
36 |
37 |
38 | class TestAssetUpsertHttp500(AssetDetail):
39 |
40 | def perform_upsert(self, serializer, lookup):
41 | super().perform_upsert(serializer, lookup)
42 | raise AttributeError('test exception')
43 |
44 |
45 | class TestCollectionAssetUpsertHttp500(CollectionAssetDetail):
46 |
47 | def perform_upsert(self, serializer, lookup):
48 | super().perform_upsert(serializer, lookup)
49 | raise AttributeError('test exception')
50 |
--------------------------------------------------------------------------------
/.env.default:
--------------------------------------------------------------------------------
1 | HTTP_PORT=8000
2 | LOGS_DIR=logs
3 | DB_USER=postgres
4 | DB_PW=postgres
5 | DB_NAME=service_stac_local
6 | DB_NAME_TEST=test_service_stac_local
7 | DB_HOST=localhost
8 | DB_PORT=15432
9 | DEBUG=True
10 | DEBUG_PROPAGATE_API_EXCEPTIONS=False
11 |
12 | LEGACY_AWS_ACCESS_KEY_ID=minioadmin
13 | LEGACY_AWS_SECRET_ACCESS_KEY=minioadmin
14 |
15 | LEGACY_AWS_S3_BUCKET_NAME=service-stac-local
16 | LEGACY_AWS_S3_REGION_NAME=eu-central-1
17 | LEGACY_AWS_S3_ENDPOINT_URL=http://127.0.0.1:9090
18 | LEGACY_AWS_S3_CUSTOM_DOMAIN=127.0.0.1:9090/service-stac-local
19 | SECRET_KEY=dummy
20 | HEALTHCHECK_ENDPOINT=healthcheck
21 | ALLOWED_HOSTS=*
22 | MANAGED_BUCKET_COLLECTION_PATTERNS=ch.meteoschweiz.ogd-,ch.bgdi-test.
23 |
24 | # these are just here for completeness
25 | AWS_ROLE_ARN=some-arn
26 | AWS_WEB_IDENTITY_TOKEN_FILE=token
27 |
28 | AWS_S3_BUCKET_NAME=service-stac-local-managed
29 | AWS_S3_REGION_NAME=eu-central-1
30 | AWS_S3_ENDPOINT_URL=http://127.0.0.1:9090
31 | AWS_S3_CUSTOM_DOMAIN=127.0.0.1:9090/service-stac-local
32 |
33 | # SET THIS value to a URL that points to a jpeg file that is publicly
34 | # reachable in the web
35 | # this will be used by the unit tests for external assets
36 | EXTERNAL_TEST_ASSET_URL=https://prod-swisstopoch-hcms-sdweb.imgix.net/2024/07/04/04d3aafe-99b1-4589-934c-337583eb5564.jpeg
37 | EXTERNAL_TEST_ASSET_URL_2=https://prod-swisstopoch-hcms-sdweb.imgix.net/2023/11/14/606881aa-ab86-47f8-8067-cd360f6b48b5.jpg
38 |
39 | # Those settings are required in order to run E2E tests against the localhost
40 | SESSION_EXPIRE_AT_BROWSER_CLOSE=False
41 | SESSION_COOKIE_AGE=28800
42 | SESSION_COOKIE_SECURE=False
43 |
--------------------------------------------------------------------------------
/app/tests/tests_09/test_landing_page.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 |
3 | from tests.tests_09.base_test import STAC_BASE_V
4 | from tests.tests_09.base_test import STAC_VERSION
5 | from tests.tests_09.base_test import StacBaseTestCase
6 |
7 |
8 | class IndexTestCase(StacBaseTestCase):
9 |
10 | def setUp(self):
11 | self.client = Client()
12 |
13 | def test_landing_page(self):
14 | response = self.client.get(f"/{STAC_BASE_V}/")
15 | self.assertEqual(response.status_code, 200)
16 | self.assertCors(response)
17 | self.assertEqual(response['Content-Type'], 'application/json')
18 | required_keys = ['description', 'id', 'stac_version', 'links']
19 | self.assertEqual(
20 | set(required_keys).difference(response.json().keys()),
21 | set(),
22 | msg="missing required attribute in json answer"
23 | )
24 | self.assertEqual(response.json()['id'], 'ch')
25 | self.assertEqual(response.json()['stac_version'], STAC_VERSION)
26 | for link in response.json()['links']:
27 | required_keys = ['href', 'rel']
28 | self.assertEqual(
29 | set(required_keys).difference(link.keys()),
30 | set(),
31 | msg="missing required attribute in the answer['links'] array"
32 | )
33 |
34 | def test_landing_page_redirect(self):
35 | response = self.client.get(f"/{STAC_BASE_V}")
36 | self.assertEqual(response.status_code, 301)
37 | self.assertLocationHeader(f"/{STAC_BASE_V}/", response)
38 | self.assertCors(response)
39 |
--------------------------------------------------------------------------------
/app/tests/tests_09/test_get_token_endpoint.py:
--------------------------------------------------------------------------------
1 | from django.contrib.auth import get_user_model
2 | from django.test import TestCase
3 | from django.urls import reverse
4 |
5 | from rest_framework.authtoken.models import Token
6 | from rest_framework.test import APIClient
7 |
8 | from tests.utils import get_http_error_description
9 |
10 |
11 | class GetTokenEndpointTestCase(TestCase):
12 |
13 | def setUp(self):
14 | self.client = APIClient()
15 | self.username = 'SherlockHolmes'
16 | self.password = '221B_BakerStreet'
17 | self.user = get_user_model().objects.create_user(
18 | self.username, 'top@secret.co.uk', self.password
19 | )
20 |
21 | def test_get_token_with_valid_credentials(self):
22 | url = reverse('get-token')
23 | response = self.client.post(url, {'username': self.username, 'password': self.password})
24 | self.assertEqual(200, response.status_code, msg=get_http_error_description(response.json()))
25 | generated_token = response.data["token"]
26 | token_from_db = Token.objects.get(user=self.user)
27 | self.assertEqual(
28 | generated_token,
29 | token_from_db.key,
30 | msg="Generated token and token stored in DB do not match."
31 | )
32 |
33 | def test_get_token_with_invalid_credentials(self):
34 | url = reverse('get-token')
35 | response = self.client.post(url, {'username': self.username, 'password': 'wrong_password'})
36 | self.assertEqual(
37 | 400, response.status_code, msg="Token for unauthorized user has been created."
38 | )
39 |
--------------------------------------------------------------------------------
/app/tests/tests_10/test_get_token_endpoint.py:
--------------------------------------------------------------------------------
1 | from django.contrib.auth import get_user_model
2 | from django.test import TestCase
3 | from django.urls import reverse
4 |
5 | from rest_framework.authtoken.models import Token
6 | from rest_framework.test import APIClient
7 |
8 | from tests.utils import get_http_error_description
9 |
10 |
11 | class GetTokenEndpointTestCase(TestCase):
12 |
13 | def setUp(self):
14 | self.client = APIClient()
15 | self.username = 'SherlockHolmes'
16 | self.password = '221B_BakerStreet'
17 | self.user = get_user_model().objects.create_user(
18 | self.username, 'top@secret.co.uk', self.password
19 | )
20 |
21 | def test_get_token_with_valid_credentials(self):
22 | url = reverse('get-token')
23 | response = self.client.post(url, {'username': self.username, 'password': self.password})
24 | self.assertEqual(200, response.status_code, msg=get_http_error_description(response.json()))
25 | generated_token = response.data["token"]
26 | token_from_db = Token.objects.get(user=self.user)
27 | self.assertEqual(
28 | generated_token,
29 | token_from_db.key,
30 | msg="Generated token and token stored in DB do not match."
31 | )
32 |
33 | def test_get_token_with_invalid_credentials(self):
34 | url = reverse('get-token')
35 | response = self.client.post(url, {'username': self.username, 'password': 'wrong_password'})
36 | self.assertEqual(
37 | 400, response.status_code, msg="Token for unauthorized user has been created."
38 | )
39 |
--------------------------------------------------------------------------------
/app/tests/tests_10/test_landing_page.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 |
3 | from tests.tests_10.base_test import STAC_BASE_V
4 | from tests.tests_10.base_test import STAC_VERSION
5 | from tests.tests_10.base_test import StacBaseTestCase
6 |
7 |
8 | class IndexTestCase(StacBaseTestCase):
9 |
10 | def setUp(self):
11 | self.client = Client()
12 |
13 | def test_landing_page(self):
14 | response = self.client.get(f"/{STAC_BASE_V}/")
15 | self.assertEqual(response.status_code, 200)
16 | self.assertCors(response)
17 | self.assertEqual(response['Content-Type'], 'application/json')
18 | required_keys = ['description', 'id', 'stac_version', 'links', 'type']
19 | self.assertEqual(
20 | set(required_keys).difference(response.json().keys()),
21 | set(),
22 | msg="missing required attribute in json answer"
23 | )
24 | self.assertEqual(response.json()['id'], 'ch')
25 | self.assertEqual(response.json()['stac_version'], STAC_VERSION)
26 | for link in response.json()['links']:
27 | required_keys = ['href', 'rel']
28 | self.assertEqual(
29 | set(required_keys).difference(link.keys()),
30 | set(),
31 | msg="missing required attribute in the answer['links'] array"
32 | )
33 |
34 | def test_landing_page_redirect(self):
35 | response = self.client.get(f"/{STAC_BASE_V}")
36 | self.assertEqual(response.status_code, 301)
37 | self.assertLocationHeader(f"/{STAC_BASE_V}/", response)
38 | self.assertCors(response)
39 |
--------------------------------------------------------------------------------
/spec/transaction/components/examples.yaml:
--------------------------------------------------------------------------------
1 | openapi: 3.0.3
2 | components:
3 | examples:
4 | inprogress:
5 | summary: In progress upload example
6 | value:
7 | upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YigDnuM06hfJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG
8 | status: in-progress
9 | number_parts: 1
10 | urls:
11 | - url: https://data.geo.admin.ch/ch.swisstopo.pixelkarte-farbe-pk50.noscale/smr200-200-4-2019/smr50-263-2016-2056-kgrs-2.5.tiff
12 | part: 1
13 | expires: '2019-08-24T14:15:22Z'
14 | created: '2019-08-24T14:15:22Z'
15 | file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC
16 | completed:
17 | summary: Completed upload example
18 | value:
19 | upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YigDnuM06hfJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG
20 | status: completed
21 | number_parts: 1
22 | created: '2019-08-24T14:15:22Z'
23 | completed: '2019-08-24T14:15:22Z'
24 | file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC
25 | aborted:
26 | summary: Aborted upload example
27 | value:
28 | upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YigDnuM06hfJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG
29 | status: completed
30 | number_parts: 1
31 | created: '2019-08-24T14:15:22Z'
32 | aborted: '2019-08-24T14:15:22Z'
33 | file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC
34 |
--------------------------------------------------------------------------------
/app/stac_api/management/commands/manage_superuser.py:
--------------------------------------------------------------------------------
1 | from typing import Any
2 |
3 | import environ
4 |
5 | from django.contrib.auth import get_user_model
6 |
7 | from stac_api.utils import CustomBaseCommand
8 |
9 | env = environ.Env()
10 |
11 |
12 | class Command(CustomBaseCommand):
13 | """Create or update superuser from information from the environment
14 |
15 | This command is used to make sure that the superuser is created and
16 | configured. The data for it will be created centrally in terraform.
17 | This will help with the password rotation.
18 | """
19 |
20 | help = "Superuser management (creating or updating)"
21 |
22 | def handle(self, *args: Any, **options: Any) -> None:
23 | User = get_user_model() # pylint: disable=invalid-name
24 | username = env.str('DJANGO_SUPERUSER_USERNAME', default='').strip()
25 | email = env.str('DJANGO_SUPERUSER_EMAIL', default='').strip()
26 | password = env.str('DJANGO_SUPERUSER_PASSWORD', default='').strip()
27 |
28 | if not username or not email or not password:
29 | self.print_error('Environment variables not set or empty')
30 | return
31 |
32 | try:
33 | admin = User.objects.get(username=username)
34 | operation = 'Updated'
35 | except User.DoesNotExist:
36 | admin = User.objects.create(username=username)
37 | operation = 'Created'
38 |
39 | admin.set_password(password)
40 | admin.email = email
41 | admin.is_staff = True
42 | admin.is_superuser = True
43 | admin.save()
44 |
45 | self.print_success('%s the superuser %s', operation, username)
46 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0067_update_conformance.py:
--------------------------------------------------------------------------------
1 | from django.db import migrations
2 |
3 |
4 | def update_conformance(apps, schema_editor):
5 | # Add item-search conformance
6 | LandingPage = apps.get_model("stac_api", "LandingPage")
7 | lp = LandingPage.objects.get(version='v1')
8 | lp.conformsTo = [
9 | 'https://api.stacspec.org/v1.0.0/core',
10 | 'https://api.stacspec.org/v1.0.0/collections',
11 | 'https://api.stacspec.org/v1.0.0/ogcapi-features',
12 | 'https://api.stacspec.org/v1.0.0/item-search',
13 | 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core',
14 | 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30',
15 | 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson',
16 | ]
17 | lp.save()
18 |
19 |
20 | def reverse_update_conformance(apps, schema_editor):
21 | # Remove item-search conformance
22 | LandingPage = apps.get_model("stac_api", "LandingPage")
23 | lp = LandingPage.objects.get(version='v1')
24 | lp.conformsTo = [
25 | 'https://api.stacspec.org/v1.0.0/core',
26 | 'https://api.stacspec.org/v1.0.0/collections',
27 | 'https://api.stacspec.org/v1.0.0/ogcapi-features',
28 | 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core',
29 | 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30',
30 | 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson',
31 | ]
32 | lp.save()
33 |
34 |
35 | class Migration(migrations.Migration):
36 | dependencies = [
37 | ("stac_api", "0066_collectionasset_is_external"),
38 | ]
39 |
40 | operations = [migrations.RunPython(update_conformance, reverse_update_conformance)]
41 |
--------------------------------------------------------------------------------
/app/middleware/api_gateway_middleware.py:
--------------------------------------------------------------------------------
1 | from middleware import api_gateway
2 |
3 | from django.conf import settings
4 | from django.contrib.auth.backends import RemoteUserBackend
5 | from django.contrib.auth.middleware import PersistentRemoteUserMiddleware
6 |
7 |
8 | class ApiGatewayMiddleware(PersistentRemoteUserMiddleware):
9 | """Persist user authentication based on the API Gateway headers."""
10 | header = api_gateway.REMOTE_USER_HEADER
11 |
12 | def process_request(self, request):
13 | if not settings.FEATURE_AUTH_ENABLE_APIGW:
14 | return None
15 |
16 | if request.path.startswith(f'/{settings.STAC_BASE}/v'):
17 | # API authentication is only done using ApiGatewayAuthentication to avoid creating a new
18 | # session with each request
19 | return None
20 |
21 | api_gateway.validate_username_header(request)
22 | return super().process_request(request)
23 |
24 |
25 | class ApiGatewayUserBackend(RemoteUserBackend):
26 | """ This backend is to be used in conjunction with the ``ApiGatewayMiddleware` and the
27 | ``ApiGatewayAuthentication``.
28 |
29 | Until proper authorization is implemented, all remote users authenticated via API Gateway
30 | headers are treated as superusers.
31 |
32 | """
33 |
34 | def authenticate(self, request, remote_user):
35 | if not settings.FEATURE_AUTH_ENABLE_APIGW:
36 | return None
37 |
38 | user = super().authenticate(request, remote_user)
39 | if user and not user.is_superuser:
40 | # promote authenticated user to superuser for now until proper authorization is
41 | # implemented
42 | user.is_superuser = True
43 | user.save()
44 | return user
45 |
--------------------------------------------------------------------------------
/app/templates/uploadtemplate.html:
--------------------------------------------------------------------------------
1 | {% extends "admin/base_site.html" %}
2 | {% block content %}
3 |
4 |
5 |
11 |
12 | {% load static %}
13 |
14 |
15 |
16 |
17 |
18 |
19 |
Upload file for {{ collection_name }}/{{ item_name }}/{{ asset_name }}
20 |
21 |
22 | ⓘ This file upload will use the multipart form upload API in the background.
23 | The file will be uploaded to S3 using a presigned url. Maximum file size is 536,870,903 bytes (approximately 512 MB).
24 |
25 |
26 | ⚠
27 | This file upload will use the multipart form upload API in the background.
28 | The file will be uploaded to s3 using a presigned url. Max. file size is 5GB
29 |
30 |
31 |
File upload
32 |
41 |
42 |
Current status
43 |
44 | ready to upload
45 |
46 |
47 | {% endblock %}
48 |
--------------------------------------------------------------------------------
/spec/README.md:
--------------------------------------------------------------------------------
1 | # Specs
2 |
3 | The API of this service is based on the [STAC API Specification](https://github.com/radiantearth/stac-api-spec) in version `0.9.0`, which itself is based on the [STAC Specification](https://github.com/radiantearth/stac-spec/tree/v0.9.0) and the [_OGC API - Features_](https://github.com/opengeospatial/ogcapi-features) specification. The default STAC spec is amended by geoadmin-specific parts that are explicitly mentioned in the spec, as well as adapted examples that resemble geoadmin-specific use cases.
4 |
5 | The spec is OpenAPI 3.0 compliant. The files are located in `spec/` and slightly split for better understanding. Two different versions of the spec can be compiled from these source files into `spec/static/` folder: an `openapi.yaml` file that contains the 'public' part with the REST endpoint and HTTP methods (mostly GET) defined in the standard spec, and an `openapitransactional.yaml` file that is intended for internal usage and contains info about the additional `/asset` endpoint and additional writing possibilities.
6 |
7 | The spec files can be compiled with
8 |
9 | ```bash
10 | make build-specs
11 | ```
12 |
13 | and previewed locally with the little html ReDoc wrappers `spec/static/api.html` and `spec/static/apitransactional.html`. The can be served locally by invoking
14 |
15 | ```bash
16 | make serve-specs
17 | ```
18 |
19 | which starts a simple http server that can be reached under `http://0.0.0.0:8090` (default port is `8090`, if you need another one, check the Makefile on how to do this).
20 |
21 | The generated files along with html wrappers are also included as static targets and are available under `https:///api/stac/v0.9/static/api.html`.
22 |
23 | Once the specs has been built they need to be validated with
24 |
25 | ```bash
26 | make lint-specs
27 | ```
28 |
--------------------------------------------------------------------------------
/spec/transaction/components/responses.yaml:
--------------------------------------------------------------------------------
1 | # openapi: 3.0.3
2 | components:
3 | responses:
4 | Assets:
5 | description: >-
6 | The response is a document consisting of all assets of the feature.
7 | content:
8 | application/json:
9 | schema:
10 | $ref: "./schemas.yaml#/components/schemas/assets"
11 | Asset:
12 | description: >-
13 | The response is a document consisting of one asset of the feature.
14 | headers:
15 | ETag:
16 | $ref: "../../components/headers.yaml#/components/headers/ETag"
17 | content:
18 | application/json:
19 | schema:
20 | $ref: "./schemas.yaml#/components/schemas/asset"
21 | DeletedResource:
22 | description: Status of the delete resource
23 | content:
24 | application/json:
25 | schema:
26 | description: >-
27 | Information about the deleted resource and a link to the parent resource
28 | type: object
29 | properties:
30 | code:
31 | type: integer
32 | example: 200
33 | description:
34 | type: string
35 | example: Resource successfully deleted
36 | links:
37 | type: array
38 | items:
39 | $ref: "../../components/schemas.yaml#/components/schemas/link"
40 | description: >-
41 | The array contain at least a link to the parent resource (`rel: parent`).
42 | example:
43 | - href: https://data.geo.admin.ch/api/stac/v1/collections/ch.swisstopo.pixelkarte-farbe-pk50.noscale
44 | rel: parent
45 | required:
46 | - code
47 | - links
48 |
--------------------------------------------------------------------------------
/update_to_latest.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # This script updates the Pipfile automatically. It will update all version strings of type
3 | # "~=x.x" to their respective latest version. Version strings of dependencies that use other
4 | # version specifiers (like "*") will be left untouched. All dependencies will be updated
5 | # (with "pipenv update") in the process.
6 | # A regex can be optionally specified as first argument. In this case, only the version strings
7 | # of packages matching the regex will be updated. (e.g. update_to_latest.sh 'django.*')
8 | # This script is meant as a helper only. Use git to revert unwanted changes made by this script.
9 |
10 | cd "$(dirname "$0")" || exit
11 | #If an argument was passed to the script, it will be used as a regular expression
12 | #Else we will simply match any package name
13 | regexp=${1:-'\S+'}
14 | line_regexp="^($regexp) = \"~=[0-9\\.]+\"(.*)$"
15 |
16 | #Generate an array of all packages that need to be updated and switch their version to "*"
17 | packages_to_modify=( $(cat Pipfile | sed -En "s/$line_regexp/\1/ip") )
18 | echo "The script will try to update the following packages: ${packages_to_modify[*]}"
19 | read -p "Do you want to continue? [Y|N] " -n 1 -r
20 | echo
21 | [[ ! $REPLY =~ ^[Yy]$ ]] && exit
22 | sed -Ei "s/$line_regexp/\1 = \"*\"\2/i" Pipfile
23 |
24 | # Update the packages to the latest version
25 | pipenv update --dev
26 |
27 | # Set the current version in the Pipfile (only for the packages that were updated)
28 | updateVersions=""
29 | while read -r name version ; do
30 | if [[ " ${packages_to_modify[*]} " == *" $name "* ]] ; then
31 | updateVersions+="/^$name =/s/\"\\*\"/$version/"
32 | updateVersions+=$'\n'
33 | fi
34 | done < <(pipenv run pip freeze | sed -E 's/==([0-9]+)\.([0-9]+)\.[0-9]+\w*/ "~=\1.\2"/')
35 | sed -Ei -f <(echo "$updateVersions") Pipfile
36 |
--------------------------------------------------------------------------------
/app/tests/tests_09/test_renderer.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 | from django.test import TestCase
3 |
4 | from tests.tests_09.base_test import STAC_BASE_V
5 |
6 |
7 | class RendererTestCase(TestCase):
8 |
9 | def setUp(self):
10 | self.client = Client()
11 |
12 | def test_content_type_default(self):
13 | response = self.client.get(f"/{STAC_BASE_V}/search")
14 | self.assertEqual(response.status_code, 200)
15 | self.assertEqual(response['Content-Type'], 'application/json')
16 | self.assertEqual(response.json()['type'], 'FeatureCollection')
17 |
18 | def test_content_type_json(self):
19 | response = self.client.get(f"/{STAC_BASE_V}/search?format=json")
20 | self.assertEqual(response.status_code, 200)
21 | self.assertEqual(response['Content-Type'], 'application/json')
22 | self.assertEqual(response.json()['type'], 'FeatureCollection')
23 |
24 | response = self.client.get(f"/{STAC_BASE_V}/search", headers={'Accept': 'application/json'})
25 | self.assertEqual(response.status_code, 200)
26 | self.assertEqual(response['Content-Type'], 'application/json')
27 | self.assertEqual(response.json()['type'], 'FeatureCollection')
28 |
29 | def test_content_type_geojson(self):
30 | response = self.client.get(f"/{STAC_BASE_V}/search?format=geojson")
31 | self.assertEqual(response.status_code, 200)
32 | self.assertEqual(response['Content-Type'], 'application/geo+json')
33 | self.assertEqual(response.json()['type'], 'FeatureCollection')
34 |
35 | response = self.client.get(
36 | f"/{STAC_BASE_V}/search", headers={'Accept': 'application/geo+json'}
37 | )
38 | self.assertEqual(response.status_code, 200)
39 | self.assertEqual(response['Content-Type'], 'application/geo+json')
40 | self.assertEqual(response.json()['type'], 'FeatureCollection')
41 |
--------------------------------------------------------------------------------
/app/tests/tests_10/test_renderer.py:
--------------------------------------------------------------------------------
1 | from django.test import Client
2 | from django.test import TestCase
3 |
4 | from tests.tests_10.base_test import STAC_BASE_V
5 |
6 |
7 | class RendererTestCase(TestCase):
8 |
9 | def setUp(self):
10 | self.client = Client()
11 |
12 | def test_content_type_default(self):
13 | response = self.client.get(f"/{STAC_BASE_V}/search")
14 | self.assertEqual(response.status_code, 200)
15 | self.assertEqual(response['Content-Type'], 'application/json')
16 | self.assertEqual(response.json()['type'], 'FeatureCollection')
17 |
18 | def test_content_type_json(self):
19 | response = self.client.get(f"/{STAC_BASE_V}/search?format=json")
20 | self.assertEqual(response.status_code, 200)
21 | self.assertEqual(response['Content-Type'], 'application/json')
22 | self.assertEqual(response.json()['type'], 'FeatureCollection')
23 |
24 | response = self.client.get(f"/{STAC_BASE_V}/search", headers={'Accept': 'application/json'})
25 | self.assertEqual(response.status_code, 200)
26 | self.assertEqual(response['Content-Type'], 'application/json')
27 | self.assertEqual(response.json()['type'], 'FeatureCollection')
28 |
29 | def test_content_type_geojson(self):
30 | response = self.client.get(f"/{STAC_BASE_V}/search?format=geojson")
31 | self.assertEqual(response.status_code, 200)
32 | self.assertEqual(response['Content-Type'], 'application/geo+json')
33 | self.assertEqual(response.json()['type'], 'FeatureCollection')
34 |
35 | response = self.client.get(
36 | f"/{STAC_BASE_V}/search", headers={'Accept': 'application/geo+json'}
37 | )
38 | self.assertEqual(response.status_code, 200)
39 | self.assertEqual(response['Content-Type'], 'application/geo+json')
40 | self.assertEqual(response.json()['type'], 'FeatureCollection')
41 |
--------------------------------------------------------------------------------
/spec/transaction/components/parameters.yaml:
--------------------------------------------------------------------------------
1 | openapi: 3.0.3
2 | components:
3 | parameters:
4 | uploadId:
5 | name: uploadId
6 | in: path
7 | description: Local identifier of an asset's upload.
8 | required: true
9 | schema:
10 | type: string
11 | presignedUrl:
12 | name: presignedUrl
13 | in: path
14 | description: >-
15 | Presigned url returned by [Create a new Asset's multipart upload](#operation/createAssetUpload).
16 |
17 | Note: the url returned by the above endpoint is the full url including
18 | scheme, host and path
19 | required: true
20 | schema:
21 | type: string
22 | IfMatchWrite:
23 | name: If-Match
24 | in: header
25 | schema:
26 | type: string
27 | description: >-
28 | The RFC7232 `If-Match` header field makes the PUT/PATCH/DEL request method conditional. It is
29 | composed of a comma separated list of ETags or value "*".
30 |
31 |
32 | The server compares the client's ETags (sent with `If-Match`) with the ETag for its
33 | current version of the resource, and if both values don't match (that is, the resource has changed),
34 | the server sends back a `412 Precondition Failed` status, without a body, which tells the client that
35 | he would overwrite another changes of the resource.
36 | example: "d01af8b8ebbf899e30095be8754b377ddb0f0ed0f7fddbc33ac23b0d1969736b"
37 | IdempotencyKey:
38 | name: Idempotency-Key
39 | in: header
40 | schema:
41 | type: string
42 | description: >-
43 | A unique ID for the operation.
44 | This allows making the operation idempotent, so more fault tolerant.
45 | See IETF draft [draft-ietf-httpapi-idempotency-key-header](https://datatracker.ietf.org/doc/draft-ietf-httpapi-idempotency-key-header/).
46 | example: "8e03978e-40d5-43e8-bc93-6894a57f9324"
47 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | service-stac is available for use under the following license, commonly known
2 | as the 3-clause (or "modified") BSD license:
3 |
4 | ==============================
5 |
6 | Copyright (c) 2021, swisstopo
7 | All rights reserved.
8 |
9 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
10 |
11 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
12 |
13 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
14 |
15 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
16 |
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
18 |
19 | ==============================
20 |
21 | Portions of service-stac are based on work of others. A non-exhaustive enumeration:
22 |
23 | [Django](https://github.com/django/django)
24 |
25 | [Django Rest Framework](https://github.com/encode/django-rest-framework)
26 |
--------------------------------------------------------------------------------
/adr/2020_08_13_application_architecture.md:
--------------------------------------------------------------------------------
1 | # Application Architecture
2 |
3 | > `Status: accepted`
4 |
5 | > `Date: 2020-08-13`
6 |
7 | ## Context
8 | `service-stac` is a new service with use cases that are different to the ones in all existing services, specially the fully CRUD (create, read, update, delete) REST interface. Therefore, the choice of the application structure cannot simply be borrowed from an existing service. The use cases for this service will involve heavy read/write access to data via JSON REST API interface of a substantial amount of assets and metadata objects (tens to hundreds of Millions). Additionally, data must be editable manually, at least in a start/migration phase.
9 |
10 | ## Decision
11 | The following decisions are taken regarding the application architecture
12 | - Programming Language **Python**: Python is used in most of the backend services and is the programming language that's best known within devs, so no reason to change here.
13 | - Application Framework **Django**: Django is used as application framework. Django is very mature and has a wide user community. It comes with excellent documentation and a powerfull ORM. Futhermore, there are well-supported and maintained extensions, e.g. for designing REST API's that can considerably reduce the amount of boilerplate code needed for serializing, authentication, ....
14 | - Asset Storage **S3**: Since this service runs on AWS assets will be stored in S3 object store.
15 | - Metadata Storage **PostGIS**: Since Metadata can contain arbitrary geoJSON-supported geometries, Postgres along with the PostGIS extension is used as storage for metadata.
16 |
17 | The application architecture does initially only involve syncronous operations. Async tasks will be reconsider once certain write operations don't meet performance requirements anymore.
18 |
19 | ## Consequences
20 | Developers not familiar with Django will have to walk through the [Django tutorial](https://docs.djangoproject.com/en/dev/intro/) before they get started with development.
21 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0006_auto_20210419_1409.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.7 on 2021-04-19 14:09
2 |
3 | import django.db.models.deletion
4 | from django.db import migrations
5 | from django.db import models
6 |
7 |
8 | class Migration(migrations.Migration):
9 |
10 | dependencies = [
11 | ('stac_api', '0005_auto_20210408_0821'),
12 | ]
13 |
14 | operations = [
15 | migrations.AlterField(
16 | model_name='item',
17 | name='collection',
18 | field=models.ForeignKey(
19 | help_text=
20 | '\n
\n Search Usage:\n
\n
\n arg will make a non exact search checking if arg is part of\n the collection ID\n
\n
\n Multiple arg can be used, separated by spaces. This will search for all\n collections ID containing all arguments.\n
\n
\n "collectionID" will make an exact search for the specified collection.\n
\n
\n Examples :\n
\n
\n Searching for pixelkarte will return all collections which have\n pixelkarte as a part of their collection ID\n
\n
\n Searching for pixelkarte 2016 4 will return all collection\n which have pixelkarte, 2016 AND 4 as part of their collection ID\n
\n
\n Searching for ch.swisstopo.pixelkarte.example will yield only this\n collection, if this collection exists. Please note that it would not return\n a collection named ch.swisstopo.pixelkarte.example.2.\n
6 | arg will make a non exact search checking if arg is part of the collection ID
7 |
8 |
9 | Multiple arg can be used, separated by spaces. This will search for all
10 | collections ID containing all arguments.
11 |
12 |
13 | "arg" will make an exact search on the collection ID field.
14 |
15 |
16 |
17 | Examples :
18 |
19 |
20 | Searching for pixelkarte will return all collections which have pixelkarte as
21 | a part of their collection ID
22 |
23 |
24 | Searching for pixelkarte 2016 4 will return all collections which have
25 | pixelkarte, 2016 AND 4 as part of their or collection ID.
26 |
27 |
28 | Searching for "collection-example-03" will return only the collection
29 | collection-example-03 and nothing else, since Collections ID are unique.
30 |
31 |
`;
32 | var popup = document.createElement("DIV");
33 | popup.className = 'SearchUsage'
34 | popup.innerHTML += innerhtml;
35 | var searchbar = document.getElementById("toolbar");
36 | if (searchbar) {
37 | searchbar.className = 'NameHighlights'
38 | searchbar.appendChild(popup);
39 |
40 | var span = document.querySelectorAll('.NameHighlights');
41 | for (var i = span.length; i--;) {
42 | (function () {
43 | var t;
44 | span[i].onmouseover = function () {
45 | hideAll();
46 | clearTimeout(t);
47 | this.className = 'NameHighlightsHover';
48 | };
49 | span[i].onmouseout = function () {
50 | var self = this;
51 | t = setTimeout(function () {
52 | self.className = 'NameHighlights';
53 | }, 300);
54 | };
55 | })();
56 | }
57 |
58 | function hideAll() {
59 | for (var i = span.length; i--;) {
60 | span[i].className = 'NameHighlights';
61 | }
62 | };
63 | }
64 | };
65 |
--------------------------------------------------------------------------------
/app/stac_api/management/commands/profile_cursor_paginator.py:
--------------------------------------------------------------------------------
1 | import cProfile
2 | import os
3 | import pstats
4 |
5 | from django.conf import settings
6 |
7 | from rest_framework.pagination import CursorPagination
8 | from rest_framework.request import Request
9 | from rest_framework.test import APIRequestFactory
10 |
11 | from stac_api.models.item import Item
12 | from stac_api.utils import CustomBaseCommand
13 |
14 | STAC_BASE_V = f'{settings.STAC_BASE}/v1'
15 |
16 |
17 | class Command(CustomBaseCommand):
18 | help = """Paginator paginate_queryset() profiling command
19 |
20 | Profiling of the method paginator.paginate_queryset(qs, request)
21 |
22 | See https://docs.python.org/3.7/library/profile.html
23 | """
24 |
25 | def add_arguments(self, parser):
26 | super().add_arguments(parser)
27 | parser.add_argument(
28 | '--collection',
29 | type=str,
30 | default='perftest-collection-0',
31 | help="Collection ID to use for the queryset profiling"
32 | )
33 | parser.add_argument('--limit', type=int, default=100, help="Limit to use for the queryset")
34 | parser.add_argument('--sort', type=str, default='tottime', help="Profiling output sorting")
35 | parser.add_argument(
36 | '--lines', type=str, default=50, help="Profiling output numbers of line to show"
37 | )
38 |
39 | def handle(self, *args, **options):
40 | # pylint: disable=import-outside-toplevel,possibly-unused-variable
41 | collection_id = self.options["collection"]
42 | qs = Item.objects.filter(collection__name=collection_id).prefetch_related('assets', 'links')
43 | request = Request(
44 | APIRequestFactory().
45 | get(f'{STAC_BASE_V}/collections/{collection_id}/items?limit={self.options["limit"]}')
46 | )
47 | paginator = CursorPagination()
48 |
49 | cProfile.runctx(
50 | 'paginator.paginate_queryset(qs, request)',
51 | None,
52 | locals(),
53 | f'{settings.BASE_DIR}/{os.environ["LOGS_DIR"]}/stats-file',
54 | sort=self.options['sort']
55 | )
56 | # pylint: disable=duplicate-code
57 | stats = pstats.Stats(f'{settings.BASE_DIR}/{os.environ["LOGS_DIR"]}/stats-file')
58 | stats.sort_stats(self.options['sort']).print_stats()
59 |
60 | self.print_success('Done')
61 |
--------------------------------------------------------------------------------
/app/stac_api/sample_data/swissTLMRegio/collection.json:
--------------------------------------------------------------------------------
1 | {
2 | "stac_version": "0.9.0",
3 | "stac_extensions": [],
4 | "id": "ch.swisstopo.swisstlmregio",
5 | "title": "swissTLMRegio",
6 | "description": "swissTLMRegio is a 2D landscape model, which represents the natural and man-made features of the landscape in vector format. With its high generalisation grade it is a reference dataset for applications requiring a regional or national overview. swissTLMRegio describes about 950 000 objects with their location, shape and neighbourhood connections (topology) as well as the object type and further attributes. swissTLMRegiois composed of 6 thematic topics. Each topic contains one or more feature classes: Transportation, Hydrography, Landcover, Buildings, Miscellaneous, Names..",
7 | "summaries": {
8 | "proj:epsg": [
9 | 2056
10 | ]
11 | },
12 | "extent": {
13 | "spatial": {
14 | "bbox": [
15 | [
16 | 5.327213305905472,
17 | 45.30427347827474,
18 | 11.403439825339375,
19 | 48.19113734655604
20 | ]
21 | ]
22 | },
23 | "temporal": {
24 | "interval": [
25 | [
26 | "2020-10-18T00:00:00Z",
27 | "2020-10-18T00:00:00Z"
28 | ]
29 | ]
30 | }
31 | },
32 | "providers": [
33 | {
34 | "name": "Federal Office of Topography - swisstopo",
35 | "roles": [
36 | "producer",
37 | "licensor"
38 | ],
39 | "url": "https://www.swisstopo.admin.ch"
40 | }
41 | ],
42 | "license": "proprietary",
43 | "created": "2020-10-22T12:30:00Z",
44 | "updated": "2020-10-22T12:30:00Z",
45 | "links": [
46 | {
47 | "href": "https://data.geo.admin.ch/api/stac/v0.9/collections/ch.swisstopo.swisstlmregio",
48 | "rel": "self"
49 | },
50 | {
51 | "href": "https://data.geo.admin.ch/api/stac/v0.9/",
52 | "rel": "root"
53 | },
54 | {
55 | "href": "https://data.geo.admin.ch/api/stac/v0.9/collections/ch.swisstopo.swisstlmregio/items",
56 | "rel": "items"
57 | },
58 | {
59 | "href": "https://www.swisstopo.admin.ch/en/home/meta/conditions/geodata/free-geodata.html",
60 | "rel": "license",
61 | "title": "Licence for the free geodata of the Federal Office of Topography swisstopo"
62 | },
63 | {
64 | "href": "https://www.geocat.ch/geonetwork/srv/eng/catalog.search#/metadata/2a190233-498a-46c4-91ca-509a97d797a2",
65 | "rel": "describedby"
66 | }
67 | ]
68 | }
--------------------------------------------------------------------------------
/app/middleware/logging.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import time
3 |
4 | from django.conf import settings
5 | from django.http import HttpResponse
6 | from django.http import JsonResponse
7 |
8 | logger = logging.getLogger(__name__)
9 |
10 |
11 | class RequestResponseLoggingMiddleware:
12 | # characters that should not be urlencoded in the log statements
13 | url_safe = ',:/'
14 |
15 | def __init__(self, get_response):
16 | self.get_response = get_response
17 | # One-time configuration and initialization.
18 |
19 | def __call__(self, request):
20 | # Code to be executed for each request before
21 | # the view (and later middleware) are called.
22 | extra = {
23 | "request": request,
24 | "request.query": request.GET.urlencode(RequestResponseLoggingMiddleware.url_safe)
25 | }
26 |
27 | if request.method.upper() in [
28 | "PATCH", "POST", "PUT"
29 | ] and request.content_type == "application/json" and not request.path.startswith(
30 | '/api/stac/admin'
31 | ):
32 | extra["request.payload"] = request.body.decode()[:settings.
33 | LOGGING_MAX_REQUEST_PAYLOAD_SIZE]
34 |
35 | logger.debug(
36 | "Request %s %s?%s",
37 | request.method.upper(),
38 | request.path,
39 | request.GET.urlencode(RequestResponseLoggingMiddleware.url_safe),
40 | extra=extra
41 | )
42 | start = time.time()
43 |
44 | response = self.get_response(request)
45 |
46 | extra = {
47 | "request": request,
48 | "response": {
49 | "code": response.status_code,
50 | "headers": dict(response.items()),
51 | "duration": time.time() - start
52 | },
53 | }
54 |
55 | # Not all response types have a 'content' attribute,
56 | # HttpResponse and JSONResponse sure have
57 | # (e.g. WhiteNoiseFileResponse doesn't)
58 | if isinstance(response, (HttpResponse, JsonResponse)):
59 | extra["response"]["payload"] = response.content.decode(
60 | )[:settings.LOGGING_MAX_RESPONSE_PAYLOAD_SIZE]
61 |
62 | logger.info(
63 | "Response %s %s %s?%s",
64 | response.status_code,
65 | request.method.upper(),
66 | request.path,
67 | request.GET.urlencode(RequestResponseLoggingMiddleware.url_safe),
68 | extra=extra
69 | )
70 | # Code to be executed for each request/response after
71 | # the view is called.
72 |
73 | return response
74 |
--------------------------------------------------------------------------------
/app/stac_api/migrations/0047_prefill_counter_tables.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.0.7 on 2024-07-23 15:14
2 |
3 | from django.db import migrations
4 |
5 |
6 | # The migration SQL in this file was created manually.
7 | class Migration(migrations.Migration):
8 | migrate_sql = '''
9 | -- Fill gsd count based on asset values.
10 | TRUNCATE stac_api_gsdcount;
11 | INSERT INTO stac_api_gsdcount (collection_id, value, count)
12 | SELECT item.collection_id, asset.eo_gsd, COUNT(*) AS count
13 | FROM stac_api_asset asset
14 | INNER JOIN stac_api_item item ON asset.item_id = item.id
15 | GROUP BY item.collection_id, asset.eo_gsd;
16 |
17 | -- Fill geoadmin_lang count based on asset values.
18 | TRUNCATE stac_api_geoadminlangcount;
19 | INSERT INTO stac_api_geoadminlangcount (collection_id, value, count)
20 | SELECT item.collection_id, asset.geoadmin_lang, COUNT(*) AS count
21 | FROM stac_api_asset asset
22 | INNER JOIN stac_api_item item ON asset.item_id = item.id
23 | GROUP BY item.collection_id, asset.geoadmin_lang;
24 |
25 | -- Fill geoadmin_variant count based on asset values.
26 | TRUNCATE stac_api_geoadminvariantcount;
27 | INSERT INTO stac_api_geoadminvariantcount (collection_id, value, count)
28 | SELECT item.collection_id, asset.geoadmin_variant, COUNT(*) AS count
29 | FROM stac_api_asset asset
30 | INNER JOIN stac_api_item item ON asset.item_id = item.id
31 | GROUP BY item.collection_id, asset.geoadmin_variant;
32 |
33 | -- Fill proj_epsg count based on asset and collection asset values.
34 | TRUNCATE stac_api_projepsgcount;
35 | INSERT INTO stac_api_projepsgcount (collection_id, value, count)
36 | SELECT collection_id, proj_epsg, COUNT(*) AS count
37 | FROM (
38 | SELECT item.collection_id, asset.proj_epsg
39 | FROM stac_api_asset asset
40 | INNER JOIN stac_api_item item ON asset.item_id = item.id
41 | UNION
42 | SELECT collection_id, proj_epsg
43 | FROM stac_api_collectionasset
44 | ) assets
45 | GROUP BY collection_id, proj_epsg;
46 | '''
47 |
48 | reverse_sql = '''
49 | TRUNCATE stac_api_gsdcount;
50 |
51 | TRUNCATE stac_api_geoadminlangcount;
52 |
53 | TRUNCATE stac_api_geoadminvariantcount;
54 |
55 | TRUNCATE stac_api_projepsgcount;
56 | '''
57 |
58 | dependencies = [
59 | ('stac_api', '0046_geoadminlangcount_geoadminvariantcount_gsdcount_and_more'),
60 | ]
61 |
62 | operations = [migrations.RunSQL(sql=migrate_sql, reverse_sql=reverse_sql)]
63 |
--------------------------------------------------------------------------------
/app/config/settings_dev.py:
--------------------------------------------------------------------------------
1 | """
2 | Django settings for project project.
3 |
4 | Generated by 'django-admin startproject' using Django 3.1.
5 |
6 | For more information on this file, see
7 | https://docs.djangoproject.com/en/3.1/topics/settings/
8 |
9 | For the full list of settings and their values, see
10 | https://docs.djangoproject.com/en/3.1/ref/settings/
11 | """
12 | import environ
13 |
14 | from .settings_prod import * # pylint: disable=wildcard-import, unused-wildcard-import
15 |
16 | env = environ.Env()
17 |
18 | # SECURITY WARNING: don't run with debug turned on in production!
19 | DEBUG = env.bool('DEBUG', False)
20 |
21 | # If set to True, this will enable logger.debug prints of the output of
22 | # EXPLAIN.. ANALYZE of certain queries and the corresponding SQL statement.
23 | DEBUG_ENABLE_DB_EXPLAIN_ANALYZE = env.bool('DEBUG_ENABLE_DB_EXPLAIN_ANALYZE', 'False')
24 |
25 | if DEBUG:
26 | print('WARNING - running in debug mode !')
27 |
28 | # Quick-start development settings - unsuitable for production
29 | # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
30 |
31 | ALLOWED_HOSTS = ['*']
32 |
33 | # django-extensions
34 | # ------------------------------------------------------------------------------
35 | if DEBUG:
36 | INSTALLED_APPS += ['django_extensions', 'debug_toolbar']
37 |
38 | if DEBUG:
39 | MIDDLEWARE = [
40 | 'debug_toolbar.middleware.DebugToolbarMiddleware',
41 | ] + MIDDLEWARE
42 |
43 | # configuration for debug_toolbar
44 | # see https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config
45 | DEBUG_TOOLBAR_CONFIG = {'SHOW_TOOLBAR_CALLBACK': 'middleware.debug.check_toolbar_env'}
46 |
47 | # use the default staticfiles mechanism
48 | # STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
49 |
50 | if DEBUG:
51 | DEBUG_PROPAGATE_API_EXCEPTIONS = env.bool('DEBUG_PROPAGATE_API_EXCEPTIONS', 'False')
52 |
53 | SHELL_PLUS_POST_IMPORTS = ['from tests.data_factory import Factory']
54 |
55 | # Regex patterns of collections that should go to the managed bucket
56 | MANAGED_BUCKET_COLLECTION_PATTERNS = env.list(
57 | 'MANAGED_BUCKET_COLLECTION_PATTERNS', default=["ch.meteoschweiz.ogd-"]
58 | )
59 |
60 | # Since it's impossible to recreate the service-account situation with minio
61 | # we inject some configuration in here to access the second bucket
62 | # in the same way as first bucket, via access/secrets
63 | # Like this we can leave the base (prod) configuration clean, while fixing
64 | # the local setup
65 | AWS_SETTINGS['managed']['access_type'] = "key"
66 | AWS_SETTINGS['managed']['ACCESS_KEY_ID'] = env("LEGACY_AWS_ACCESS_KEY_ID")
67 | AWS_SETTINGS['managed']['SECRET_ACCESS_KEY'] = env("LEGACY_AWS_SECRET_ACCESS_KEY")
68 |
--------------------------------------------------------------------------------
/adr/2025_03_04_cache_settings_update.md:
--------------------------------------------------------------------------------
1 | # Cache Settings Update
2 |
3 | > `Status: Accepted`
4 |
5 | > `Date: 2025-03-04`
6 |
7 | > `Participants: Brice Schaffner, Christoph Böcklin, Benjamin Sugden`
8 |
9 | ## Context
10 |
11 | Cache settings have been implemented using the `update_interval` field that set at the asset upload
12 | level and propagated up to the collection level (see [Cache Settings](2023_02_27_cache_settings.md)).
13 | The propagation up the hierarchy was done via PG trigger and worked fine in the beginning.
14 |
15 | But with the growing numbers of assets, items and collections, the PG trigger started to be slow
16 | which had a significant impact on the write Endpoints.
17 |
18 | ### Proposal
19 |
20 | To avoid performance issue due to aggregation accross all assets within a collection, we can set
21 | the cache settings at the collection level and not at the upload level. Also in order to keep it
22 | very simple stupid we can directly set the cache-control header value on the collection instead
23 | of an update interval. So the collection would decide the cache settings for all of its child call.
24 |
25 | To do so we create a new field `cache_control_header` in the collection field. To start this field
26 | is only available on the admin interface and not on the API.
27 |
28 | For collection aggregation endpoints, like the search endpoint or the collections list endpoint,
29 | which can potentially contain more than one collection, we disable the cache in order to avoid any
30 | caching issue and keep the logic simple.
31 |
32 | ## Decision
33 |
34 | - Remove the `update_interval` field from item and collection model
35 | - Keep the `update_interval` field in the upload and asset models
36 | - Mark the `update_interval` field as deprecated in the models and in the openapi spec
37 | - `update_interval` is kept as a hint but has no effect anymore
38 | - Set the cache-control header of the search and collections list endpoints configurable via
39 | environment variable with a default to no cache.
40 | - Add a new `cache_control_header` field to the collection model
41 | - Add the new `cache_control_header` field to the admin interface
42 | - For the GET endpoints of collection detail, items, item detail, assets, assets detail and asset
43 | data download, we either use the value of the related collection `cache_control_header` field or
44 | the default configured in the environment variable (as for before)
45 |
46 | ## Consequences
47 |
48 | - Actually we have 3 collections that have the `update_interval` field used, so for those once this
49 | is deployed we need to manually set the value.
50 |
51 | ## References
52 |
53 | - [Cache Settings](2023_02_27_cache_settings.md)
54 | - [Jira PB-1126](https://swissgeoplatform.atlassian.net/issues/PB-1126)
55 |
--------------------------------------------------------------------------------
/app/config/logging-cfg-unittest-ci.yml:
--------------------------------------------------------------------------------
1 | version: 1
2 | disable_existing_loggers: False # this allow to get logger at module level
3 |
4 | root:
5 | handlers:
6 | - console
7 | level: DEBUG
8 | propagate: True
9 |
10 | loggers:
11 | tests:
12 | level: DEBUG
13 | handlers:
14 | - console-tests
15 | botocore:
16 | level: INFO
17 | boto3:
18 | level: INFO
19 | s3transfer:
20 | level: INFO
21 | stac_api:
22 | level: DEBUG
23 | middleware:
24 | level: DEBUG
25 | django:
26 | level: INFO
27 | django.db:
28 | level: INFO
29 | django.utils.autoreload:
30 | level: INFO
31 | gunicorn.error:
32 | handlers:
33 | - console
34 | gunicorn.access:
35 | handlers:
36 | - console
37 |
38 | filters:
39 | isotime:
40 | (): logging_utilities.filters.TimeAttribute
41 | isotime: False
42 | utc_isotime: True
43 | django:
44 | (): logging_utilities.filters.django_request.JsonDjangoRequest
45 | attr_name: request
46 | include_keys:
47 | - request.path
48 | - request.method
49 | - request.headers
50 | exclude_keys:
51 | - request.headers.Authorization
52 | - request.headers.Proxy-Authorization
53 | - request.headers.Cookie
54 |
55 | formatters:
56 | standard:
57 | (): logging_utilities.formatters.extra_formatter.ExtraFormatter
58 | format: "[%(utc_isotime)s] %(levelname)-8s - %(name)-26s : %(message)s"
59 | extra_fmt: " - collection: %(collection)s - item: %(item)s - asset: %(asset)s - duration: %(duration)s"
60 | # extra_pretty_print: True
61 | standard-file:
62 | (): logging_utilities.formatters.extra_formatter.ExtraFormatter
63 | format: "[%(utc_isotime)s] %(levelname)-8s - %(name)-26s : %(message)s"
64 | extra_fmt: " - extra: %s"
65 | extra_pretty_print: True
66 | json:
67 | (): logging_utilities.formatters.json_formatter.JsonFormatter
68 | add_always_extra: True
69 | filter_attributes:
70 | - utc_isotime
71 | remove_empty: True
72 | fmt:
73 | time: utc_isotime
74 | level: levelname
75 | logger: name
76 | module: module
77 | function: funcName
78 | process: process
79 | thread: thread
80 | exc_info: exc_info
81 | message: message
82 |
83 | handlers:
84 | console-tests:
85 | level: WARNING
86 | class: logging.StreamHandler
87 | formatter: standard
88 | stream: ext://sys.stdout
89 | filters:
90 | - isotime
91 | - django
92 | console:
93 | level: CRITICAL
94 | class: logging.StreamHandler
95 | formatter: standard
96 | stream: ext://sys.stderr
97 | filters:
98 | - isotime
99 | - django
100 |
--------------------------------------------------------------------------------
/app/stac_api/signals.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | from django.db.models import ProtectedError
4 | from django.db.models.signals import pre_delete
5 | from django.dispatch import receiver
6 |
7 | from stac_api.models.collection import CollectionAsset
8 | from stac_api.models.collection import CollectionAssetUpload
9 | from stac_api.models.item import Asset
10 | from stac_api.models.item import AssetUpload
11 |
12 | logger = logging.getLogger(__name__)
13 |
14 |
15 | @receiver(pre_delete, sender=AssetUpload)
16 | def check_on_going_upload(sender, instance, **kwargs):
17 | if instance.status == AssetUpload.Status.IN_PROGRESS:
18 | logger.error(
19 | "Cannot delete asset %s due to upload %s which is still in progress",
20 | instance.asset.name,
21 | instance.upload_id,
22 | extra={
23 | 'upload_id': instance.upload_id,
24 | 'asset': instance.asset.name,
25 | 'item': instance.asset.item.name,
26 | 'collection': instance.asset.item.collection.name
27 | }
28 | )
29 | raise ProtectedError(
30 | f"Asset {instance.asset.name} has still an upload in progress", [instance]
31 | )
32 |
33 |
34 | @receiver(pre_delete, sender=CollectionAssetUpload)
35 | def check_on_going_collection_asset_upload(sender, instance, **kwargs):
36 | if instance.status == CollectionAssetUpload.Status.IN_PROGRESS:
37 | logger.error(
38 | "Cannot delete collection asset %s due to upload %s which is still in progress",
39 | instance.asset.name,
40 | instance.upload_id,
41 | extra={
42 | 'upload_id': instance.upload_id,
43 | 'asset': instance.asset.name,
44 | 'collection': instance.asset.collection.name
45 | }
46 | )
47 | raise ProtectedError(
48 | f"Collection Asset {instance.asset.name} has still an upload in progress", [instance]
49 | )
50 |
51 |
52 | @receiver(pre_delete, sender=Asset)
53 | def delete_s3_asset(sender, instance, **kwargs):
54 | # The file is not automatically deleted by Django
55 | # when the object holding its reference is deleted
56 | # hence it has to be done here.
57 | if not instance.is_external:
58 | logger.info("The asset %s is deleted from s3", instance.file.name)
59 | instance.file.delete(save=False)
60 |
61 |
62 | @receiver(pre_delete, sender=CollectionAsset)
63 | def delete_s3_collection_asset(sender, instance, **kwargs):
64 | # The file is not automatically deleted by Django
65 | # when the object holding its reference is deleted
66 | # hence it has to be done here.
67 | if not instance.is_external:
68 | logger.info("The collection asset %s is deleted from s3", instance.file.name)
69 | instance.file.delete(save=False)
70 |
--------------------------------------------------------------------------------
/app/stac_api/templates/js/admin/item_help_search.js:
--------------------------------------------------------------------------------
1 | window.onload = function() {
2 | var innerhtml = `
3 | Search Usage:
4 |
5 |
6 | arg will make a non exact search checking if arg is part of the
7 | collection ID and /or the item ID
8 |
9 |
10 | Multiple arg can be used, separated by spaces. This will search for all
11 | collections ID and item ID containing all arguments.
12 |
13 |
14 | "arg" will make an exact search on the Item ID field.
15 |
16 |
17 | "collectionID/itemID/" will make an exact search on a whole item path.
18 |
19 |
20 |
21 | Examples :
22 |
23 |
24 | Searching for pixelkarte will return all items which have pixelkarte as
25 | a part of their item ID or collection ID
26 |
27 |
28 | Searching for pixelkarte 2016 4 will return all items which have pixelkarte,
29 | 2016 AND 4 as part of their item ID or collection ID.
30 |
31 |
32 | Searching for "item-example-03" will yield all items named asset-example-03 in
33 | all collections. If there is one in the ch.swisstopo.swisstlm/ collection and
34 | one in the ch.swisstopo.pixelkarte-farbe-pk200.noscale collection, it will
35 | return two results
36 |
37 |
38 | Searching for "ch.swisstopo.swisstlm/item-example-03" will return the desired
39 | item and nothing else.
40 |
6 | arg will make a non exact search checking if arg is
7 | part of the collection ID, the item ID and/or the asset ID
8 |
9 |
10 | Multiple arg can be used, separated by spaces. This will search for all assets
11 | ID, items ID / or collections ID containing all arguments.
12 |
13 |
14 | "arg" will make an exact search on the asset ID field.
15 |
16 |
17 | "collectionID/itemID/assetID" will make an exact search on a whole path.
18 |
19 |
20 | Examples :
21 |
22 |
23 | Searching for pixelkarte will return all collections which have
24 | pixelkarte as a part of their asset ID, item ID or collection ID
25 |
26 |
27 | Searching for pixelkarte 2016 4 will return all collection which have
28 | pixelkarte, 2016 AND 4 as part of their asset ID, item ID or collection ID.
29 |
30 |
31 | Searching for "asset-example-03" will yield all assets named asset-example-03
32 | in all items and collections. If there is one in the
33 | ch.swisstopo.swisstlm/swisstlmregio-2020 item and one in the
34 | ch.swisstopo.pixelkarte-farbe-pk200.noscale/smr200-200-2-2016 item,
35 | it will return two results
36 |
37 |
38 | Searching for "ch.swisstopo.swisstlm/swisstlmregio-2020/asset-example-03" will
39 | return the desired asset and nothing else.
40 |
41 |
`;
42 | var popup = document.createElement("DIV");
43 | popup.className = 'SearchUsage'
44 | popup.innerHTML += innerhtml;
45 | var searchbar = document.getElementById("toolbar");
46 | if (searchbar) {
47 | searchbar.className = 'NameHighlights'
48 | searchbar.appendChild(popup);
49 |
50 | var span = document.querySelectorAll('.NameHighlights');
51 | for (var i = span.length; i--;) {
52 | (function () {
53 | var t;
54 | span[i].onmouseover = function () {
55 | hideAll();
56 | clearTimeout(t);
57 | this.className = 'NameHighlightsHover';
58 | };
59 | span[i].onmouseout = function () {
60 | var self = this;
61 | t = setTimeout(function () {
62 | self.className = 'NameHighlights';
63 | }, 300);
64 | };
65 | })();
66 | }
67 |
68 | function hideAll() {
69 | for (var i = span.length; i--;) {
70 | span[i].className = 'NameHighlights';
71 | }
72 | };
73 | }
74 | };
75 |
--------------------------------------------------------------------------------
/app/stac_api/sample_data/swissMapRaster200/collection.json:
--------------------------------------------------------------------------------
1 | {
2 | "stac_version": "0.9.0",
3 | "stac_extensions": [],
4 | "id": "ch.swisstopo.pixelkarte-farbe-pk200.noscale",
5 | "title": "National Map 1:200'000",
6 | "description": "The National Map 1:200,000 is a topographic map giving an overview of Switzerland. The map perimeter is divided into four individual sheets (sheet 1-4). The map content is updated periodically in a four-year cycle. The National Map 1:200,000 is published in analogue format as a printed map and in digital format as a Swiss Map Raster. The printed map is available folded and unfolded. The Swiss Map Raster is the digital National Map 1:200,000 and is delivered as a georeferenced TIF file (raster format). The map content is separated into individual colour layers with no direct bearing on the individual map elements. The pixel maps are available as a colour combination corresponding to fixed swisstopo standards (254 dpi and 508 dpi) or separated as colour layers (binary file 508 dpi). They can be ordered as an individual sheet or as any individually defined section of the map series.",
7 | "summaries": {
8 | "eo:gsd": [
9 | 10,
10 | 20
11 | ],
12 | "geoadmin:variant": [
13 | "kgrs",
14 | "komb",
15 | "krel"
16 | ],
17 | "proj:epsg": [
18 | 2056
19 | ]
20 | },
21 | "extent": {
22 | "spatial": {
23 | "bbox": [
24 | [
25 | 5.602407647225925,
26 | 45.50393724771029,
27 | 10.747774424579303,
28 | 48.02711914887954
29 | ]
30 | ]
31 | },
32 | "temporal": {
33 | "interval": [
34 | [
35 | "2016-12-31T00:00:00Z",
36 | "2016-12-31T00:00:00Z"
37 | ]
38 | ]
39 | }
40 | },
41 | "providers": [
42 | {
43 | "name": "Federal Office of Topography - swisstopo",
44 | "roles": [
45 | "producer",
46 | "licensor"
47 | ],
48 | "url": "https://www.swisstopo.admin.ch"
49 | }
50 | ],
51 | "license": "proprietary",
52 | "created": "2020-10-22T12:30:00Z",
53 | "updated": "2020-10-22T12:30:00Z",
54 | "links": [
55 | {
56 | "href": "https://data.geo.admin.ch/api/stac/v0.9/collections/ch.swisstopo.pixelkarte-farbe-pk200.noscale",
57 | "rel": "self"
58 | },
59 | {
60 | "href": "https://data.geo.admin.ch/api/stac/v0.9/",
61 | "rel": "root"
62 | },
63 | {
64 | "href": "https://data.geo.admin.ch/api/stac/v0.9/collections/ch.swisstopo.pixelkarte-farbe-pk200.noscale/items",
65 | "rel": "items"
66 | },
67 | {
68 | "href": "https://www.swisstopo.admin.ch/en/home/meta/conditions/geodata/free-geodata.html",
69 | "rel": "license",
70 | "title": "Licence for the free geodata of the Federal Office of Topography swisstopo"
71 | },
72 | {
73 | "href": "https://www.geocat.ch/geonetwork/srv/eng/catalog.search#/metadata/5f75341e-2050-4ed9-a8de-ee7637490565",
74 | "rel": "describedby"
75 | }
76 | ]
77 | }
--------------------------------------------------------------------------------
/app/stac_api/migrations/0005_auto_20210408_0821.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.1.7 on 2021-04-08 08:21
2 |
3 | import django.core.serializers.json
4 | import django.core.validators
5 | import django.db.models.deletion
6 | from django.db import migrations
7 | from django.db import models
8 |
9 | import stac_api.models.general
10 |
11 |
12 | class Migration(migrations.Migration):
13 |
14 | dependencies = [
15 | ('stac_api', '0004_auto_20210408_0659'),
16 | ]
17 |
18 | operations = [
19 | migrations.CreateModel(
20 | name='AssetUpload',
21 | fields=[
22 | ('id', models.BigAutoField(primary_key=True, serialize=False)),
23 | ('upload_id', models.CharField(max_length=255)),
24 | (
25 | 'status',
26 | models.CharField(
27 | choices=[(None, ''), ('in-progress', 'In Progress'),
28 | ('completed', 'Completed'), ('aborted', 'Aborted')],
29 | default='in-progress',
30 | max_length=32
31 | )
32 | ),
33 | (
34 | 'number_parts',
35 | models.IntegerField(
36 | validators=[
37 | django.core.validators.MinValueValidator(1),
38 | django.core.validators.MaxValueValidator(100)
39 | ]
40 | )
41 | ),
42 | (
43 | 'urls',
44 | models.JSONField(
45 | blank=True,
46 | default=list,
47 | encoder=django.core.serializers.json.DjangoJSONEncoder
48 | )
49 | ),
50 | ('created', models.DateTimeField(auto_now_add=True)),
51 | ('ended', models.DateTimeField(blank=True, default=None, null=True)),
52 | ('checksum_multihash', models.CharField(max_length=255)),
53 | (
54 | 'etag',
55 | models.CharField(default=stac_api.models.general.compute_etag, max_length=56)
56 | ),
57 | (
58 | 'asset',
59 | models.ForeignKey(
60 | on_delete=django.db.models.deletion.CASCADE,
61 | related_name='+',
62 | to='stac_api.asset'
63 | )
64 | ),
65 | ],
66 | ),
67 | migrations.AddConstraint(
68 | model_name='assetupload',
69 | constraint=models.UniqueConstraint(
70 | fields=('asset', 'upload_id'), name='unique_together'
71 | ),
72 | ),
73 | migrations.AddConstraint(
74 | model_name='assetupload',
75 | constraint=models.UniqueConstraint(
76 | condition=models.Q(status='in-progress'),
77 | fields=('asset', 'status'),
78 | name='unique_in_progress'
79 | ),
80 | ),
81 | ]
82 |
--------------------------------------------------------------------------------
/app/stac_api/management/commands/reset_counter_tables.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | from django.db import connection
4 |
5 | from stac_api.utils import CustomBaseCommand
6 |
7 |
8 | class Command(CustomBaseCommand):
9 | help = """Reset the summary counter tables.
10 |
11 | Truncates all the summary counter tables and repopulates with current data to make sure they are
12 | in sync with the values in the asset table. Unless the triggers are disabled or values in the
13 | counter tables are changed manually, this should not be required.
14 | """
15 |
16 | def handle(self, *args, **options):
17 | self.print_success('running query to update counter tables...')
18 |
19 | start = time.monotonic()
20 | with connection.cursor() as cursor:
21 | cursor.execute(
22 | """
23 | -- Fill gsd count based on asset values.
24 | TRUNCATE stac_api_gsdcount;
25 | INSERT INTO stac_api_gsdcount (collection_id, value, count)
26 | SELECT item.collection_id, asset.eo_gsd, COUNT(*) AS count
27 | FROM stac_api_asset asset
28 | INNER JOIN stac_api_item item ON asset.item_id = item.id
29 | GROUP BY item.collection_id, asset.eo_gsd;
30 |
31 | -- Fill geoadmin_lang count based on asset values.
32 | TRUNCATE stac_api_geoadminlangcount;
33 | INSERT INTO stac_api_geoadminlangcount (collection_id, value, count)
34 | SELECT item.collection_id, asset.geoadmin_lang, COUNT(*) AS count
35 | FROM stac_api_asset asset
36 | INNER JOIN stac_api_item item ON asset.item_id = item.id
37 | GROUP BY item.collection_id, asset.geoadmin_lang;
38 |
39 | -- Fill geoadmin_variant count based on asset values.
40 | TRUNCATE stac_api_geoadminvariantcount;
41 | INSERT INTO stac_api_geoadminvariantcount (collection_id, value, count)
42 | SELECT item.collection_id, asset.geoadmin_variant, COUNT(*) AS count
43 | FROM stac_api_asset asset
44 | INNER JOIN stac_api_item item ON asset.item_id = item.id
45 | GROUP BY item.collection_id, asset.geoadmin_variant;
46 |
47 | -- Fill proj_epsg count based on asset and collection asset values.
48 | TRUNCATE stac_api_projepsgcount;
49 | INSERT INTO stac_api_projepsgcount (collection_id, value, count)
50 | SELECT collection_id, proj_epsg, COUNT(*) AS count
51 | FROM (
52 | SELECT item.collection_id, asset.proj_epsg
53 | FROM stac_api_asset asset
54 | INNER JOIN stac_api_item item ON asset.item_id = item.id
55 | UNION
56 | SELECT collection_id, proj_epsg
57 | FROM stac_api_collectionasset
58 | ) assets
59 | GROUP BY collection_id, proj_epsg;
60 | """
61 | )
62 | self.print_success(
63 | f"successfully updated counter tables in {(time.monotonic()-start):.3f}s"
64 | )
65 |
--------------------------------------------------------------------------------
/app/tests/tests_09/test_pgtriggers.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | from tests.tests_09.base_test import StacBaseTransactionTestCase
4 | from tests.tests_09.data_factory import Factory
5 | from tests.tests_09.sample_data.asset_samples import FILE_CONTENT_1
6 | from tests.utils import MockS3PerTestMixin
7 |
8 | logger = logging.getLogger(__name__)
9 |
10 |
11 | class PgTriggersFileSizeTestCase(MockS3PerTestMixin, StacBaseTransactionTestCase):
12 |
13 | def setUp(self):
14 | super().setUp()
15 | self.factory = Factory()
16 | self.collection = self.factory.create_collection_sample().model
17 | self.item = self.factory.create_item_sample(collection=self.collection).model
18 |
19 | # Add a second item
20 | self.item2 = self.factory.create_item_sample(collection=self.collection,).model
21 |
22 | def test_pgtrigger_file_size(self):
23 | self.factory = Factory()
24 | file_size = len(FILE_CONTENT_1)
25 |
26 | self.assertEqual(self.collection.total_data_size, 0)
27 | self.assertEqual(self.item.total_data_size, 0)
28 |
29 | # check collection's and item's file size on asset update
30 | asset1 = self.factory.create_asset_sample(self.item, sample='asset-1', db_create=True)
31 | self.collection.refresh_from_db()
32 | self.assertEqual(self.collection.total_data_size, file_size)
33 | self.assertEqual(self.item.total_data_size, file_size)
34 | self.assertEqual(asset1.model.file_size, file_size)
35 |
36 | # check collection's and item's file size on asset update
37 | asset2 = self.factory.create_asset_sample(self.item, sample='asset-2', db_create=True)
38 | self.collection.refresh_from_db()
39 | self.assertEqual(self.collection.total_data_size, 2 * file_size)
40 | self.assertEqual(self.item.total_data_size, 2 * file_size)
41 | self.assertEqual(asset2.model.file_size, file_size)
42 |
43 | # check collection's and item's file size on adding an empty asset
44 | asset3 = self.factory.create_asset_sample(self.item, sample='asset-no-file', db_create=True)
45 | self.collection.refresh_from_db()
46 |
47 | self.assertEqual(self.collection.total_data_size, 2 * file_size)
48 | self.assertEqual(self.item.total_data_size, 2 * file_size)
49 | self.assertEqual(asset3.model.file_size, 0)
50 |
51 | # check collection's and item's file size when updating asset of another item
52 | asset4 = self.factory.create_asset_sample(self.item2, sample='asset-2', db_create=True)
53 | self.collection.refresh_from_db()
54 |
55 | self.assertEqual(
56 | self.collection.total_data_size,
57 | 3 * file_size,
58 | )
59 | self.assertEqual(self.item.total_data_size, 2 * file_size)
60 | self.assertEqual(self.item2.total_data_size, file_size)
61 |
62 | # check collection's and item's file size when deleting asset
63 | asset1.model.delete()
64 | self.item.refresh_from_db()
65 | self.collection.refresh_from_db()
66 |
67 | self.assertEqual(self.collection.total_data_size, 2 * file_size)
68 | self.assertEqual(self.item.total_data_size, 1 * file_size)
69 |
--------------------------------------------------------------------------------
/app/tests/tests_10/test_pgtriggers.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | from tests.tests_10.base_test import StacBaseTransactionTestCase
4 | from tests.tests_10.data_factory import Factory
5 | from tests.tests_10.sample_data.asset_samples import FILE_CONTENT_1
6 | from tests.utils import MockS3PerTestMixin
7 |
8 | logger = logging.getLogger(__name__)
9 |
10 |
11 | class PgTriggersFileSizeTestCase(MockS3PerTestMixin, StacBaseTransactionTestCase):
12 |
13 | def setUp(self):
14 | super().setUp()
15 | self.factory = Factory()
16 | self.collection = self.factory.create_collection_sample().model
17 | self.item = self.factory.create_item_sample(collection=self.collection).model
18 |
19 | # Add a second item
20 | self.item2 = self.factory.create_item_sample(collection=self.collection,).model
21 |
22 | def test_pgtrigger_file_size(self):
23 | self.factory = Factory()
24 | file_size = len(FILE_CONTENT_1)
25 |
26 | self.assertEqual(self.collection.total_data_size, 0)
27 | self.assertEqual(self.item.total_data_size, 0)
28 |
29 | # check collection's and item's file size on asset update
30 | asset1 = self.factory.create_asset_sample(self.item, sample='asset-1', db_create=True)
31 | self.collection.refresh_from_db()
32 | self.assertEqual(self.collection.total_data_size, file_size)
33 | self.assertEqual(self.item.total_data_size, file_size)
34 | self.assertEqual(asset1.model.file_size, file_size)
35 |
36 | # check collection's and item's file size on asset update
37 | asset2 = self.factory.create_asset_sample(self.item, sample='asset-2', db_create=True)
38 | self.collection.refresh_from_db()
39 | self.assertEqual(self.collection.total_data_size, 2 * file_size)
40 | self.assertEqual(self.item.total_data_size, 2 * file_size)
41 | self.assertEqual(asset2.model.file_size, file_size)
42 |
43 | # check collection's and item's file size on adding an empty asset
44 | asset3 = self.factory.create_asset_sample(self.item, sample='asset-no-file', db_create=True)
45 | self.collection.refresh_from_db()
46 |
47 | self.assertEqual(self.collection.total_data_size, 2 * file_size)
48 | self.assertEqual(self.item.total_data_size, 2 * file_size)
49 | self.assertEqual(asset3.model.file_size, 0)
50 |
51 | # check collection's and item's file size when updating asset of another item
52 | asset4 = self.factory.create_asset_sample(self.item2, sample='asset-2', db_create=True)
53 | self.collection.refresh_from_db()
54 |
55 | self.assertEqual(
56 | self.collection.total_data_size,
57 | 3 * file_size,
58 | )
59 | self.assertEqual(self.item.total_data_size, 2 * file_size)
60 | self.assertEqual(self.item2.total_data_size, file_size)
61 |
62 | # check collection's and item's file size when deleting asset
63 | asset1.model.delete()
64 | self.item.refresh_from_db()
65 | self.collection.refresh_from_db()
66 |
67 | self.assertEqual(self.collection.total_data_size, 2 * file_size)
68 | self.assertEqual(self.item.total_data_size, 1 * file_size)
69 |
--------------------------------------------------------------------------------
/app/stac_api/storages.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from datetime import datetime
3 | from time import mktime
4 | from wsgiref.handlers import format_date_time
5 |
6 | from django.conf import settings
7 |
8 | from storages.backends.s3boto3 import S3Boto3Storage
9 |
10 | from stac_api.utils import AVAILABLE_S3_BUCKETS
11 | from stac_api.utils import get_s3_cache_control_value
12 |
13 | logger = logging.getLogger(__name__)
14 |
15 |
16 | class BaseS3Storage(S3Boto3Storage):
17 | # pylint: disable=abstract-method
18 |
19 | object_sha256 = None
20 | cache_control_header = None
21 | asset_content_type = None
22 |
23 | access_key = None
24 | secret_key = None
25 | bucket_name = None
26 | endpoint_url = None
27 |
28 | def __init__(self, s3_bucket: AVAILABLE_S3_BUCKETS):
29 | s3_config = settings.AWS_SETTINGS[s3_bucket.name]
30 |
31 | if s3_config['access_type'] == 'key':
32 | self.access_key = s3_config['ACCESS_KEY_ID']
33 | self.secret_key = s3_config['SECRET_ACCESS_KEY']
34 | # else: let it infer from the environment
35 |
36 | self.bucket_name = s3_config['S3_BUCKET_NAME']
37 | self.endpoint_url = s3_config['S3_ENDPOINT_URL']
38 |
39 | super().__init__()
40 | self.object_sha256 = None
41 | self.cache_control_header = None
42 | self.asset_content_type = None
43 |
44 | def get_object_parameters(self, name):
45 | """
46 | Returns a dictionary that is passed to file upload. Override this
47 | method to adjust this on a per-object basis to set e.g ContentDisposition.
48 |
49 | Args:
50 | name: string
51 | file name
52 |
53 | Returns:
54 | Parameters from AWS_S3_OBJECT_PARAMETERS plus the file sha256 checksum as MetaData
55 | """
56 | params = super().get_object_parameters(name)
57 |
58 | # Set the content-type from the assets metadata
59 | if self.asset_content_type is None:
60 | raise ValueError(f'Missing content-type for asset {name}')
61 | params["ContentType"] = self.asset_content_type
62 |
63 | if 'Metadata' not in params:
64 | params['Metadata'] = {}
65 | if self.object_sha256 is None:
66 | raise ValueError(f'Missing asset object sha256 for {name}')
67 | params['Metadata']['sha256'] = self.object_sha256
68 |
69 | if 'CacheControl' in params:
70 | logger.warning(
71 | 'Global cache-control header for S3 storage will be overwritten for %s', name
72 | )
73 | params["CacheControl"] = get_s3_cache_control_value(self.cache_control_header)
74 |
75 | stamp = mktime(datetime.now().timetuple())
76 | params['Expires'] = format_date_time(stamp + settings.STORAGE_ASSETS_CACHE_SECONDS)
77 |
78 | return params
79 |
80 |
81 | class LegacyS3Storage(BaseS3Storage):
82 | # pylint: disable=abstract-method
83 |
84 | def __init__(self):
85 | super().__init__(AVAILABLE_S3_BUCKETS.legacy)
86 |
87 |
88 | class ManagedS3Storage(BaseS3Storage):
89 | # pylint: disable=abstract-method
90 |
91 | def __init__(self):
92 | super().__init__(AVAILABLE_S3_BUCKETS.managed)
93 |
--------------------------------------------------------------------------------
/app/config/urls.py:
--------------------------------------------------------------------------------
1 | """project URL Configuration
2 |
3 | The `urlpatterns` list routes URLs to views. For more information please see:
4 | https://docs.djangoproject.com/en/3.1/topics/http/urls/
5 | Examples:
6 | Function views
7 | 1. Add an import: from my_app import views
8 | 2. Add a URL to urlpatterns: path('', views.home, name='home')
9 | Class-based views
10 | 1. Add an import: from other_app.views import Home
11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
12 | Including another URLconf
13 | 1. Import the include() function: from django.urls import include, path
14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
15 | """
16 | import time
17 |
18 | from django.conf import settings
19 | from django.contrib import admin
20 | from django.http import JsonResponse
21 | from django.urls import include
22 | from django.urls import path
23 |
24 | STAC_BASE = settings.STAC_BASE
25 |
26 |
27 | def checker(request):
28 | if settings.CHECKER_DELAY > 0:
29 | time.sleep(settings.CHECKER_DELAY)
30 |
31 | data = {"success": True, "message": "OK"}
32 |
33 | return JsonResponse(data)
34 |
35 |
36 | urlpatterns = [
37 | path('', include('django_prometheus.urls')),
38 | path('checker', checker, name='checker'),
39 | path(f'{STAC_BASE}/', include('stac_api.urls')),
40 | path(f'{STAC_BASE}/admin/', admin.site.urls),
41 | ]
42 |
43 | if settings.DEBUG:
44 | import debug_toolbar
45 |
46 | from stac_api.views.test import TestAssetUpsertHttp500
47 | from stac_api.views.test import TestCollectionAssetUpsertHttp500
48 | from stac_api.views.test import TestCollectionUpsertHttp500
49 | from stac_api.views.test import TestHttp500
50 | from stac_api.views.test import TestItemUpsertHttp500
51 |
52 | urlpatterns = [
53 | path('__debug__/', include(debug_toolbar.urls)),
54 | path('tests/test_http_500', TestHttp500.as_view()),
55 | path(
56 | 'tests/test_collection_upsert_http_500/',
57 | TestCollectionUpsertHttp500.as_view(),
58 | name='test-collection-detail-http-500'
59 | ),
60 | path(
61 | 'tests/test_item_upsert_http_500//',
62 | TestItemUpsertHttp500.as_view(),
63 | name='test-item-detail-http-500'
64 | ),
65 | path(
66 | 'tests/test_asset_upsert_http_500///',
67 | TestAssetUpsertHttp500.as_view(),
68 | name='test-asset-detail-http-500'
69 | ),
70 | path(
71 | 'tests/test_collection_asset_upsert_http_500//',
72 | TestCollectionAssetUpsertHttp500.as_view(),
73 | name='test-collection-asset-detail-http-500'
74 | ),
75 | # Add v0.9 namespace to test routes.
76 | path(
77 | 'tests/v0.9/test_asset_upsert_http_500///',
78 | include(([
79 | path("", TestAssetUpsertHttp500.as_view(), name='test-asset-detail-http-500')
80 | ],
81 | 'test_v0.9'),
82 | namespace='test_v0.9')
83 | )
84 | ] + urlpatterns
85 |
--------------------------------------------------------------------------------
16 |