├── VERSION ├── nucliadb ├── src │ ├── nucliadb │ │ ├── py.typed │ │ ├── ingest │ │ │ ├── py.typed │ │ │ ├── fields │ │ │ │ └── __init__.py │ │ │ ├── orm │ │ │ │ └── __init__.py │ │ │ ├── consumer │ │ │ │ └── __init__.py │ │ │ └── service │ │ │ │ └── exceptions.py │ │ ├── reader │ │ │ ├── py.typed │ │ │ ├── reader │ │ │ │ └── __init__.py │ │ │ └── api │ │ │ │ └── __init__.py │ │ ├── search │ │ │ ├── py.typed │ │ │ ├── api │ │ │ │ ├── __init__.py │ │ │ │ └── v1 │ │ │ │ │ └── resource │ │ │ │ │ └── __init__.py │ │ │ ├── requesters │ │ │ │ └── __init__.py │ │ │ └── search │ │ │ │ ├── __init__.py │ │ │ │ ├── chat │ │ │ │ └── __init__.py │ │ │ │ └── query_parser │ │ │ │ └── __init__.py │ │ ├── train │ │ │ ├── py.typed │ │ │ ├── api │ │ │ │ └── __init__.py │ │ │ └── generators │ │ │ │ └── __init__.py │ │ ├── writer │ │ │ ├── py.typed │ │ │ ├── api │ │ │ │ └── __init__.py │ │ │ └── resource │ │ │ │ └── __init__.py │ │ ├── standalone │ │ │ ├── py.typed │ │ │ ├── static │ │ │ │ └── favicon.ico │ │ │ └── __init__.py │ │ ├── backups │ │ │ └── __init__.py │ │ ├── common │ │ │ ├── __init__.py │ │ │ ├── maindb │ │ │ │ └── __init__.py │ │ │ ├── cluster │ │ │ │ ├── __init__.py │ │ │ │ └── standalone │ │ │ │ │ └── __init__.py │ │ │ ├── http_clients │ │ │ │ └── __init__.py │ │ │ ├── models_utils │ │ │ │ └── __init__.py │ │ │ ├── external_index_providers │ │ │ │ └── __init__.py │ │ │ ├── constants.py │ │ │ └── datamanagers │ │ │ │ └── exceptions.py │ │ ├── migrator │ │ │ └── __init__.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ └── internal │ │ │ │ └── __init__.py │ │ ├── tests │ │ │ └── __init__.py │ │ └── __init__.py │ └── migrations │ │ ├── __init__.py │ │ └── pg │ │ └── __init__.py └── tests │ ├── ingest │ ├── integration │ │ ├── orm │ │ │ └── assets │ │ │ │ ├── text.pb │ │ │ │ ├── vectors.pb │ │ │ │ ├── largemetadata.pb │ │ │ │ ├── file.png │ │ │ │ └── thumbnail.png │ │ ├── ingest │ │ │ ├── assets │ │ │ │ ├── text.pb │ │ │ │ ├── vectors.pb │ │ │ │ ├── largemetadata.pb │ │ │ │ └── file.png │ │ │ └── __init__.py │ │ ├── __init__.py │ │ └── consumer │ │ │ └── __init__.py │ ├── __init__.py │ └── unit │ │ ├── __init__.py │ │ ├── consumer │ │ └── __init__.py │ │ └── orm │ │ └── __init__.py │ ├── writer │ ├── assets │ │ ├── text001.txt │ │ └── image001.jpg │ ├── __init__.py │ └── integration │ │ └── api │ │ └── v1 │ │ └── __init__.py │ ├── testdata │ └── invoice.pdf │ ├── __init__.py │ ├── search │ ├── unit │ │ ├── __init__.py │ │ ├── api │ │ │ ├── __init__.py │ │ │ └── v1 │ │ │ │ ├── __init__.py │ │ │ │ └── resource │ │ │ │ └── __init__.py │ │ └── search │ │ │ ├── __init__.py │ │ │ ├── requesters │ │ │ └── __init__.py │ │ │ └── search │ │ │ └── __init__.py │ ├── __init__.py │ └── integration │ │ └── search │ │ └── __init__.py │ ├── train │ └── __init__.py │ ├── nucliadb │ ├── __init__.py │ ├── unit │ │ ├── __init__.py │ │ ├── common │ │ │ ├── __init__.py │ │ │ ├── maindb │ │ │ │ └── __init__.py │ │ │ ├── catalog │ │ │ │ └── __init__.py │ │ │ └── cluster │ │ │ │ └── __init__.py │ │ ├── tasks │ │ │ └── __init__.py │ │ ├── migrator │ │ │ └── __init__.py │ │ └── export_import │ │ │ └── __init__.py │ ├── benchmarks │ │ └── __init__.py │ └── integration │ │ ├── common │ │ ├── __init__.py │ │ ├── cluster │ │ │ └── __init__.py │ │ └── datamanagers │ │ │ └── __init__.py │ │ ├── migrator │ │ └── __init__.py │ │ ├── search │ │ ├── __init__.py │ │ ├── graph │ │ │ └── __init__.py │ │ └── post_retrieval │ │ │ └── __init__.py │ │ └── tasks │ │ └── __init__.py │ ├── reader │ ├── __init__.py │ └── integration │ │ ├── api │ │ └── v1 │ │ │ └── __init__.py │ │ └── reader │ │ └── __init__.py │ ├── standalone │ ├── __init__.py │ └── unit │ │ └── __init__.py │ └── ndbfixtures │ └── __init__.py ├── rustfmt.toml ├── nucliadb_sdk ├── src │ └── nucliadb_sdk │ │ ├── py.typed │ │ ├── tests │ │ └── __init__.py │ │ ├── v2 │ │ └── __init__.py │ │ └── __init__.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── test_feedback.py │ └── test_augment.py └── Makefile ├── e2e ├── requirements.txt └── run-e2e-tests.sh ├── nucliadb_utils ├── src │ └── nucliadb_utils │ │ ├── py.typed │ │ ├── cache │ │ └── __init__.py │ │ ├── fastapi │ │ └── __init__.py │ │ ├── audit │ │ └── __init__.py │ │ ├── nuclia_usage │ │ ├── __init__.py │ │ └── utils │ │ │ └── __init__.py │ │ ├── storages │ │ └── __init__.py │ │ └── store.py ├── README.md ├── Makefile └── tests │ └── unit │ ├── __init__.py │ └── storages │ └── __init__.py ├── nucliadb_dataset ├── src │ └── nucliadb_dataset │ │ ├── py.typed │ │ └── tests │ │ └── __init__.py ├── README.md ├── tests │ ├── unit │ │ ├── __init__.py │ │ └── test_dataset.py │ ├── integration │ │ └── __init__.py │ └── conftest.py └── Makefile ├── nucliadb_models ├── README.md ├── src │ └── nucliadb_models │ │ ├── py.typed │ │ ├── internal │ │ └── __init__.py │ │ ├── graph │ │ └── __init__.py │ │ ├── synonyms.py │ │ ├── processing.py │ │ └── vectorsets.py ├── Makefile ├── tests │ └── __init__.py └── pyproject.toml ├── nucliadb_telemetry ├── src │ └── nucliadb_telemetry │ │ ├── py.typed │ │ ├── tests │ │ ├── grpc │ │ │ ├── Makefile │ │ │ └── __init__.py │ │ └── __init__.py │ │ └── __init__.py ├── tests │ ├── __init__.py │ ├── unit │ │ ├── __init__.py │ │ └── fastapi │ │ │ └── __init__.py │ ├── integration │ │ └── __init__.py │ └── conftest.py └── Makefile ├── .github ├── CODEOWNERS ├── PULL_REQUEST_TEMPLATE.md ├── workflows │ └── osv_scanner.yml └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── nidx ├── migrations │ ├── 20241212151105_check_segment_records.sql │ ├── 20241211120039_merge_job_priority.sql │ ├── 20250110145554_in_flight_messages.sql │ └── 20241211121159_basic_indexes.sql ├── nidx_paragraph │ ├── stop_words │ │ ├── README.md │ │ ├── tr.json │ │ ├── da.json │ │ ├── nl.json │ │ └── sv.json │ └── Cargo.toml ├── nidx_types │ └── Cargo.toml ├── nidx_tests │ └── Cargo.toml ├── nidx_protos │ └── Cargo.toml ├── nidx_tantivy │ └── Cargo.toml ├── .sqlx │ ├── query-d2ad0a0ca2649c9e4873cfcc1fc66d2d07cc45d0f65c560b06d7b5f592f4fa8a.json │ ├── query-a891a37be5c2d7cce775c2dd33726b0318fd3839beab222a1b22bc6174604207.json │ ├── query-4ae09f2c08e2f324bee01bb8487a8f37678a1c5e9d327339235c50d4921a8949.json │ ├── query-a945191bb4b3e37d6823ed3ad499339d007d69983105de8567777d9daf517b28.json │ ├── query-cb29a6556d35ac630ee0aa885dd7341cf9573bd3efd216ff8a887b87686b03db.json │ ├── query-7a3bf27c330c468a596e8a297cf7d8b192e31e67ecc5177c1267f579e8e247c7.json │ ├── query-263c8fce6db5b03bbd012fafdba6943cbee6ed7eb8976cdef4f5b01dde7ca6fd.json │ ├── query-66edb6ea424d8681927dcddb6bac5f1239175f4775d1f40417ba15054b0c6f19.json │ ├── query-2a5d92fb1638df830a4477a7cdf24e6db6b43034b7bbe74fdfb63e8afe2c4071.json │ ├── query-aca588cca57a85e4d7fcc40c23cd87e57d53d11ca550d78e7e3d5e39e524fcd3.json │ ├── query-577109ac00ccfbd38ecaccab94116f2f46a4caf5612afa372cded197123c1e08.json │ ├── query-249b3b57c27a71baa823f1fe0f0bba9c9af36f61c28f731e58beea60ec48e687.json │ ├── query-d6cfe78eb635ba0b89ca4021a4dc8182d18ab5b197f30149cd28488eba4c1df5.json │ ├── query-ebd876fbf5362a5900e75bc05f2f11c73c406ef7da4e95097fc6a1c3d1b8bc54.json │ ├── query-b02f8aafc00a7724510772ac41269e368c5bccf03ef7b4590e0ef6fd1a1bf64f.json │ ├── query-fd97b359c7a7cdc573b2112efbafb9956ae71d8f875c38140cba1c04b7084b47.json │ ├── query-6f9c6d201c1b5712efb68c363bffd3e0169c11f2a8f925e8cd4e8808599ff7b4.json │ ├── query-7efa7c0d747afc4b6aed0586ff846c27839c3213ff7ee9f30c89b0d0f17e60e3.json │ ├── query-3fc3cb39934683de8cd475ce1368c8373453eb1e01f81587d66b9d14b109ce6e.json │ ├── query-5db25f97d8578d6d78f2f6bd4b72cc82a9b1b82805c6422d967ac63b20d99db4.json │ ├── query-7dcbb33312cc9f11ae3a6d73b1ace017a9f19a8bf8f10304fc57977c8efeadff.json │ ├── query-bfcd21ed704cd305db5c17fcdec7d92aa4ac501913c9c9514d8ff92928c0c7e7.json │ ├── query-5ec3233a3a23e926055056d46bdde17836a633066dbb5f349502648cd3ea9a60.json │ ├── query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json │ ├── query-1a561eed00f3dbe868bf5030059793300209179dc8fb73e4b57a54b5e81262fe.json │ ├── query-eef5cc6bce1cc14eba8f3e68971724ef181e88cffcedd74673615f2026b89a62.json │ ├── query-8493bb0059b013eaca42fd10cd7d04f0d06a8acaed379eff0d23f3229edde9ee.json │ ├── query-abe9f7832f2bd799ac44008da031e8d8ab52d4f5fbfc2a7e3974e8873bae55b2.json │ ├── query-1d3fca2682e25a01143da92285297f134a6a105a96f64d87e0db3abb219855e4.json │ ├── query-d9658bfd4e7170b41d03f2ddf2446d0bf54171c0d39d53bf20af2b8437f2ec48.json │ ├── query-0cfce9b29547f8f5bafa6e440f86103be7b8c4ad2fd92db9ac223f4efbe23d10.json │ ├── query-9c8062ea55d070afef68309e58fa987eb37fda44e1efbf68c8ba2af7846cc968.json │ ├── query-ef56d5fefc5774040d1ee397beadb475f6af02768c22f0e583c74062e2e821ce.json │ ├── query-bf49702b506c9a1650ece1f8e8d9f14834a902f8caefafe30ded55e2790f2188.json │ └── query-4fcbdd6657c7dc9b60b3a563dd41711b3dbcf72ce063427b7a01f8cddf34c244.json ├── nidx_binding │ ├── Cargo.toml │ └── pyproject.toml ├── nidx_text │ └── Cargo.toml ├── nidx_relation │ └── Cargo.toml └── nidx_vector │ ├── Cargo.toml │ └── src │ └── vector_types │ └── mod.rs ├── docs ├── assets │ └── images │ │ ├── node_scheme.png │ │ └── nucliadb-arch-overview.png ├── tutorials │ └── README.md ├── reference │ └── widget.md └── internal │ ├── README.md │ ├── GRAPH.md │ ├── ENV_VARS.md │ └── BLOBS.md ├── rust-toolchain.toml ├── proposals ├── records │ ├── images │ │ └── 005 │ │ │ └── overview.drawio.png │ └── template.md └── README.md ├── .git-blame-ignore-revs ├── coverity.yaml ├── nucliadb_protos ├── migrations.proto ├── standalone.proto └── reader.proto ├── charts ├── nucliadb_ingest │ ├── templates │ │ ├── purge.cronjob.yaml │ │ ├── ingest.secret.yaml │ │ ├── ingest-orm-grpc.pdb.yaml │ │ ├── ingest.pdb.yaml │ │ ├── ingest.svc.yaml │ │ ├── ingest-orm-grpc.svc.yaml │ │ ├── ingest-orm-grpc.hpa.yaml │ │ ├── ingest-headless.svc.yaml │ │ ├── ingest-processed-consumer.hpa.yaml │ │ ├── ingest-subscriber-workers.hpa.yaml │ │ ├── _helpers.tpl │ │ └── ingest.cm.yaml │ └── Chart.yaml ├── nidx │ ├── Chart.yaml │ └── templates │ │ ├── nidx-pmon.yaml │ │ ├── api-pdb.yaml │ │ ├── searcher-pdb.yaml │ │ ├── _helpers.tpl │ │ ├── worker-hpa.yaml │ │ ├── indexer-hpa.yaml │ │ ├── api-svc.yaml │ │ └── searcher-svc.yaml ├── nucliadb_reader │ ├── Chart.yaml │ └── templates │ │ ├── reader.secret.yaml │ │ ├── reader.cm.yaml │ │ ├── reader.pdb.yaml │ │ ├── reader.hpa.yaml │ │ └── reader.svc.yaml ├── nucliadb_search │ ├── Chart.yaml │ └── templates │ │ ├── search.secret.yaml │ │ ├── search.cm.yaml │ │ ├── search.pdb.yaml │ │ ├── search.hpa.yaml │ │ ├── search.svc.yaml │ │ └── _helpers.tpl ├── nucliadb_train │ ├── Chart.yaml │ └── templates │ │ ├── train.secret.yaml │ │ ├── train.pdb.yaml │ │ ├── train.hpa.yaml │ │ ├── train.svc.yaml │ │ ├── train.cm.yaml │ │ ├── _helpers.tpl │ │ └── train.vs.yaml ├── nucliadb_writer │ ├── Chart.yaml │ └── templates │ │ ├── writer.secret.yaml │ │ ├── writer.pdb.yaml │ │ ├── writer.hpa.yaml │ │ ├── writer.cm.yaml │ │ ├── writer.svc.yaml │ │ └── _helpers.tpl ├── nucliadb_shared │ ├── Chart.yaml │ └── templates │ │ ├── nucliadb.sm.yaml │ │ └── nucliadb.pm.yaml └── nucliadb │ ├── Chart.yaml │ └── templates │ ├── nidx-worker-svc.yaml │ ├── nidx-svc.yaml │ └── nucliadb-svc.yaml ├── scripts ├── install-protoc.sh ├── wait-for-service.sh └── install-system-deps.sh ├── .coveragerc ├── mypy.ini ├── Dockerfile.nidx_prebuilt ├── ruff.toml ├── .dockerignore ├── Dockerfile.nidx └── .license_header.txt /VERSION: -------------------------------------------------------------------------------- 1 | 6.10.0 -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 120 2 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/ingest/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/reader/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/search/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/train/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/writer/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb_sdk/src/nucliadb_sdk/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /e2e/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | pytest -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/standalone/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb_dataset/src/nucliadb_dataset/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb_models/README.md: -------------------------------------------------------------------------------- 1 | # NucliaDB models 2 | -------------------------------------------------------------------------------- /nucliadb_models/src/nucliadb_models/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nucliadb_telemetry/src/nucliadb_telemetry/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @nuclia/nuclia-db 2 | * @nuclia/lead 3 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/orm/assets/text.pb: -------------------------------------------------------------------------------- 1 | 2 | This is My text -------------------------------------------------------------------------------- /nucliadb/tests/writer/assets/text001.txt: -------------------------------------------------------------------------------- 1 | This is a test text file :) 2 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/ingest/assets/text.pb: -------------------------------------------------------------------------------- 1 | 2 | This is My text -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/orm/assets/vectors.pb: -------------------------------------------------------------------------------- 1 | 2 |  3 | 4 | asdasd 5 | asdasd2 -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/ingest/assets/vectors.pb: -------------------------------------------------------------------------------- 1 | 2 |  3 | 4 | asdasd 5 | asdasd2 -------------------------------------------------------------------------------- /nidx/migrations/20241212151105_check_segment_records.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE segments ADD CHECK(records > 0); 2 | -------------------------------------------------------------------------------- /docs/assets/images/node_scheme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/docs/assets/images/node_scheme.png -------------------------------------------------------------------------------- /nucliadb/tests/testdata/invoice.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/nucliadb/tests/testdata/invoice.pdf -------------------------------------------------------------------------------- /nidx/migrations/20241211120039_merge_job_priority.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE merge_jobs ADD COLUMN priority INT NOT NULL DEFAULT 0; 2 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | # if this changes, also change the version in the Dockerfiles 3 | channel = "1.90.0" 4 | -------------------------------------------------------------------------------- /nucliadb/tests/writer/assets/image001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/nucliadb/tests/writer/assets/image001.jpg -------------------------------------------------------------------------------- /nidx/nidx_paragraph/stop_words/README.md: -------------------------------------------------------------------------------- 1 | # json files generated by the extract script 2 | # they correspond to NLTK's stop word list 3 | -------------------------------------------------------------------------------- /docs/assets/images/nucliadb-arch-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/docs/assets/images/nucliadb-arch-overview.png -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/ingest/assets/largemetadata.pb: -------------------------------------------------------------------------------- 1 | 2 |  3 | holaholNAME 4 |  5 | hol2holNAME 6 | hol 7 | hola -------------------------------------------------------------------------------- /proposals/records/images/005/overview.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/proposals/records/images/005/overview.drawio.png -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/standalone/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/nucliadb/src/nucliadb/standalone/static/favicon.ico -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/orm/assets/largemetadata.pb: -------------------------------------------------------------------------------- 1 | 2 | : 3 |  4 | tok1tokNAME 5 |  6 | tok2tok2NAME 7 | tok 8 | adeu -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | 5a2040d2ed8063a9e0fed6374d291a854c3708d9 2 | d7a3531973890f059abfd43f3cb557d3960c4e30 3 | b8124344de7721709ad4c027c595fceb18ce726a 4 | -------------------------------------------------------------------------------- /coverity.yaml: -------------------------------------------------------------------------------- 1 | capture: 2 | files: 3 | exclude-regex: '(^proposals.*$|^docs.*$|(^|[/\\])charts.*$|(^.*|[/\\])docker-compose\.ya?ml$|(^.*|[/\\])test.*$)' -------------------------------------------------------------------------------- /docs/tutorials/README.md: -------------------------------------------------------------------------------- 1 | # TUTORIALS 2 | 3 | - [Export and Import](EXPORT_IMPORT.md) 4 | - [Train model and Upload info to nuclia](train_and_upload.ipynb) -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/orm/assets/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/nucliadb/tests/ingest/integration/orm/assets/file.png -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/ingest/assets/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/nucliadb/tests/ingest/integration/ingest/assets/file.png -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/orm/assets/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nuclia/nucliadb/HEAD/nucliadb/tests/ingest/integration/orm/assets/thumbnail.png -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Description 2 | Describe the proposed changes made in this PR. 3 | 4 | ### How was this PR tested? 5 | Describe how you tested this PR. 6 | -------------------------------------------------------------------------------- /nidx/nidx_types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_types" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [dependencies] 8 | uuid = "1.13.1" 9 | -------------------------------------------------------------------------------- /nucliadb_dataset/README.md: -------------------------------------------------------------------------------- 1 | # NUCLIADB TRAIN CLIENT 2 | 3 | Library to connect NucliaDB to Training APIs 4 | 5 | ## INSTALL 6 | 7 | ``` 8 | pip install nucliadb_dataset 9 | ``` 10 | -------------------------------------------------------------------------------- /nucliadb_protos/migrations.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package migrations; 3 | 4 | message MigrationInfo { 5 | uint64 current_version = 1; 6 | uint64 target_version = 2; 7 | } 8 | -------------------------------------------------------------------------------- /nidx/migrations/20250110145554_in_flight_messages.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE index_requests ( 2 | seq BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, 3 | received_at TIMESTAMP DEFAULT NOW() 4 | ); 5 | -------------------------------------------------------------------------------- /nucliadb_utils/README.md: -------------------------------------------------------------------------------- 1 | # nucliadb util python library 2 | 3 | - Nats driver 4 | - FastAPI fixes 5 | - S3/GCS drivers 6 | 7 | 8 | # Install and run tests 9 | 10 | ```bash 11 | uv sync 12 | make test 13 | ``` 14 | -------------------------------------------------------------------------------- /docs/reference/widget.md: -------------------------------------------------------------------------------- 1 | # NucliaDB Widget 2 | 3 | NucliaDB has a web widget accessible at `/widget`. This widged powered 4 | by Nuclia allows to search a Knowledge Box easily from a webpage. 5 | 6 | You can test it with your local NucliaDB! 7 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/purge.cronjob.yaml: -------------------------------------------------------------------------------- 1 | {{$data := dict "Values" .Values "Release" .Release "Chart" .Chart "command" "nucliadb-purge" "schedule" "*/15 * * * *" "cronname" "nucliadb-purge-cron"}} 2 | {{- include "purge.cronjob" $data }} 3 | -------------------------------------------------------------------------------- /docs/internal/README.md: -------------------------------------------------------------------------------- 1 | # Internal documentation for NucliaDB 2 | 3 | - [Keys structure on main DB](KV.md) 4 | - [Keys structure on blob storage](BLOBS.md) 5 | - [Internal graph](GRAPH.md) 6 | - [Node structure](NODES.md) 7 | - [Resource example](RESOURCE.md) 8 | -------------------------------------------------------------------------------- /scripts/install-protoc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | mkdir protoc 5 | cd protoc 6 | curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip 7 | unzip protoc-29.3-linux-x86_64.zip 8 | cp -r bin include /usr 9 | -------------------------------------------------------------------------------- /nidx/nidx_tests/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_tests" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [dependencies] 8 | nidx_protos = { version = "0.1.0", path = "../nidx_protos" } 9 | uuid = { version = "1.11.0", features = ["v4"] } 10 | -------------------------------------------------------------------------------- /nidx/nidx_protos/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_protos" 3 | version = "0.1.0" 4 | edition = "2024" 5 | build = "build.rs" 6 | license = "AGPL-3.0-or-later" 7 | 8 | [build-dependencies] 9 | tonic-build = "0.12.3" 10 | 11 | [dependencies] 12 | tonic = "0.12.3" 13 | prost = "0.13" 14 | prost-types = "0.13" 15 | -------------------------------------------------------------------------------- /nucliadb_telemetry/src/nucliadb_telemetry/tests/grpc/Makefile: -------------------------------------------------------------------------------- 1 | protos: 2 | python -m grpc_tools.protoc helloworld.proto -I ./../../../ --python_out=./ --mypy_out=./ --grpc_python_out=./ --mypy_grpc_out=./ 3 | python -m grpc_tools.protoc hellostreamingworld.proto -I ./../../../ --python_out=./ --mypy_out=./ --grpc_python_out=./ --mypy_grpc_out=./ -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | # blank implementations with ... 5 | ^\s*\.\.\.\s*$ 6 | # abstract methods and properties 7 | raise NotImplementedError 8 | @(abc\.)?abstract 9 | # main functions 10 | if __name__ == .__main__.: 11 | 12 | [run] 13 | omit = 14 | *setup.py 15 | */tests/* 16 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.10 3 | 4 | ; Does not have types 5 | [mypy-pytest_docker_fixtures] 6 | ignore_missing_imports = True 7 | 8 | ; Has types but not marked with py.typed 9 | [mypy-mrflagly.*] 10 | follow_untyped_imports = True 11 | 12 | ; Has types but not marked with py.typed 13 | [mypy-fastapi_versioning.*] 14 | follow_untyped_imports = True 15 | -------------------------------------------------------------------------------- /nidx/nidx_tantivy/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_tantivy" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [dependencies] 8 | anyhow = "1.0.91" 9 | nidx_types = { version = "0.1.0", path = "../nidx_types" } 10 | serde = { version = "1.0.214", features = ["derive"] } 11 | serde_json = "1.0.132" 12 | tantivy = "0.24.1" 13 | tantivy-common = "0.9.0" 14 | tempfile = "3.13.0" 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-d2ad0a0ca2649c9e4873cfcc1fc66d2d07cc45d0f65c560b06d7b5f592f4fa8a.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM indexes WHERE id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "d2ad0a0ca2649c9e4873cfcc1fc66d2d07cc45d0f65c560b06d7b5f592f4fa8a" 14 | } 15 | -------------------------------------------------------------------------------- /charts/nidx/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: nidx chart 3 | name: nidx 4 | version: 99999.99999.99999 5 | appVersion: 99999.99999.99999 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-a891a37be5c2d7cce775c2dd33726b0318fd3839beab222a1b22bc6174604207.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM merge_jobs WHERE id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "a891a37be5c2d7cce775c2dd33726b0318fd3839beab222a1b22bc6174604207" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-4ae09f2c08e2f324bee01bb8487a8f37678a1c5e9d327339235c50d4921a8949.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM index_requests WHERE seq = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "4ae09f2c08e2f324bee01bb8487a8f37678a1c5e9d327339235c50d4921a8949" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-a945191bb4b3e37d6823ed3ad499339d007d69983105de8567777d9daf517b28.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM index_requests WHERE received_at < NOW() - INTERVAL '1 minute'", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [] 8 | }, 9 | "nullable": [] 10 | }, 11 | "hash": "a945191bb4b3e37d6823ed3ad499339d007d69983105de8567777d9daf517b28" 12 | } 13 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-cb29a6556d35ac630ee0aa885dd7341cf9573bd3efd216ff8a887b87686b03db.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM merge_jobs WHERE index_id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "cb29a6556d35ac630ee0aa885dd7341cf9573bd3efd216ff8a887b87686b03db" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-7a3bf27c330c468a596e8a297cf7d8b192e31e67ecc5177c1267f579e8e247c7.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM segments WHERE id = ANY($1)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8Array" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "7a3bf27c330c468a596e8a297cf7d8b192e31e67ecc5177c1267f579e8e247c7" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-263c8fce6db5b03bbd012fafdba6943cbee6ed7eb8976cdef4f5b01dde7ca6fd.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE shards SET deleted_at = NOW() WHERE id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Uuid" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "263c8fce6db5b03bbd012fafdba6943cbee6ed7eb8976cdef4f5b01dde7ca6fd" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-66edb6ea424d8681927dcddb6bac5f1239175f4775d1f40417ba15054b0c6f19.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE merge_jobs SET running_at = NOW() WHERE id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "66edb6ea424d8681927dcddb6bac5f1239175f4775d1f40417ba15054b0c6f19" 14 | } 15 | -------------------------------------------------------------------------------- /charts/nucliadb_reader/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: nucliadb Chart 3 | name: nucliadb_reader 4 | version: 99999.99999.99999 5 | appVersion: 99999.99999.99999 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /charts/nucliadb_search/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: nucliadb Chart 3 | name: nucliadb_search 4 | version: 99999.99999.99999 5 | appVersion: 99999.99999.99999 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /charts/nucliadb_train/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: nucliadb Chart 3 | name: nucliadb_train 4 | version: 99999.99999.99999 5 | appVersion: 99999.99999.99999 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /charts/nucliadb_writer/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: nucliadb Chart 3 | name: nucliadb_writer 4 | version: 99999.99999.99999 5 | appVersion: 99999.99999.99999 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: nucliadb Ingest Chart 3 | name: nucliadb_ingest 4 | version: 99999.99999.99999 5 | appVersion: 99999.99999.99999 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /charts/nucliadb_shared/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: NucliaDB Shared Chart 3 | name: nucliadb_shared 4 | version: 99999.99999.99999 5 | appVersion: 99999.99999.99999 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /charts/nucliadb/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: NucliaDB Chart for standalone deployment 3 | name: nucliadb 4 | version: 99999.99999.99999 5 | appVersion: 88888.88888.88888 6 | keywords: 7 | - nucliadb 8 | - nuclia 9 | home: https://nuclia.com 10 | icon: https://nuclia.com/wp-content/uploads/2021/12/logo-fooer.png 11 | sources: 12 | - https://github.com/nuclia/nucliadb 13 | maintainers: 14 | - name: Nuclia 15 | email: nucliadb@nuclia.com 16 | -------------------------------------------------------------------------------- /charts/nucliadb_writer/templates/writer.secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: writer-config 5 | labels: 6 | app: writer 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | data: 12 | {{- if ((.Values.running).sentry_url) }} 13 | SENTRY_URL: {{ .Values.running.sentry_url | b64enc }} 14 | {{- end }} -------------------------------------------------------------------------------- /nidx/.sqlx/query-2a5d92fb1638df830a4477a7cdf24e6db6b43034b7bbe74fdfb63e8afe2c4071.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE segments SET delete_at = NOW() + INTERVAL '5 minutes' WHERE id = ANY($1)", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8Array" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "2a5d92fb1638df830a4477a7cdf24e6db6b43034b7bbe74fdfb63e8afe2c4071" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-aca588cca57a85e4d7fcc40c23cd87e57d53d11ca550d78e7e3d5e39e524fcd3.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE segments SET delete_at = NULL, size_bytes = $1 WHERE id = $2", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8", 9 | "Int8" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "aca588cca57a85e4d7fcc40c23cd87e57d53d11ca550d78e7e3d5e39e524fcd3" 15 | } 16 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest.secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: {{ .Release.Name }}-config 5 | labels: 6 | app: ingest 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | data: 12 | {{- if ((.Values.running).sentry_url) }} 13 | SENTRY_URL: {{ .Values.running.sentry_url | b64enc }} 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /e2e/run-e2e-tests.sh: -------------------------------------------------------------------------------- 1 | # /bin/sh 2 | 3 | set -e 4 | 5 | /opt/nucliadb/bin/pip install -r e2e/requirements.txt 6 | source /opt/nucliadb/bin/activate 7 | export DATA_PATH=data1 8 | export standalone_node_role=index 9 | nucliadb --http-port=8080 & 10 | export DATA_PATH=data2 11 | export standalone_node_role=index 12 | nucliadb --http-port=8081 & 13 | export DATA_PATH=foobar 14 | export standalone_node_role=worker 15 | nucliadb --http-port=8082 & 16 | pytest -s -vv --tb=native e2e/test_e2e.py -------------------------------------------------------------------------------- /proposals/records/template.md: -------------------------------------------------------------------------------- 1 | # [Proposal Title] 2 | 3 | [Short summary of problem and proposed solution. 1-3 sentences.] 4 | 5 | 6 | ## Current solution/situation 7 | 8 | [Detailed description of problem to be solved] 9 | 10 | 11 | ## Proposed Solution 12 | 13 | [Detailed description of proposed solution] 14 | 15 | 16 | ## Rollout plan 17 | 18 | [Describe implementation plan] 19 | 20 | 21 | ## Success Criteria 22 | 23 | [How will we know if the proposal was successful?] 24 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-577109ac00ccfbd38ecaccab94116f2f46a4caf5612afa372cded197123c1e08.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE indexes SET deleted_at = NOW(), name = name || '-deleted-' || gen_random_uuid() WHERE id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "577109ac00ccfbd38ecaccab94116f2f46a4caf5612afa372cded197123c1e08" 14 | } 15 | -------------------------------------------------------------------------------- /Dockerfile.nidx_prebuilt: -------------------------------------------------------------------------------- 1 | # 2 | # Dockerfile for nucliadb's index node, using the provided binaries 3 | # NOTE: Binaries should be built in the same image as this Dockerfile or older (for glibc compatibility) 4 | # For a version that compiles the binaries inside docker, see `Dockerfile.nidx` 5 | # 6 | 7 | FROM ubuntu:24.04 8 | 9 | RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/* 10 | COPY ./builds/nidx /usr/local/bin/nidx 11 | RUN chmod +x /usr/local/bin/nidx 12 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-249b3b57c27a71baa823f1fe0f0bba9c9af36f61c28f731e58beea60ec48e687.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE segments SET merge_job_id = $1 WHERE id = ANY($2) AND merge_job_id IS NULL", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8", 9 | "Int8Array" 10 | ] 11 | }, 12 | "nullable": [] 13 | }, 14 | "hash": "249b3b57c27a71baa823f1fe0f0bba9c9af36f61c28f731e58beea60ec48e687" 15 | } 16 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-d6cfe78eb635ba0b89ca4021a4dc8182d18ab5b197f30149cd28488eba4c1df5.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE segments\n SET delete_at = NOW() + INTERVAL '5 minutes'\n WHERE index_id = $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "d6cfe78eb635ba0b89ca4021a4dc8182d18ab5b197f30149cd28488eba4c1df5" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/migrations/20241211121159_basic_indexes.sql: -------------------------------------------------------------------------------- 1 | -- Indexes for FKs 2 | CREATE INDEX ON indexes(shard_id); 3 | CREATE INDEX ON segments(index_id); 4 | CREATE INDEX ON segments(merge_job_id); 5 | 6 | -- For deciding which indexes to sync 7 | CREATE INDEX ON indexes(updated_at); 8 | 9 | -- For taking jobs from the queue 10 | CREATE INDEX ON merge_jobs(priority); 11 | 12 | -- For purge jobs 13 | CREATE INDEX ON indexes(deleted_at); 14 | CREATE INDEX ON segments(delete_at); 15 | 16 | ANALYZE; 17 | -------------------------------------------------------------------------------- /.github/workflows/osv_scanner.yml: -------------------------------------------------------------------------------- 1 | name: OSV scheduled Scan 2 | 3 | on: 4 | schedule: 5 | - cron: "0 8 * * *" 6 | push: 7 | branches: [main] 8 | 9 | permissions: 10 | actions: read 11 | security-events: write 12 | contents: read 13 | 14 | jobs: 15 | scan-scheduled: 16 | uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v2.2.4" 17 | with: 18 | scan-args: |- 19 | nidx/Cargo.lock 20 | uv.lock 21 | fail-on-vuln: false 22 | -------------------------------------------------------------------------------- /nidx/nidx_paragraph/stop_words/tr.json: -------------------------------------------------------------------------------- 1 | ["acaba", "ama", "asl\u0131nda", "az", "baz\u0131", "belki", "biri", "birka\u00e7", "bir\u015fey", "biz", "bu", "\u00e7ok", "\u00e7\u00fcnk\u00fc", "da", "daha", "de", "defa", "diye", "e\u011fer", "en", "gibi", "hem", "hep", "hepsi", "her", "hi\u00e7", "i\u00e7in", "ile", "ise", "kez", "ki", "kim", "m\u0131", "mu", "m\u00fc", "nas\u0131l", "ne", "neden", "nerde", "nerede", "nereye", "ni\u00e7in", "niye", "o", "sanki", "\u015fey", "siz", "\u015fu", "t\u00fcm", "ve", "veya", "ya", "yani"] -------------------------------------------------------------------------------- /nidx/.sqlx/query-ebd876fbf5362a5900e75bc05f2f11c73c406ef7da4e95097fc6a1c3d1b8bc54.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM shards\n WHERE (\n deleted_at IS NOT NULL\n AND NOT EXISTS(SELECT 1 FROM indexes WHERE shard_id = shards.id)\n )", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [] 8 | }, 9 | "nullable": [] 10 | }, 11 | "hash": "ebd876fbf5362a5900e75bc05f2f11c73c406ef7da4e95097fc6a1c3d1b8bc54" 12 | } 13 | -------------------------------------------------------------------------------- /charts/nucliadb/templates/nidx-worker-svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: nidx-worker 5 | labels: 6 | app: nidx-worker 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | type: ClusterIP 13 | selector: 14 | app: nidx-worker 15 | ports: 16 | - name: nidx-indexer 17 | port: 10002 18 | protocol: TCP 19 | appProtocol: grpc 20 | -------------------------------------------------------------------------------- /charts/nucliadb_reader/templates/reader.secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: reader-config 5 | labels: 6 | app: reader 7 | app.kubernetes.io/name: reader 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | version: "{{ .Chart.Version | replace "+" "_" }}" 11 | chart: "{{ .Chart.Name }}" 12 | data: 13 | {{- if ((.Values.running).sentry_url) }} 14 | SENTRY_URL: {{ .Values.running.sentry_url | b64enc }} 15 | {{- end }} -------------------------------------------------------------------------------- /charts/nucliadb_search/templates/search.secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: search-config 5 | labels: 6 | app: search 7 | app.kubernetes.io/name: search 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | version: "{{ .Chart.Version | replace "+" "_" }}" 11 | chart: "{{ .Chart.Name }}" 12 | data: 13 | {{- if ((.Values.running).sentry_url) }} 14 | SENTRY_URL: {{ .Values.running.sentry_url | b64enc }} 15 | {{- end }} -------------------------------------------------------------------------------- /proposals/README.md: -------------------------------------------------------------------------------- 1 | # NIPs(NucliaDB Improvements Proposals) 2 | 3 | NIPs are a mechanism for proposing, discussing and planning changes to NucliaDB. 4 | 5 | ## NIP Steps 6 | 7 | - Copy `proposals/records/template.md` file to your own file in `proposals/records` 8 | and prefix it with a proposal number(just take next value in sequence): `cp proposals/records/template.md proposals/records/` 9 | - Submit PR with your proposal 10 | - Gather feedback/approval 11 | - Update proposal with implementation changes and phase updates -------------------------------------------------------------------------------- /charts/nucliadb_train/templates/train.secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: train-config 5 | labels: 6 | app: train 7 | app.kubernetes.io/name: train 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | version: "{{ .Chart.Version | replace "+" "_" }}" 11 | chart: "{{ .Chart.Name }}" 12 | data: 13 | {{- if ((.Values.running).sentry_url) }} 14 | SENTRY_URL: {{ .Values.running.sentry_url | b64enc }} 15 | {{- end }} 16 | 17 | -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | target-version = "py39" 2 | line-length = 105 3 | 4 | exclude = [ 5 | "*.pyi", 6 | "src/nucliadb_telemetry/tests/grpc/*", 7 | "src/nucliadb_utils/nuclia_usage/protos/*" 8 | ] 9 | 10 | [lint] 11 | select = [ 12 | "E", # pycodestyle errors 13 | "W", # pycodestyle warnings 14 | "F", # pyflakes 15 | "I", # isort 16 | ] 17 | 18 | ignore = [ 19 | "E501", 20 | "E203", 21 | "E252", 22 | "E701", 23 | "F541", 24 | "F901", 25 | ] 26 | 27 | [lint.isort] 28 | known-first-party = ["nucliadb*"] 29 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/*.egg-info 2 | **/nats-server 3 | **/pd-server 4 | **/tikv-server 5 | .benchmarks/ 6 | .coveragerc 7 | .dockerignore 8 | .eggs 9 | .git/ 10 | .github/ 11 | .license* 12 | .mypy_cache 13 | .pre-commit-config.yaml 14 | .pytest_cache/* 15 | CODE_OF_CONDUCT.md 16 | CODE_STYLE_PYTHON.md 17 | Dockerfile* 18 | Makefile 19 | LICENSE* 20 | README* 21 | __pycache__ 22 | bin 23 | build 24 | charts 25 | coverage.xml 26 | data 27 | include 28 | lib 29 | logs 30 | nats-server 31 | nucliadb_writer/nats-server 32 | **/target/* 33 | tests/* 34 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-b02f8aafc00a7724510772ac41269e368c5bccf03ef7b4590e0ef6fd1a1bf64f.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COUNT(*) AS \"cnt!\" FROM deletions", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "cnt!", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "b02f8aafc00a7724510772ac41269e368c5bccf03ef7b4590e0ef6fd1a1bf64f" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-fd97b359c7a7cdc573b2112efbafb9956ae71d8f875c38140cba1c04b7084b47.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "UPDATE indexes SET updated_at = NOW() WHERE id IN (\n SELECT id FROM indexes WHERE id = ANY ($1) FOR NO KEY UPDATE SKIP LOCKED\n )", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8Array" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "fd97b359c7a7cdc573b2112efbafb9956ae71d8f875c38140cba1c04b7084b47" 14 | } 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | --- 8 | 9 | **Describe the bug** 10 | A clear and concise description of what the bug is. 11 | 12 | **Steps to reproduce (if applicable)** 13 | Steps to reproduce the behavior: 14 | 15 | 1. 16 | 2. 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Configuration:** 22 | Please provide: 23 | 24 | 1. Output of `pip list | grep nucliadb` 25 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-6f9c6d201c1b5712efb68c363bffd3e0169c11f2a8f925e8cd4e8808599ff7b4.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COUNT(*) as \"count!: i64\" FROM shards", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "count!: i64", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "6f9c6d201c1b5712efb68c363bffd3e0169c11f2a8f925e8cd4e8808599ff7b4" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-7efa7c0d747afc4b6aed0586ff846c27839c3213ff7ee9f30c89b0d0f17e60e3.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COUNT(*) as \"count!: i64\" FROM indexes", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "count!: i64", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "7efa7c0d747afc4b6aed0586ff846c27839c3213ff7ee9f30c89b0d0f17e60e3" 20 | } 21 | -------------------------------------------------------------------------------- /nucliadb_models/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: format 2 | format: 3 | ruff check --fix --config=../ruff.toml . 4 | ruff format --config=../ruff.toml . 5 | 6 | .PHONY: lint 7 | lint: 8 | ruff check --config=../ruff.toml . 9 | ruff format --check --config=../ruff.toml . 10 | mypy --config-file=../mypy.ini . 11 | 12 | .PHONY: test 13 | test: 14 | pytest -sv --tb=native tests 15 | 16 | 17 | .PHONY: test-cov 18 | test-cov: 19 | pytest -rfE -sxv --cov=nucliadb_models --cov-config=../.coveragerc --tb=native --cov-report term-missing:skip-covered --cov-report xml tests 20 | -------------------------------------------------------------------------------- /nucliadb_utils/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: format 2 | format: 3 | ruff check --fix --config=../ruff.toml . 4 | ruff format --config=../ruff.toml . 5 | 6 | 7 | .PHONY: lint 8 | lint: 9 | ruff check --config=../ruff.toml . 10 | ruff format --check --config=../ruff.toml . 11 | mypy --config-file=../mypy.ini src 12 | 13 | .PHONY: test 14 | test: 15 | pytest -sv --tb=native tests 16 | 17 | .PHONY: test-cov 18 | test-cov: 19 | pytest -rfE -sxv --cov=nucliadb_utils --cov-config=../.coveragerc --tb=native --cov-report term-missing:skip-covered --cov-report xml tests 20 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-3fc3cb39934683de8cd475ce1368c8373453eb1e01f81587d66b9d14b109ce6e.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COUNT(*) as \"count!: i64\" FROM deletions", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "count!: i64", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "3fc3cb39934683de8cd475ce1368c8373453eb1e01f81587d66b9d14b109ce6e" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-5db25f97d8578d6d78f2f6bd4b72cc82a9b1b82805c6422d967ac63b20d99db4.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO index_requests DEFAULT VALUES RETURNING seq", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "seq", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | false 17 | ] 18 | }, 19 | "hash": "5db25f97d8578d6d78f2f6bd4b72cc82a9b1b82805c6422d967ac63b20d99db4" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-7dcbb33312cc9f11ae3a6d73b1ace017a9f19a8bf8f10304fc57977c8efeadff.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COUNT(*) as \"count!: i64\" FROM merge_jobs", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "count!: i64", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "7dcbb33312cc9f11ae3a6d73b1ace017a9f19a8bf8f10304fc57977c8efeadff" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-bfcd21ed704cd305db5c17fcdec7d92aa4ac501913c9c9514d8ff92928c0c7e7.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COUNT(*) as \"count!: i64\" FROM segments", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "count!: i64", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "bfcd21ed704cd305db5c17fcdec7d92aa4ac501913c9c9514d8ff92928c0c7e7" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-5ec3233a3a23e926055056d46bdde17836a633066dbb5f349502648cd3ea9a60.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COUNT(*) AS \"cnt!\" FROM segments WHERE delete_at IS NULL", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "cnt!", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "5ec3233a3a23e926055056d46bdde17836a633066dbb5f349502648cd3ea9a60" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/nidx_binding/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_binding" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [lib] 8 | name = "nidx_binding" 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | anyhow = "1.0.93" 13 | nidx = { version = "0.1.0", path = "..", default-features = false } 14 | nidx_protos = { version = "0.1.0", path = "../nidx_protos" } 15 | pyo3 = "0.27" 16 | tempfile = "3.14.0" 17 | tokio = "1.41.1" 18 | tokio-util = { version = "0.7.12", features = ["io", "io-util", "compat"] } 19 | tracing-subscriber = "0.3.19" 20 | -------------------------------------------------------------------------------- /charts/nucliadb_reader/templates/reader.cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: reader-config 5 | labels: 6 | app: reader 7 | app.kubernetes.io/name: reader 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | version: "{{ .Chart.Version | replace "+" "_" }}" 11 | chart: "{{ .Chart.Name }}" 12 | data: 13 | JAEGER_ENABLED: {{ .Values.tracing.enabled | quote }} 14 | SERVING_PORT: {{ .Values.serving.port | quote }} 15 | METRICS_PORT: {{ .Values.serving.metricsPort | quote }} -------------------------------------------------------------------------------- /charts/nucliadb_train/templates/train.pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: train 5 | labels: 6 | app.kubernetes.io/instance: {{ .Release.Name }} 7 | app.kubernetes.io/managed-by: {{ .Release.Service }} 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: train 14 | release: "{{ .Release.Name }}" 15 | heritage: "{{ .Release.Service }}" 16 | minAvailable: 1 # simple for now, we can add more complex rules later 17 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT pg_advisory_xact_lock($1)", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "pg_advisory_xact_lock", 9 | "type_info": "Void" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [ 14 | "Int8" 15 | ] 16 | }, 17 | "nullable": [ 18 | null 19 | ] 20 | }, 21 | "hash": "a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247" 22 | } 23 | -------------------------------------------------------------------------------- /charts/nucliadb_reader/templates/reader.pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: reader 5 | labels: 6 | app.kubernetes.io/instance: {{ .Release.Name }} 7 | app.kubernetes.io/managed-by: {{ .Release.Service }} 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: reader 14 | release: "{{ .Release.Name }}" 15 | heritage: "{{ .Release.Service }}" 16 | minAvailable: 1 # simple for now, we can add more complex rules later 17 | -------------------------------------------------------------------------------- /charts/nucliadb_search/templates/search.cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: search-config 5 | labels: 6 | app: search 7 | app.kubernetes.io/name: search 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | version: "{{ .Chart.Version | replace "+" "_" }}" 11 | chart: "{{ .Chart.Name }}" 12 | data: 13 | JAEGER_ENABLED: {{ .Values.tracing.enabled | quote }} 14 | SERVING_PORT: {{ .Values.serving.port | quote }} 15 | METRICS_PORT: {{ .Values.serving.metricsPort | quote }} 16 | -------------------------------------------------------------------------------- /charts/nucliadb_search/templates/search.pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: search 5 | labels: 6 | app.kubernetes.io/instance: {{ .Release.Name }} 7 | app.kubernetes.io/managed-by: {{ .Release.Service }} 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: search 14 | release: "{{ .Release.Name }}" 15 | heritage: "{{ .Release.Service }}" 16 | minAvailable: 1 # simple for now, we can add more complex rules later 17 | -------------------------------------------------------------------------------- /charts/nucliadb_writer/templates/writer.pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: writer 5 | labels: 6 | app.kubernetes.io/instance: {{ .Release.Name }} 7 | app.kubernetes.io/managed-by: {{ .Release.Service }} 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: writer 14 | release: "{{ .Release.Name }}" 15 | heritage: "{{ .Release.Service }}" 16 | minAvailable: 1 # simple for now, we can add more complex rules later 17 | -------------------------------------------------------------------------------- /charts/nidx/templates/nidx-pmon.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: PodMonitor 3 | metadata: 4 | name: nidx-pod-monitor 5 | labels: 6 | version: "{{ .Chart.Version | replace "+" "_" }}" 7 | chart: "{{ .Chart.Name }}" 8 | release: "{{ .Release.Name }}" 9 | heritage: "{{ .Release.Service }}" 10 | spec: 11 | namespaceSelector: 12 | matchNames: 13 | - "{{ .Release.Namespace }}" 14 | selector: 15 | matchLabels: 16 | nidxMetrics: "enabled" 17 | podMetricsEndpoints: 18 | - port: metrics 19 | interval: "30s" 20 | path: /metrics 21 | -------------------------------------------------------------------------------- /charts/nucliadb/templates/nidx-svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: nidx 5 | labels: 6 | app: nidx 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | type: ClusterIP 13 | selector: 14 | app: nidx 15 | ports: 16 | - name: nidx-api 17 | port: 10000 18 | protocol: TCP 19 | appProtocol: grpc 20 | - name: nidx-searcher 21 | port: 10001 22 | protocol: TCP 23 | appProtocol: grpc 24 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-1a561eed00f3dbe868bf5030059793300209179dc8fb73e4b57a54b5e81262fe.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "DELETE FROM indexes\n WHERE (\n deleted_at IS NOT NULL\n AND NOT EXISTS(SELECT 1 FROM segments WHERE index_id = indexes.id)\n AND NOT EXISTS(SELECT 1 FROM deletions where index_id = indexes.id)\n )", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [] 8 | }, 9 | "nullable": [] 10 | }, 11 | "hash": "1a561eed00f3dbe868bf5030059793300209179dc8fb73e4b57a54b5e81262fe" 12 | } 13 | -------------------------------------------------------------------------------- /charts/nidx/templates/api-pdb.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.api.pdb.enabled }} 2 | apiVersion: policy/v1 3 | kind: PodDisruptionBudget 4 | metadata: 5 | name: nidx-api 6 | labels: 7 | app: nidx-api 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | selector: 14 | matchLabels: 15 | app: nidx-api 16 | release: "{{ .Release.Name }}" 17 | heritage: "{{ .Release.Service }}" 18 | minAvailable: {{ .Values.api.pdb.maxUnavailable | default 1 }} 19 | {{- end }} -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest-orm-grpc.pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: ingest-orm-grpc 5 | labels: 6 | app.kubernetes.io/instance: {{ .Release.Name }} 7 | app.kubernetes.io/managed-by: {{ .Release.Service }} 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: ingest-orm-grpc 14 | release: "{{ .Release.Name }}" 15 | heritage: "{{ .Release.Service }}" 16 | minAvailable: 1 # simple for now, we can add more complex rules later 17 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-eef5cc6bce1cc14eba8f3e68971724ef181e88cffcedd74673615f2026b89a62.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "WITH indexes_to_delete AS (\n SELECT indexes.id\n FROM indexes\n WHERE indexes.deleted_at IS NOT NULL\n )\n DELETE FROM deletions USING indexes_to_delete\n WHERE deletions.index_id = indexes_to_delete.id", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [] 8 | }, 9 | "nullable": [] 10 | }, 11 | "hash": "eef5cc6bce1cc14eba8f3e68971724ef181e88cffcedd74673615f2026b89a62" 12 | } 13 | -------------------------------------------------------------------------------- /Dockerfile.nidx: -------------------------------------------------------------------------------- 1 | # 2 | # Dockerfile for nucliadb's index node, compiling nidx in the docker build. 3 | # For a version that uses pre-built binaries (outside Docker) see `Dockerfile.nidx_prebuild` 4 | # 5 | 6 | FROM rust:1.90.0-trixie AS builder 7 | RUN apt update && apt install -y protobuf-compiler 8 | COPY nucliadb_protos/*proto /app/nucliadb_protos/ 9 | COPY nidx /app/nidx 10 | RUN cd /app/nidx && cargo build --locked --release 11 | 12 | FROM debian:trixie-slim AS nidx 13 | RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/* 14 | COPY --from=builder /app/nidx/target/release/nidx /usr/local/bin 15 | -------------------------------------------------------------------------------- /charts/nucliadb_shared/templates/nucliadb.sm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | name: ndb-sm 5 | labels: 6 | version: "{{ .Chart.Version | replace "+" "_" }}" 7 | chart: "{{ .Chart.Name }}" 8 | release: "{{ .Release.Name }}" 9 | heritage: "{{ .Release.Service }}" 10 | spec: 11 | namespaceSelector: 12 | matchNames: 13 | - "{{ .Release.Namespace }}" 14 | selector: 15 | matchLabels: 16 | metrics: "enabled" 17 | endpoints: 18 | - port: metrics 19 | interval: {{ ((.Values.serviceMonitor).interval) | default "60s" }} 20 | path: /metrics -------------------------------------------------------------------------------- /nidx/nidx_text/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_text" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [dependencies] 8 | tempfile = "3" 9 | itertools = "0.12" 10 | 11 | tantivy = "0.24.1" 12 | nidx_types = { version = "0.1.0", path = "../nidx_types" } 13 | nidx_protos = { version = "0.1.0", path = "../nidx_protos" } 14 | nidx_tantivy = { version = "0.1.0", path = "../nidx_tantivy" } 15 | anyhow = "1.0.91" 16 | tracing = "0.1.40" 17 | uuid = "1.12.0" 18 | serde = { version = "1.0.210", features = ["derive"] } 19 | 20 | [dev-dependencies] 21 | nidx_tests = { path = "../nidx_tests" } 22 | -------------------------------------------------------------------------------- /scripts/wait-for-service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MAX_TIMEOUT=60 4 | COMMAND="nc -zv 127.0.0.1 $1" 5 | 6 | start_time=$(date +%s) 7 | while true; do 8 | $COMMAND 9 | exit_code=$? 10 | current_time=$(date +%s) 11 | elapsed_time=$((current_time - start_time)) 12 | 13 | if [ $exit_code -eq 0 ]; then 14 | echo "Running on 127.0.0.1 $1." 15 | break 16 | elif [ $elapsed_time -ge $MAX_TIMEOUT ]; then 17 | echo "Not runing within the maximum timeout of $MAX_TIMEOUT seconds." 18 | exit 1 19 | break 20 | else 21 | echo "Waiting to come up..." 22 | sleep 1 # You can adjust the sleep time as needed 23 | fi 24 | done 25 | -------------------------------------------------------------------------------- /charts/nucliadb_train/templates/train.hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: train 5 | labels: 6 | app: train 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: train 16 | minReplicas: {{.Values.autoscaling.minReplicas}} 17 | maxReplicas: {{.Values.autoscaling.maxReplicas}} 18 | metrics: {{- toYaml .Values.autoscaling.metrics | nindent 4}} 19 | -------------------------------------------------------------------------------- /charts/nucliadb_reader/templates/reader.hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: reader 5 | labels: 6 | app: reader 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: reader 16 | minReplicas: {{.Values.autoscaling.minReplicas}} 17 | maxReplicas: {{.Values.autoscaling.maxReplicas}} 18 | metrics: {{- toYaml .Values.autoscaling.metrics | nindent 4}} 19 | -------------------------------------------------------------------------------- /charts/nucliadb_search/templates/search.hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: search 5 | labels: 6 | app: search 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: search 16 | minReplicas: {{.Values.autoscaling.minReplicas}} 17 | maxReplicas: {{.Values.autoscaling.maxReplicas}} 18 | metrics: {{- toYaml .Values.autoscaling.metrics | nindent 4}} 19 | -------------------------------------------------------------------------------- /charts/nucliadb_writer/templates/writer.hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: writer 5 | labels: 6 | app: writer 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: writer 16 | minReplicas: {{.Values.autoscaling.minReplicas}} 17 | maxReplicas: {{.Values.autoscaling.maxReplicas}} 18 | metrics: {{- toYaml .Values.autoscaling.metrics | nindent 4}} 19 | -------------------------------------------------------------------------------- /charts/nidx/templates/searcher-pdb.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.searcher.pdb.enabled }} 2 | apiVersion: policy/v1 3 | kind: PodDisruptionBudget 4 | metadata: 5 | name: nidx-searcher 6 | labels: 7 | app: nidx-searcher 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | selector: 14 | matchLabels: 15 | app: nidx-searcher 16 | release: "{{ .Release.Name }}" 17 | heritage: "{{ .Release.Service }}" 18 | maxUnavailable: {{ .Values.searcher.pdb.maxUnavailable | default 1 }} 19 | {{- end }} -------------------------------------------------------------------------------- /charts/nucliadb_shared/templates/nucliadb.pm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: PodMonitor 3 | metadata: 4 | name: ndb-pm 5 | labels: 6 | version: "{{ .Chart.Version | replace "+" "_" }}" 7 | chart: "{{ .Chart.Name }}" 8 | release: "{{ .Release.Name }}" 9 | heritage: "{{ .Release.Service }}" 10 | spec: 11 | namespaceSelector: 12 | matchNames: 13 | - "{{ .Release.Namespace }}" 14 | selector: 15 | matchLabels: 16 | metrics: "enabled" 17 | podMetricsEndpoints: 18 | - port: metrics 19 | interval: {{ ((.Values.podMonitor).interval) | default "60s" }} 20 | path: /metrics 21 | -------------------------------------------------------------------------------- /charts/nucliadb_train/templates/train.svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: train 5 | labels: 6 | app: train 7 | metrics: "enabled" 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | type: ClusterIP 14 | selector: 15 | app: train 16 | ports: 17 | - name: grpc-train 18 | port: {{ .Values.serving.grpc_port }} 19 | - name: http-train 20 | port: {{ .Values.serving.port }} 21 | - name: metrics 22 | port: {{ .Values.serving.metricsPort}} -------------------------------------------------------------------------------- /charts/nucliadb_train/templates/train.cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: train-config 5 | labels: 6 | app: train 7 | app.kubernetes.io/name: train 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | version: "{{ .Chart.Version | replace "+" "_" }}" 11 | chart: "{{ .Chart.Name }}" 12 | data: 13 | GRPC_PORT: {{ .Values.serving.grpc_port | quote }} 14 | SERVING_PORT: {{ .Values.serving.port | quote }} 15 | JAEGER_ENABLED: {{ .Values.tracing.enabled | quote }} 16 | METRICS_PORT: {{ .Values.serving.metricsPort | quote }} 17 | 18 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-8493bb0059b013eaca42fd10cd7d04f0d06a8acaed379eff0d23f3229edde9ee.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COALESCE(MAX(seq), 1) AS \"seq!\" FROM segments\n UNION\n SELECT COALESCE(MAX(seq), 1) AS \"seq!\" FROM deletions", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "seq!", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "8493bb0059b013eaca42fd10cd7d04f0d06a8acaed379eff0d23f3229edde9ee" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/nidx_relation/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_relation" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [dependencies] 8 | anyhow = "1.0.92" 9 | deunicode = "1.3.2" 10 | nidx_protos = { version = "0.1.0", path = "../nidx_protos" } 11 | nidx_tantivy = { version = "0.1.0", path = "../nidx_tantivy" } 12 | nidx_types = { version = "0.1.0", path = "../nidx_types" } 13 | serde = { version = "1.0.219", features = ["derive"] } 14 | tantivy = "0.24" 15 | tracing = "0.1.41" 16 | uuid = "1.16.0" 17 | 18 | [dev-dependencies] 19 | tempfile = "3" 20 | lazy_static = "1.4.0" 21 | nidx_tests = { path = "../nidx_tests" } 22 | -------------------------------------------------------------------------------- /nucliadb_sdk/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /charts/nucliadb_writer/templates/writer.cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: writer-config 5 | labels: 6 | app: writer 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | data: 12 | DM_REDIS_HOST: {{ .Values.config.dm_redis_host }} 13 | DM_REDIS_PORT: {{ .Values.config.dm_redis_port | quote }} 14 | {{- if .Values.tracing.enabled }} 15 | JAEGER_ENABLED: "True" 16 | {{- end }} 17 | SERVING_PORT: {{ .Values.serving.port | quote }} 18 | METRICS_PORT: {{ .Values.serving.metricsPort | quote }} 19 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest.pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: ingest 5 | labels: 6 | app.kubernetes.io/instance: {{ .Release.Name }} 7 | app.kubernetes.io/managed-by: {{ .Release.Service }} 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: ingest 14 | app.kubernetes.io/name: ingest 15 | app.kubernetes.io/instance: {{ .Release.Name }} 16 | app.kubernetes.io/managed-by: {{ .Release.Service }} 17 | minAvailable: 1 # simple for now, we can add more complex rules later 18 | -------------------------------------------------------------------------------- /nucliadb_dataset/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /nucliadb_models/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /nucliadb_telemetry/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-abe9f7832f2bd799ac44008da031e8d8ab52d4f5fbfc2a7e3974e8873bae55b2.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT COALESCE(MIN(seq) - 1, (SELECT last_value FROM index_requests_seq_seq)) AS \"seq!\"\n FROM index_requests WHERE received_at > NOW() - INTERVAL '1 minute'", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "seq!", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | null 17 | ] 18 | }, 19 | "hash": "abe9f7832f2bd799ac44008da031e8d8ab52d4f5fbfc2a7e3974e8873bae55b2" 20 | } 21 | -------------------------------------------------------------------------------- /nucliadb_sdk/src/nucliadb_sdk/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /nucliadb_telemetry/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest.svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: ingest 5 | labels: 6 | app: ingest 7 | metrics: "enabled" 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | type: ClusterIP 14 | selector: 15 | app: ingest 16 | ports: 17 | - name: grpc-ingest 18 | port: {{ .Values.serving.grpc }} 19 | protocol: TCP 20 | appProtocol: grpc 21 | - name: metrics 22 | port: {{ .Values.serving.metricsPort}} 23 | protocol: TCP 24 | appProtocol: http -------------------------------------------------------------------------------- /nucliadb_dataset/tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /nucliadb_telemetry/tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /nucliadb_telemetry/tests/unit/fastapi/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /charts/nucliadb_reader/templates/reader.svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: reader 5 | labels: 6 | app: reader 7 | metrics: "enabled" 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | type: ClusterIP 14 | selector: 15 | app: reader 16 | ports: 17 | - name: http-reader 18 | port: {{ .Values.serving.port }} 19 | protocol: TCP 20 | appProtocol: http 21 | - name: metrics 22 | port: {{ .Values.serving.metricsPort}} 23 | protocol: TCP 24 | appProtocol: http 25 | -------------------------------------------------------------------------------- /charts/nucliadb_search/templates/search.svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: search 5 | labels: 6 | app: search 7 | metrics: "enabled" 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | type: ClusterIP 14 | selector: 15 | app: search 16 | ports: 17 | - name: http-search 18 | port: {{ .Values.serving.port }} 19 | protocol: TCP 20 | appProtocol: http 21 | - name: metrics 22 | port: {{ .Values.serving.metricsPort}} 23 | protocol: TCP 24 | appProtocol: http 25 | -------------------------------------------------------------------------------- /charts/nucliadb_writer/templates/writer.svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: writer 5 | labels: 6 | app: writer 7 | metrics: "enabled" 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | type: ClusterIP 14 | selector: 15 | app: writer 16 | ports: 17 | - name: http-writer 18 | port: {{ .Values.serving.port }} 19 | protocol: TCP 20 | appProtocol: http 21 | - name: metrics 22 | port: {{ .Values.serving.metricsPort}} 23 | protocol: TCP 24 | appProtocol: http 25 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-1d3fca2682e25a01143da92285297f134a6a105a96f64d87e0db3abb219855e4.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT NOW() - MIN(updated_at) FROM indexes WHERE shard_id = ANY($1) AND updated_at > $2 AND deleted_at IS NULL", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "?column?", 9 | "type_info": "Interval" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [ 14 | "UuidArray", 15 | "Timestamp" 16 | ] 17 | }, 18 | "nullable": [ 19 | null 20 | ] 21 | }, 22 | "hash": "1d3fca2682e25a01143da92285297f134a6a105a96f64d87e0db3abb219855e4" 23 | } 24 | -------------------------------------------------------------------------------- /nucliadb_dataset/src/nucliadb_dataset/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /nucliadb_models/src/nucliadb_models/internal/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /nucliadb_sdk/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: format 2 | format: 3 | ruff check --fix --config=../ruff.toml . 4 | ruff format --config=../ruff.toml . 5 | 6 | .PHONY: lint 7 | lint: 8 | ruff check --config=../ruff.toml . 9 | ruff format --check --config=../ruff.toml . 10 | mypy --config-file=../mypy.ini . 11 | 12 | .PHONY: test 13 | test: 14 | TEST_LOCAL_NUCLIADB=START pytest -rfE -sv --tb=native tests 15 | 16 | .PHONY: test-cov 17 | test-cov: 18 | TEST_LOCAL_NUCLIADB=START pytest \ 19 | -rfE -sxv \ 20 | --cov=nucliadb_sdk \ 21 | --cov=nucliadb_models \ 22 | --cov-config=../.coveragerc \ 23 | --cov-report term-missing:skip-covered \ 24 | --cov-report xml \ 25 | --tb=native \ 26 | tests 27 | -------------------------------------------------------------------------------- /nucliadb_telemetry/src/nucliadb_telemetry/tests/grpc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | -------------------------------------------------------------------------------- /nucliadb_telemetry/src/nucliadb_telemetry/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # 16 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest-orm-grpc.svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: ingest-orm-grpc 5 | labels: 6 | app: ingest-orm-grpc 7 | metrics: "enabled" 8 | version: "{{ .Chart.Version | replace "+" "_" }}" 9 | chart: "{{ .Chart.Name }}" 10 | release: "{{ .Release.Name }}" 11 | heritage: "{{ .Release.Service }}" 12 | spec: 13 | type: ClusterIP 14 | selector: 15 | app: ingest-orm-grpc 16 | ports: 17 | - name: grpc-ingest 18 | port: {{ .Values.serving.grpc }} 19 | protocol: TCP 20 | appProtocol: grpc 21 | - name: metrics 22 | port: {{ .Values.serving.metricsPort}} 23 | protocol: TCP 24 | appProtocol: http -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | --- 8 | 9 | **Is your feature request related to a problem? Please describe.** 10 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 11 | 12 | **Describe the solution you'd like** 13 | A clear and concise description of what you want to happen. 14 | 15 | **Describe alternatives you've considered** 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | **Additional context** 19 | Add any other context or information about the feature request here. 20 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest-orm-grpc.hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: ingest-orm-grpc 5 | labels: 6 | app: ingest-orm-grpc 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: ingest-orm-grpc 16 | minReplicas: {{.Values.ingest_orm_grpc_autoscaling.minReplicas}} 17 | maxReplicas: {{.Values.ingest_orm_grpc_autoscaling.maxReplicas}} 18 | metrics: {{- toYaml .Values.ingest_orm_grpc_autoscaling.metrics | nindent 4}} 19 | -------------------------------------------------------------------------------- /nucliadb_protos/standalone.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package standalone; 4 | 5 | message NodeActionRequest { 6 | string service = 1; 7 | string action = 2; 8 | bytes payload = 3; 9 | } 10 | 11 | message NodeActionResponse { 12 | bytes payload = 1; 13 | } 14 | 15 | message NodeInfoRequest { 16 | 17 | } 18 | 19 | message NodeInfoResponse { 20 | string id = 1; 21 | string address = 2; 22 | uint32 shard_count = 3; 23 | uint64 available_disk = 4; 24 | uint64 total_disk = 5; 25 | } 26 | 27 | 28 | service StandaloneClusterService { 29 | rpc NodeAction(NodeActionRequest) returns (NodeActionResponse) {} 30 | rpc NodeInfo(NodeInfoRequest) returns (NodeInfoResponse) {} 31 | } 32 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-d9658bfd4e7170b41d03f2ddf2446d0bf54171c0d39d53bf20af2b8437f2ec48.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT segments.id AS \"id: SegmentId\"\n FROM segments\n LEFT JOIN indexes ON segments.index_id = indexes.id\n WHERE segments.delete_at < NOW() OR indexes.deleted_at IS NOT NULL", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "id: SegmentId", 9 | "type_info": "Int8" 10 | } 11 | ], 12 | "parameters": { 13 | "Left": [] 14 | }, 15 | "nullable": [ 16 | false 17 | ] 18 | }, 19 | "hash": "d9658bfd4e7170b41d03f2ddf2446d0bf54171c0d39d53bf20af2b8437f2ec48" 20 | } 21 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-0cfce9b29547f8f5bafa6e440f86103be7b8c4ad2fd92db9ac223f4efbe23d10.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "WITH oldest_segments AS (\n SELECT index_id, MIN(seq) AS seq FROM segments\n WHERE delete_at IS NULL\n GROUP BY index_id\n )\n DELETE FROM deletions USING oldest_segments\n WHERE deletions.index_id = oldest_segments.index_id\n AND deletions.seq <= oldest_segments.seq\n AND deletions.seq <= $1", 4 | "describe": { 5 | "columns": [], 6 | "parameters": { 7 | "Left": [ 8 | "Int8" 9 | ] 10 | }, 11 | "nullable": [] 12 | }, 13 | "hash": "0cfce9b29547f8f5bafa6e440f86103be7b8c4ad2fd92db9ac223f4efbe23d10" 14 | } 15 | -------------------------------------------------------------------------------- /nidx/nidx_paragraph/stop_words/da.json: -------------------------------------------------------------------------------- 1 | ["og", "i", "jeg", "det", "at", "en", "den", "til", "er", "som", "p\u00e5", "de", "med", "han", "af", "for", "ikke", "der", "var", "mig", "sig", "men", "et", "har", "om", "vi", "min", "havde", "ham", "hun", "nu", "over", "da", "fra", "du", "ud", "sin", "dem", "os", "op", "man", "hans", "hvor", "eller", "hvad", "skal", "selv", "her", "alle", "vil", "blev", "kunne", "ind", "n\u00e5r", "v\u00e6re", "dog", "noget", "ville", "jo", "deres", "efter", "ned", "skulle", "denne", "end", "dette", "mit", "ogs\u00e5", "under", "have", "dig", "anden", "hende", "mine", "alt", "meget", "sit", "sine", "vor", "mod", "disse", "hvis", "din", "nogle", "hos", "blive", "mange", "ad", "bliver", "hendes", "v\u00e6ret", "thi", "jer", "s\u00e5dan"] -------------------------------------------------------------------------------- /nucliadb_dataset/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: format 2 | format: 3 | ruff check --fix --config=../ruff.toml . 4 | ruff format --config=../ruff.toml . 5 | 6 | .PHONY: lint 7 | lint: 8 | ruff check --config=../ruff.toml . 9 | ruff format --check --config=../ruff.toml . 10 | mypy --config-file=../mypy.ini . 11 | 12 | .PHONY: test 13 | test: 14 | TEST_LOCAL_NUCLIADB=START pytest -rfE -sv --tb=native tests 15 | 16 | .PHONY: test-cov 17 | test-cov: 18 | TEST_LOCAL_NUCLIADB=START pytest \ 19 | -rfE -sxv \ 20 | --cov=nucliadb_dataset \ 21 | --cov=nucliadb_models \ 22 | --cov=nucliadb_sdk \ 23 | --cov-config=../.coveragerc \ 24 | --cov-report term-missing:skip-covered \ 25 | --cov-report xml \ 26 | --cov-append \ 27 | --tb=native \ 28 | tests 29 | -------------------------------------------------------------------------------- /nidx/nidx_binding/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=1.7,<2.0"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "nidx_binding" 7 | description = "Bindings for nidx (part of nucliadb)" 8 | license = "AGPL-3.0-or-later" 9 | authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }] 10 | requires-python = ">=3.10, <4" 11 | classifiers = [ 12 | "Programming Language :: Rust", 13 | "Programming Language :: Python :: Implementation :: CPython", 14 | "Programming Language :: Python :: Implementation :: PyPy", 15 | ] 16 | version = "6.10.0" 17 | 18 | [project.urls] 19 | Homepage = "https://nuclia.com" 20 | Repository = "https://github.com/nuclia/nucliadb" 21 | 22 | [tool.maturin] 23 | features = ["pyo3/extension-module"] 24 | -------------------------------------------------------------------------------- /nucliadb_telemetry/src/nucliadb_telemetry/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import logging 16 | 17 | logger = logging.getLogger("nucliadb_telemetry") 18 | -------------------------------------------------------------------------------- /nucliadb_sdk/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | pytest_plugins = [ 16 | "pytest_docker_fixtures", 17 | "nucliadb_sdk.tests.fixtures", 18 | ] 19 | -------------------------------------------------------------------------------- /nidx/nidx_paragraph/stop_words/nl.json: -------------------------------------------------------------------------------- 1 | ["de", "en", "van", "ik", "te", "dat", "die", "in", "een", "hij", "het", "niet", "zijn", "is", "was", "op", "aan", "met", "als", "voor", "had", "er", "maar", "om", "hem", "dan", "zou", "of", "wat", "mijn", "men", "dit", "zo", "door", "over", "ze", "zich", "bij", "ook", "tot", "je", "mij", "uit", "der", "daar", "haar", "naar", "heb", "hoe", "heeft", "hebben", "deze", "u", "want", "nog", "zal", "me", "zij", "nu", "ge", "geen", "omdat", "iets", "worden", "toch", "al", "waren", "veel", "meer", "doen", "toen", "moet", "ben", "zonder", "kan", "hun", "dus", "alles", "onder", "ja", "eens", "hier", "wie", "werd", "altijd", "doch", "wordt", "wezen", "kunnen", "ons", "zelf", "tegen", "na", "reeds", "wil", "kon", "niets", "uw", "iemand", "geweest", "andere"] -------------------------------------------------------------------------------- /nucliadb_models/src/nucliadb_models/graph/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from . import ( # noqa: F401 17 | requests, 18 | responses, 19 | ) 20 | -------------------------------------------------------------------------------- /charts/nidx/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Render a list of environment variables from a map of key-value pairs. 3 | 4 | Possible values for the map are: 5 | - string: the value is rendered as a string 6 | example: 7 | key: value 8 | - map: the value is rendered as a YAML object 9 | example: 10 | key: 11 | valueFrom: 12 | secretKeyRef: 13 | name: secret-name 14 | key: secret-key 15 | */}} 16 | {{- define "toEnv" }} 17 | {{- range $key, $value := . }} 18 | {{- if kindIs "string" $value }} 19 | - name: "{{ $key }}" 20 | value: {{ tpl $value $ | quote }} 21 | {{- else if kindIs "map" $value}} 22 | - name: "{{ $key }}" 23 | {{ $value | toYaml | indent 2}} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest-headless.svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: ingest-headless 5 | labels: 6 | app: ingest 7 | app.kubernetes.io/name: ingest 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | version: "{{ .Chart.Version | replace "+" "_" }}" 11 | chart: "{{ .Chart.Name }}" 12 | spec: 13 | selector: 14 | app: ingest 15 | chart: "{{ .Chart.Name }}" 16 | clusterIP: None 17 | ports: 18 | - name: grpc-ingest 19 | port: {{ .Values.serving.grpc }} 20 | protocol: TCP 21 | appProtocol: grpc 22 | - name: metrics 23 | port: {{ .Values.serving.metricsPort}} 24 | protocol: TCP 25 | appProtocol: http -------------------------------------------------------------------------------- /nidx/nidx_paragraph/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_paragraph" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [dependencies] 8 | anyhow = "1.0.92" 9 | itertools = "0.13.0" 10 | lazy_static = "1.5.0" 11 | levenshtein_automata = "0.2.1" 12 | nidx_protos = { version = "0.1.0", path = "../nidx_protos" } 13 | nidx_tantivy = { version = "0.1.0", path = "../nidx_tantivy" } 14 | nidx_types = { version = "0.1.0", path = "../nidx_types" } 15 | nom = "7" # same version as used by tantivy 16 | once_cell = "1.20.2" 17 | regex = "1.11.1" 18 | serde_json = "1.0.132" 19 | tantivy = "0.24.1" 20 | tantivy-common = "0.9.0" 21 | tantivy-fst = "0.5.0" 22 | tracing = "0.1.40" 23 | 24 | [build-dependencies] 25 | 26 | [dev-dependencies] 27 | tempfile = "3.13.0" 28 | -------------------------------------------------------------------------------- /charts/nidx/templates/worker-hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: nidx-worker 5 | labels: 6 | app: nidx-worker 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: nidx-worker 16 | minReplicas: {{ dig "hpa" "minReplicas" 1 .Values.worker}} 17 | maxReplicas: {{ dig "hpa" "maxReplicas" 10 .Values.worker}} 18 | {{- with .Values.worker.hpa }} 19 | metrics: {{- toYaml .metrics | nindent 4}} 20 | {{- with .behavior }} 21 | behavior: {{- toYaml . | nindent 4}} 22 | {{- end }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest-processed-consumer.hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: ingest-processed-consumer 5 | labels: 6 | app: ingest-processed-consumer 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: ingest-processed-consumer 16 | minReplicas: {{.Values.ingest_processed_consumer_autoscaling.minReplicas}} 17 | maxReplicas: {{.Values.ingest_processed_consumer_autoscaling.maxReplicas}} 18 | metrics: {{- toYaml .Values.ingest_processed_consumer_autoscaling.metrics | nindent 4}} 19 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest-subscriber-workers.hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: ingest-subscriber-workers 5 | labels: 6 | app: ingest-subscriber-workers 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: ingest-subscriber-workers 16 | minReplicas: {{.Values.ingest_subscriber_workers_autoscaling.minReplicas}} 17 | maxReplicas: {{.Values.ingest_subscriber_workers_autoscaling.maxReplicas}} 18 | metrics: {{- toYaml .Values.ingest_subscriber_workers_autoscaling.metrics | nindent 4}} 19 | -------------------------------------------------------------------------------- /nucliadb_sdk/src/nucliadb_sdk/v2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .sdk import NucliaDB, NucliaDBAsync, Region 16 | 17 | __all__ = ("NucliaDB", "NucliaDBAsync", "Region") 18 | -------------------------------------------------------------------------------- /charts/nidx/templates/indexer-hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: nidx-indexer 5 | labels: 6 | app: nidx-indexer 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | scaleTargetRef: 13 | apiVersion: apps/v1 14 | kind: Deployment 15 | name: nidx-indexer 16 | minReplicas: {{ dig "hpa" "minReplicas" 1 .Values.indexer}} 17 | maxReplicas: {{ dig "hpa" "maxReplicas" 10 .Values.indexer}} 18 | {{- with .Values.indexer.hpa }} 19 | metrics: {{- toYaml .metrics | nindent 4}} 20 | {{- with .behavior }} 21 | behavior: {{- toYaml . | nindent 4}} 22 | {{- end }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Render a list of environment variables from a map of key-value pairs. 3 | 4 | Possible values for the map are: 5 | - string: the value is rendered as a string 6 | example: 7 | key: value 8 | - map: the value is rendered as a YAML object 9 | example: 10 | key: 11 | valueFrom: 12 | secretKeyRef: 13 | name: secret-name 14 | key: secret-key 15 | */}} 16 | {{- define "toEnv" }} 17 | {{- range $key, $value := . }} 18 | {{- if kindIs "string" $value }} 19 | - name: "{{ $key }}" 20 | value: {{ tpl $value $ | quote }} 21 | {{- else if kindIs "map" $value}} 22 | - name: "{{ $key }}" 23 | {{ $value | toYaml | indent 2}} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /charts/nucliadb_search/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Render a list of environment variables from a map of key-value pairs. 3 | 4 | Possible values for the map are: 5 | - string: the value is rendered as a string 6 | example: 7 | key: value 8 | - map: the value is rendered as a YAML object 9 | example: 10 | key: 11 | valueFrom: 12 | secretKeyRef: 13 | name: secret-name 14 | key: secret-key 15 | */}} 16 | {{- define "toEnv" }} 17 | {{- range $key, $value := . }} 18 | {{- if kindIs "string" $value }} 19 | - name: "{{ $key }}" 20 | value: {{ tpl $value $ | quote }} 21 | {{- else if kindIs "map" $value}} 22 | - name: "{{ $key }}" 23 | {{ $value | toYaml | indent 2}} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /charts/nucliadb_train/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Render a list of environment variables from a map of key-value pairs. 3 | 4 | Possible values for the map are: 5 | - string: the value is rendered as a string 6 | example: 7 | key: value 8 | - map: the value is rendered as a YAML object 9 | example: 10 | key: 11 | valueFrom: 12 | secretKeyRef: 13 | name: secret-name 14 | key: secret-key 15 | */}} 16 | {{- define "toEnv" }} 17 | {{- range $key, $value := . }} 18 | {{- if kindIs "string" $value }} 19 | - name: "{{ $key }}" 20 | value: {{ tpl $value $ | quote }} 21 | {{- else if kindIs "map" $value}} 22 | - name: "{{ $key }}" 23 | {{ $value | toYaml | indent 2}} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /charts/nucliadb_writer/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Render a list of environment variables from a map of key-value pairs. 3 | 4 | Possible values for the map are: 5 | - string: the value is rendered as a string 6 | example: 7 | key: value 8 | - map: the value is rendered as a YAML object 9 | example: 10 | key: 11 | valueFrom: 12 | secretKeyRef: 13 | name: secret-name 14 | key: secret-key 15 | */}} 16 | {{- define "toEnv" }} 17 | {{- range $key, $value := . }} 18 | {{- if kindIs "string" $value }} 19 | - name: "{{ $key }}" 20 | value: {{ tpl $value $ | quote }} 21 | {{- else if kindIs "map" $value}} 22 | - name: "{{ $key }}" 23 | {{ $value | toYaml | indent 2}} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /nucliadb_models/src/nucliadb_models/synonyms.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | from pydantic import BaseModel 16 | 17 | 18 | class KnowledgeBoxSynonyms(BaseModel): 19 | synonyms: dict[str, list[str]] 20 | -------------------------------------------------------------------------------- /nucliadb_dataset/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | pytest_plugins = [ 16 | "pytest_docker_fixtures", 17 | "nucliadb_sdk.tests.fixtures", 18 | "nucliadb_dataset.tests.fixtures", 19 | ] 20 | -------------------------------------------------------------------------------- /nucliadb_telemetry/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | pytest_plugins = [ 17 | "pytest_docker_fixtures", 18 | "nucliadb_utils.tests.nats", 19 | "nucliadb_telemetry.tests.telemetry", 20 | ] 21 | -------------------------------------------------------------------------------- /charts/nidx/templates/api-svc.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.api.service.enabled }} 2 | kind: Service 3 | apiVersion: v1 4 | metadata: 5 | name: nidx-api 6 | labels: 7 | app: nidx-api 8 | metrics: "enabled" 9 | version: "{{ .Chart.Version | replace "+" "_" }}" 10 | chart: "{{ .Chart.Name }}" 11 | release: "{{ .Release.Name }}" 12 | heritage: "{{ .Release.Service }}" 13 | {{- with .Values.api.service.labels }} 14 | {{- toYaml . | nindent 4 }} 15 | {{- end }} 16 | {{- with .Values.api.service.annotations }} 17 | annotations: 18 | {{- toYaml . | nindent 4 }} 19 | {{- end }} 20 | spec: 21 | type: {{ .Values.api.service.type | default "ClusterIP" }} 22 | selector: 23 | app: nidx-api 24 | ports: 25 | - name: nidx-api 26 | port: 10000 27 | protocol: TCP 28 | appProtocol: grpc 29 | {{- end }} -------------------------------------------------------------------------------- /docs/internal/GRAPH.md: -------------------------------------------------------------------------------- 1 | # Components 2 | 3 | ```mermaid 4 | graph TD 5 | O[One] -->|Embed| W(Writer) 6 | O -->|Embed| R(Reader) 7 | O -->|Embed| S(Search) 8 | W -->|GRPC| I(Ingest) 9 | R -->|GRPC| I 10 | W -->|TXN| I 11 | S -->|Embed| I 12 | I -->|GRPC| N1(Node 1) 13 | S -->|GRPC| N1 14 | I -->|GRPC| N2(Node 2) 15 | I -.->|SWIM| N1 16 | I -.->|SWIM| N2 17 | S -.->|SWIM| N1 18 | S -.->|SWIM| N2 19 | S -->|GRPC| N2 20 | I --> G(GCS-S3-AFS) 21 | R --> G 22 | I --> T(TIKV-REDIS) 23 | R --> T 24 | S --> T 25 | R --> C(Cache-Pubsub) 26 | W --> C 27 | S --> C 28 | subgraph node 29 | N1 -.->|SWIM| N2 30 | N2 -.->|SWIM| N1 31 | N3 -.->|SWIM| N1 32 | N1 -.->|SWIM| N3(Node 3) 33 | N3 -.->|SWIM| N2 34 | N2 -.->|SWIM| N3 35 | end 36 | ``` 37 | -------------------------------------------------------------------------------- /nucliadb_models/src/nucliadb_models/processing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Optional 16 | 17 | from pydantic import BaseModel 18 | 19 | 20 | class PushProcessingOptions(BaseModel): 21 | ml_text: Optional[bool] = True 22 | -------------------------------------------------------------------------------- /scripts/install-system-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | UNAME="$(uname -s)" 5 | case "${UNAME}" in 6 | Linux*) machine=linux ;; 7 | Darwin*) machine=macos ;; 8 | CYGWIN*) machine=cygwin ;; 9 | MINGW*) machine=mingw ;; 10 | *) machine="UNKNOWN:${UNAME}" ;; 11 | esac 12 | 13 | if [ "$machine" == "linux" ]; then 14 | SUDO='' 15 | if (($EUID != 0)); then 16 | SUDO='sudo' 17 | fi 18 | 19 | source /etc/os-release 20 | case $ID in 21 | debian | ubuntu | mint) 22 | $SUDO apt-get -y update || true 23 | $SUDO apt-get install -y libdw-dev pkg-config libssl-dev cpanminus 24 | $SUDO cpanm IPC::Cmd 25 | ;; 26 | 27 | fedora | rhel | centos) 28 | $SUDO yum update -y 29 | $SUDO yum -y install elfutils-devel pkgconfig openssl-devel perl-IPC-Cmd 30 | ;; 31 | 32 | *) 33 | echo -n "unsupported linux distro" 34 | ;; 35 | esac 36 | fi 37 | -------------------------------------------------------------------------------- /docs/internal/ENV_VARS.md: -------------------------------------------------------------------------------- 1 | # Environment variables 2 | 3 | Usage guidelines: 4 | - namespace should be used that describes the usage and dependency 5 | - same settings should be used in env vars, helm charts and CLI usage 6 | - reuse like settings like http and grpc server host and port 7 | 8 | 9 | Settings[to be implemented yet]: 10 | - `ENVIRONMENT` 11 | - `LOG_LEVEL` 12 | - `LOGGER_LEVELS` 13 | - `DEBUG` 14 | - `HTTP_HOST` 15 | - `HTTP_CORS_ORIGINS` 16 | - `GRPC_HOST` 17 | - `METRICS_HTTP_HOST` 18 | - `SEARCH_TIMEOUT`: Time in seconds for which the search endpoints wait for the index node search request response before we timeout and return an HTTP error. 19 | - `SEARCH_CACHE_REDIS_HOST`: Hostname of the redis instance used as cache by the search component. 20 | - `SEARCH_CACHE_REDIS_PORT`: Port of the redis instance used as cache by the search component. -------------------------------------------------------------------------------- /nidx/.sqlx/query-9c8062ea55d070afef68309e58fa987eb37fda44e1efbf68c8ba2af7846cc968.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO shards (kbid) VALUES ($1) RETURNING *", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "id", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "kbid", 14 | "type_info": "Uuid" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "deleted_at", 19 | "type_info": "Timestamp" 20 | } 21 | ], 22 | "parameters": { 23 | "Left": [ 24 | "Uuid" 25 | ] 26 | }, 27 | "nullable": [ 28 | false, 29 | false, 30 | true 31 | ] 32 | }, 33 | "hash": "9c8062ea55d070afef68309e58fa987eb37fda44e1efbf68c8ba2af7846cc968" 34 | } 35 | -------------------------------------------------------------------------------- /nucliadb_sdk/tests/test_feedback.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import nucliadb_sdk 16 | 17 | 18 | def test_feedback(docs_dataset, sdk: nucliadb_sdk.NucliaDB): 19 | sdk.feedback(kbid=docs_dataset, ident="foo", good=True, task="CHAT", feedback="bar") 20 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-ef56d5fefc5774040d1ee397beadb475f6af02768c22f0e583c74062e2e821ce.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT * FROM shards WHERE id = $1 AND deleted_at IS NULL", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "id", 9 | "type_info": "Uuid" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "kbid", 14 | "type_info": "Uuid" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "deleted_at", 19 | "type_info": "Timestamp" 20 | } 21 | ], 22 | "parameters": { 23 | "Left": [ 24 | "Uuid" 25 | ] 26 | }, 27 | "nullable": [ 28 | false, 29 | false, 30 | true 31 | ] 32 | }, 33 | "hash": "ef56d5fefc5774040d1ee397beadb475f6af02768c22f0e583c74062e2e821ce" 34 | } 35 | -------------------------------------------------------------------------------- /nucliadb_sdk/src/nucliadb_sdk/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from nucliadb_sdk.v2 import NucliaDB, NucliaDBAsync, Region, exceptions 16 | 17 | __all__ = ( 18 | "NucliaDB", 19 | "NucliaDBAsync", 20 | "Region", 21 | "exceptions", 22 | ) 23 | -------------------------------------------------------------------------------- /nucliadb_protos/reader.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "google/protobuf/timestamp.proto"; 4 | 5 | // Ask for information to be retrieved 6 | 7 | // Public interface of nucliadb to connect external 8 | 9 | // Get information from trainer 10 | // Authentication based on certificate 11 | 12 | 13 | message ResourceGet { 14 | string kb = 1; 15 | string uuid = 2; 16 | 17 | bool core = 3; 18 | bool files = 4; 19 | bool links = 5; 20 | bool relations = 6; 21 | bool extracted = 7; 22 | bool vectors = 8; 23 | bool metadata = 9; 24 | bool large_metadata = 10; 25 | bool origin = 11; 26 | bool conversation = 12; 27 | 28 | } 29 | 30 | service TrainingService { 31 | rpc GetLearningToRank(X) returns (X) {} 32 | rpc GetResource(ResourceGet) returns (resources.Resource) {} 33 | rpc GetResourcesList(X) returns (X) {} 34 | } 35 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-bf49702b506c9a1650ece1f8e8d9f14834a902f8caefafe30ded55e2790f2188.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "SELECT * FROM deletions WHERE index_id = $1 AND seq <= $2", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "index_id", 9 | "type_info": "Int8" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "seq", 14 | "type_info": "Int8" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "keys", 19 | "type_info": "TextArray" 20 | } 21 | ], 22 | "parameters": { 23 | "Left": [ 24 | "Int8", 25 | "Int8" 26 | ] 27 | }, 28 | "nullable": [ 29 | false, 30 | false, 31 | false 32 | ] 33 | }, 34 | "hash": "bf49702b506c9a1650ece1f8e8d9f14834a902f8caefafe30ded55e2790f2188" 35 | } 36 | -------------------------------------------------------------------------------- /charts/nidx/templates/searcher-svc.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.searcher.service.enabled }} 2 | kind: Service 3 | apiVersion: v1 4 | metadata: 5 | name: nidx-searcher 6 | labels: 7 | app: nidx-searcher 8 | metrics: "enabled" 9 | version: "{{ .Chart.Version | replace "+" "_" }}" 10 | chart: "{{ .Chart.Name }}" 11 | release: "{{ .Release.Name }}" 12 | heritage: "{{ .Release.Service }}" 13 | {{- with .Values.searcher.service.labels }} 14 | {{- toYaml . | nindent 4 }} 15 | {{- end }} 16 | {{- with .Values.searcher.service.annotations }} 17 | annotations: 18 | {{- toYaml . | nindent 4 }} 19 | {{- end }} 20 | spec: 21 | type: {{ .Values.searcher.service.type | default "ClusterIP" }} 22 | selector: 23 | app: nidx-searcher 24 | ports: 25 | - name: nidx-searcher 26 | port: 10001 27 | protocol: TCP 28 | appProtocol: grpc 29 | {{- end }} -------------------------------------------------------------------------------- /nucliadb_sdk/tests/test_augment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import nucliadb_sdk 15 | from nucliadb_models.augment import AugmentRequest 16 | 17 | 18 | def test_retrieve(docs_dataset: str, sdk: nucliadb_sdk.NucliaDB): 19 | _ = sdk._augment(kbid=docs_dataset, content=AugmentRequest()) 20 | -------------------------------------------------------------------------------- /charts/nucliadb/templates/nucliadb-svc.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: nucliadb 5 | labels: 6 | app: nucliadb 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | type: ClusterIP 13 | selector: 14 | app: nucliadb 15 | ports: 16 | - name: http 17 | {{- if .Values.service }} 18 | port: {{ .Values.service.http_port }} 19 | targetPort: 8080 20 | protocol: TCP 21 | appProtocol: http 22 | {{- end }} 23 | - name: train 24 | port: 8040 25 | protocol: TCP 26 | appProtocol: grpc 27 | - name: grpc 28 | port: 8060 29 | protocol: TCP 30 | appProtocol: grpc 31 | - name: node 32 | port: 10009 33 | protocol: TCP 34 | appProtocol: grpc 35 | -------------------------------------------------------------------------------- /charts/nucliadb_train/templates/train.vs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1beta1 2 | kind: VirtualService 3 | metadata: 4 | name: train 5 | spec: 6 | gateways: 7 | - "{{ .Values.vs.gateway }}" 8 | hosts: 9 | - "{{ .Values.zone }}.{{ .Values.vs.host }}" 10 | http: 11 | - name: nucliadb_train 12 | match: 13 | - uri: 14 | regex: '^/api/v\d+/kb/[^/]+/trainset$' 15 | method: 16 | regex: "GET|POST|OPTIONS" 17 | - uri: 18 | regex: '^/api/v\d+/kb/[^/]+/trainset/[^/]+$' 19 | method: 20 | regex: "GET|POST|OPTIONS" 21 | 22 | retries: 23 | attempts: 3 24 | retryOn: connect-failure 25 | route: 26 | - destination: 27 | port: 28 | number: {{.Values.serving.port}} 29 | host: "train.{{ .Release.Namespace }}.svc.cluster.local" 30 | -------------------------------------------------------------------------------- /nidx/nidx_vector/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "nidx_vector" 3 | version = "0.1.0" 4 | edition = "2024" 5 | license = "AGPL-3.0-or-later" 6 | 7 | [dependencies] 8 | nidx_types = { version = "0.1.0", path = "../nidx_types" } 9 | tempfile = "3.13.0" 10 | uuid = { version = "1.10.0", features = ["v4", "serde"] } 11 | rand = { version = "0.8.4", features = ["small_rng"] } 12 | memmap2 = "0.5.3" 13 | thiserror = "1.0.31" 14 | bit-set = "0.8.0" 15 | libc = "0.2.153" 16 | lazy_static = "1.4.0" 17 | nidx_protos = { version = "0.1.0", path = "../nidx_protos" } 18 | tracing = "0.1.40" 19 | serde = { version = "1.0.210", features = ["derive"] } 20 | anyhow = "1.0.89" 21 | rustc-hash = "2.1" 22 | stream-vbyte = "0.4.1" 23 | fst = "0.4.7" 24 | bit-vec = "0.8.0" 25 | bincode = "2.0.1" 26 | rayon = "1.10.0" 27 | simsimd = "4.3.1" 28 | 29 | [dev-dependencies] 30 | lazy_static = "1.4.0" 31 | rstest = "0.19.0" 32 | -------------------------------------------------------------------------------- /charts/nucliadb_ingest/templates/ingest.cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ .Release.Name }}-config 5 | labels: 6 | app: ingest 7 | version: "{{ .Chart.Version | replace "+" "_" }}" 8 | chart: "{{ .Chart.Name }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | data: 12 | GRPC_PORT: {{ .Values.serving.grpc | quote }} 13 | METRICS_PORT: {{ .Values.serving.metricsPort | quote }} 14 | PULL_TIME_ERROR_BACKOFF: {{ .Values.config.pull_time_error_backoff | quote }} 15 | {{- if .Values.debug }} 16 | LOG_LEVEL: "DEBUG" 17 | {{- end }} 18 | INDEX_JETSTREAM_SERVERS: {{ toJson .Values.indexing.index_jetstream_servers | quote }} 19 | INDEX_JETSTREAM_AUTH: {{ .Values.indexing.index_jetstream_auth }} 20 | TOTAL_REPLICAS: {{ .Values.replicaCount | quote }} 21 | JAEGER_ENABLED: {{ .Values.tracing.enabled | quote }} 22 | -------------------------------------------------------------------------------- /nidx/.sqlx/query-4fcbdd6657c7dc9b60b3a563dd41711b3dbcf72ce063427b7a01f8cddf34c244.json: -------------------------------------------------------------------------------- 1 | { 2 | "db_name": "PostgreSQL", 3 | "query": "INSERT INTO deletions (index_id, seq, keys) VALUES ($1, $2, $3) RETURNING *", 4 | "describe": { 5 | "columns": [ 6 | { 7 | "ordinal": 0, 8 | "name": "index_id", 9 | "type_info": "Int8" 10 | }, 11 | { 12 | "ordinal": 1, 13 | "name": "seq", 14 | "type_info": "Int8" 15 | }, 16 | { 17 | "ordinal": 2, 18 | "name": "keys", 19 | "type_info": "TextArray" 20 | } 21 | ], 22 | "parameters": { 23 | "Left": [ 24 | "Int8", 25 | "Int8", 26 | "TextArray" 27 | ] 28 | }, 29 | "nullable": [ 30 | false, 31 | false, 32 | false 33 | ] 34 | }, 35 | "hash": "4fcbdd6657c7dc9b60b3a563dd41711b3dbcf72ce063427b7a01f8cddf34c244" 36 | } 37 | -------------------------------------------------------------------------------- /nucliadb/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb_telemetry/Makefile: -------------------------------------------------------------------------------- 1 | 2 | protos: 3 | python -m grpc_tools.protoc ./nucliadb_telemetry/tests/grpc/helloworld.proto -I ./ --python_out=./ --mypy_out=./ --grpc_python_out=./ --mypy_grpc_out=./ 4 | python -m grpc_tools.protoc ./nucliadb_telemetry/tests/grpc/hellostreamingworld.proto -I ./ --python_out=./ --mypy_out=./ --grpc_python_out=./ --mypy_grpc_out=./ 5 | 6 | .PHONY: format 7 | format: 8 | ruff check --fix --config=../ruff.toml . 9 | ruff format --config=../ruff.toml . 10 | 11 | .PHONY: lint 12 | lint: 13 | ruff check --config=../ruff.toml . 14 | ruff format --check --config=../ruff.toml . 15 | mypy --config-file=../mypy.ini . --show-traceback 16 | 17 | .PHONY: test 18 | test: 19 | pytest -sv --tb=native tests 20 | 21 | .PHONY: test-cov 22 | test-cov: 23 | pytest -rfE -sxv --cov=nucliadb_telemetry --cov-config=../.coveragerc --tb=native --cov-report term-missing:skip-covered --cov-report xml tests 24 | -------------------------------------------------------------------------------- /nucliadb/tests/search/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/tests/train/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb_utils/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/src/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/reader/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/search/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/standalone/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/standalone/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/tests/writer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /.license_header.txt: -------------------------------------------------------------------------------- 1 | // Copyright (C) {\d+} Bosutech XXI S.L. 2 | // 3 | // nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | // For commercial licensing, contact us at info@nuclia.com. 5 | // 6 | // AGPL: 7 | // This program is free software: you can redistribute it and/or modify 8 | // it under the terms of the GNU Affero General Public License as 9 | // published by the Free Software Foundation, either version 3 of the 10 | // License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, 13 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU Affero General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU Affero General Public License 18 | // along with this program. If not, see . 19 | // 20 | -------------------------------------------------------------------------------- /nucliadb/src/migrations/pg/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/backups/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/migrator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/unit/consumer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/unit/orm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/search/unit/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/search/unit/search/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb_utils/tests/unit/storages/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /docs/internal/BLOBS.md: -------------------------------------------------------------------------------- 1 | # nucliadb Key Blob Store 2 | 3 | The prefix for storing binary data on nucliadb are: 4 | 5 | - `/kbs/{kbid}/` 6 | - `/r/` Scan all resources 7 | - `{rid}` 8 | - `/files/{field}` 9 | - `/extracted` 10 | - `/thumbnail` 11 | - `/preview` 12 | - `/largemetadata` 13 | - `/metadata` 14 | - `/rows` 15 | - `/pages` 16 | - `/{page}` 17 | - `/links/{link}` 18 | - `/extracted` 19 | - `/thumbnail` 20 | - `/preview` 21 | - `/largemetadata` 22 | - `/metadata` 23 | - `/simple/{field}` 24 | - `/extracted` 25 | - `/thumbnail` 26 | - `/conversation/{field}` 27 | - `/extracted` 28 | - `/thumbnail` 29 | - `/formatedtext/{field}` 30 | - `/extracted` 31 | - `/thumbnail` 32 | - `/blocktext/{field}` 33 | - `/extracted` 34 | - `/thumbnail` 35 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/maindb/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/ingest/fields/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/ingest/orm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/reader/reader/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/search/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/search/requesters/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/search/search/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/standalone/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/train/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/writer/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/consumer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/common/maindb/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/search/integration/search/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/tests/search/unit/api/v1/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb_dataset/tests/unit/test_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from nucliadb_dataset.dataset import NucliaDataset 16 | 17 | 18 | def test_inst_base_class(): 19 | try: 20 | NucliaDataset() 21 | except TypeError: 22 | pass 23 | else: 24 | raise AssertionError("Base class should'nt be instantiable") 25 | -------------------------------------------------------------------------------- /nucliadb_models/src/nucliadb_models/vectorsets.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Bosutech XXI S.L. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | from pydantic import BaseModel 17 | 18 | 19 | class CreatedVectorSet(BaseModel): 20 | id: str 21 | 22 | 23 | class VectorSetListItem(BaseModel): 24 | id: str 25 | 26 | 27 | class VectorSetList(BaseModel): 28 | vectorsets: list[VectorSetListItem] 29 | -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/cache/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/fastapi/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/cluster/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/cluster/standalone/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/ingest/consumer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/ingest/service/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/models/internal/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/search/search/chat/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/train/generators/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/writer/resource/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/ingest/integration/ingest/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/migrator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/reader/integration/api/v1/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/reader/integration/reader/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/search/unit/search/requesters/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | -------------------------------------------------------------------------------- /nucliadb/tests/search/unit/search/search/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/writer/integration/api/v1/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/audit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/http_clients/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/models_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/search/api/v1/resource/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/migrator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/search/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/common/catalog/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/common/cluster/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/unit/export_import/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/search/unit/api/v1/resource/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb_models/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "nucliadb_models" 7 | version = "6.10.0" 8 | license = "Apache-2.0" 9 | readme = "README.md" 10 | authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }] 11 | classifiers = [ 12 | "Development Status :: 4 - Beta", 13 | "Programming Language :: Python", 14 | "Programming Language :: Python :: 3.10", 15 | "Programming Language :: Python :: 3.11", 16 | "Programming Language :: Python :: 3.12", 17 | "Programming Language :: Python :: 3 :: Only", 18 | "Topic :: Software Development :: Libraries :: Python Modules", 19 | ] 20 | requires-python = ">=3.10, <4" 21 | # https://github.com/pydantic/pydantic/issues/11978 22 | dependencies = ["pydantic>=2.6,!=2.11.5,!=2.11.6"] 23 | 24 | [project.urls] 25 | Homepage = "https://nuclia.com" 26 | Repository = "https://github.com/nuclia/nucliadb" 27 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/search/search/query_parser/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/common/cluster/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/search/graph/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/nuclia_usage/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/external_index_providers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/common/datamanagers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/nuclia_usage/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/nucliadb/integration/search/post_retrieval/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | -------------------------------------------------------------------------------- /nucliadb/tests/ndbfixtures/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | """NucliaDB fixtures.""" 22 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | AVG_PARAGRAPH_SIZE_BYTES = 10_000 21 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/reader/api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | DEFAULT_RESOURCE_LIST_PAGE_SIZE = 20 21 | -------------------------------------------------------------------------------- /nidx/nidx_paragraph/stop_words/sv.json: -------------------------------------------------------------------------------- 1 | ["och", "det", "att", "i", "en", "jag", "hon", "som", "han", "p\u00e5", "den", "med", "var", "sig", "f\u00f6r", "s\u00e5", "till", "\u00e4r", "men", "ett", "om", "hade", "de", "av", "icke", "mig", "du", "henne", "d\u00e5", "sin", "nu", "har", "inte", "hans", "honom", "skulle", "hennes", "d\u00e4r", "min", "man", "ej", "vid", "kunde", "n\u00e5got", "fr\u00e5n", "ut", "n\u00e4r", "efter", "upp", "vi", "dem", "vara", "vad", "\u00f6ver", "\u00e4n", "dig", "kan", "sina", "h\u00e4r", "ha", "mot", "alla", "under", "n\u00e5gon", "eller", "allt", "mycket", "sedan", "ju", "denna", "sj\u00e4lv", "detta", "\u00e5t", "utan", "varit", "hur", "ingen", "mitt", "ni", "bli", "blev", "oss", "din", "dessa", "n\u00e5gra", "deras", "blir", "mina", "samma", "vilken", "er", "s\u00e5dan", "v\u00e5r", "blivit", "dess", "inom", "mellan", "s\u00e5dant", "varf\u00f6r", "varje", "vilka", "ditt", "vem", "vilket", "sitta", "s\u00e5dana", "vart", "dina", "vars", "v\u00e5rt", "v\u00e5ra", "ert", "era", "vilkas"] -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/storages/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | MB = 1024 * 1024 21 | CHUNK_SIZE = 5 * MB 22 | -------------------------------------------------------------------------------- /nidx/nidx_vector/src/vector_types/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2021 Bosutech XXI S.L. 2 | // 3 | // nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | // For commercial licensing, contact us at info@nuclia.com. 5 | // 6 | // AGPL: 7 | // This program is free software: you can redistribute it and/or modify 8 | // it under the terms of the GNU Affero General Public License as 9 | // published by the Free Software Foundation, either version 3 of the 10 | // License, or (at your option) any later version. 11 | // 12 | // This program is distributed in the hope that it will be useful, 13 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU Affero General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU Affero General Public License 18 | // along with this program. If not, see . 19 | // 20 | 21 | pub mod dense_f32; 22 | pub mod rabitq; 23 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | 21 | import logging 22 | 23 | logger = logging.getLogger("nucliadb") 24 | -------------------------------------------------------------------------------- /nucliadb_utils/src/nucliadb_utils/store.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | 20 | from typing import Any, Dict 21 | 22 | MAIN: Dict[str, Any] = {} 23 | -------------------------------------------------------------------------------- /nucliadb/src/nucliadb/common/datamanagers/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2021 Bosutech XXI S.L. 2 | # 3 | # nucliadb is offered under the AGPL v3.0 and as commercial software. 4 | # For commercial licensing, contact us at info@nuclia.com. 5 | # 6 | # AGPL: 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU Affero General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with this program. If not, see . 19 | # 20 | class KnowledgeBoxNotFound(Exception): 21 | pass 22 | --------------------------------------------------------------------------------