├── .circleci └── config.yml ├── .editorconfig ├── .gitignore ├── .pre-commit-config.yaml ├── .prettierignore ├── .prettierrc.cjs ├── LICENSE ├── README.md ├── app ├── VERSION ├── commands │ ├── agent │ │ ├── archiver.py │ │ └── controller.py │ ├── cache │ │ └── clear.py │ ├── calculate.py │ ├── db │ │ ├── backup.py │ │ ├── clean.py │ │ ├── restore.py │ │ └── snapshots.py │ ├── destroy.py │ ├── gpu.py │ ├── group │ │ └── children.py │ ├── import.py │ ├── info.py │ ├── listen.py │ ├── log │ │ ├── abort.py │ │ ├── clean.py │ │ ├── get.py │ │ └── rerun.py │ ├── mixins │ │ ├── config.py │ │ ├── db.py │ │ ├── log.py │ │ ├── message.py │ │ ├── module.py │ │ ├── notification.py │ │ ├── platform.py │ │ └── schedule.py │ ├── module │ │ ├── add.py │ │ ├── create.py │ │ ├── init.py │ │ └── install.py │ ├── notification │ │ ├── clear.py │ │ ├── remove.py │ │ └── save.py │ ├── run.py │ ├── scale.py │ ├── send.py │ ├── service │ │ ├── follow.py │ │ └── lock │ │ │ ├── clear.py │ │ │ ├── set.py │ │ │ └── wait.py │ ├── task.py │ ├── template │ │ └── generate.py │ ├── test.py │ ├── user │ │ └── rotate.py │ └── version.py ├── components │ ├── config_store.py │ ├── data.py │ ├── destroy.py │ ├── group.py │ ├── models.py │ ├── post_destroy.py │ ├── post_run.py │ ├── pre_destroy.py │ ├── pre_run.py │ ├── profile.py │ ├── roles.py │ └── run.py ├── data │ ├── base │ │ ├── id_resource.py │ │ └── name_resource.py │ ├── cache │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── config │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── dataset │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── group │ │ ├── cache.py │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── host │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ ├── 0002_alter_host_token_alter_host_user.py │ │ │ └── __init__.py │ │ └── models.py │ ├── log │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── mixins │ │ ├── group.py │ │ ├── provider.py │ │ └── resource.py │ ├── module │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── notification │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── scaling_event │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── schedule │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ ├── state │ │ ├── migrations │ │ │ ├── 0001_initial.py │ │ │ └── __init__.py │ │ └── models.py │ └── user │ │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ │ └── models.py ├── deploy.sh ├── help │ └── en │ │ ├── cache.yml │ │ ├── calculate.yml │ │ ├── config.yml │ │ ├── dataset.yml │ │ ├── db.yml │ │ ├── environment.yml │ │ ├── group.yml │ │ ├── import.yml │ │ ├── log.yml │ │ ├── module.yml │ │ ├── notification.yml │ │ ├── schedule.yml │ │ ├── service.yml │ │ ├── state.yml │ │ └── user.yml ├── plugins │ ├── base.py │ ├── calculation │ │ ├── addition.py │ │ ├── base.py │ │ ├── cov.py │ │ ├── division.py │ │ ├── functions │ │ │ └── date.py │ │ ├── min_max_scale.py │ │ ├── multiplication.py │ │ ├── pchange.py │ │ ├── stdev.py │ │ ├── subtraction.py │ │ └── zscore.py │ ├── data.py │ ├── data_processor │ │ ├── base.py │ │ ├── drop_duplicates.py │ │ ├── drop_na.py │ │ ├── shuffle.py │ │ └── sort.py │ ├── dataset │ │ ├── base.py │ │ ├── collection.py │ │ └── period.py │ ├── encryption │ │ ├── aes256.py │ │ ├── aes256_user.py │ │ └── base.py │ ├── field_processor │ │ ├── base.py │ │ ├── bool_to_number.py │ │ └── combined_text.py │ ├── formatter │ │ ├── base.py │ │ ├── capitalize.py │ │ ├── date.py │ │ ├── date_time.py │ │ ├── integer.py │ │ ├── joiner.py │ │ ├── lower.py │ │ ├── number.py │ │ ├── remove_suffix.py │ │ ├── string.py │ │ ├── title.py │ │ └── upper.py │ ├── function │ │ ├── base.py │ │ ├── calculations.py │ │ ├── capitalize.py │ │ ├── data_atomic_fields.py │ │ ├── data_dynamic_fields.py │ │ ├── data_id.py │ │ ├── data_key.py │ │ ├── data_query_fields.py │ │ ├── data_relation_fields.py │ │ ├── data_reverse_relation_fields.py │ │ ├── data_scope_fields.py │ │ ├── default.py │ │ ├── filter.py │ │ ├── flatten.py │ │ ├── join.py │ │ ├── keys.py │ │ ├── lstrip.py │ │ ├── mock_data.py │ │ ├── normalize.py │ │ ├── prefix.py │ │ ├── random_keys.py │ │ ├── random_values.py │ │ ├── rstrip.py │ │ ├── split.py │ │ ├── substitute.py │ │ ├── time.py │ │ ├── time_range.py │ │ ├── value.py │ │ └── values.py │ ├── mixins │ │ ├── cli_task.py │ │ ├── csv_source.py │ │ ├── list_calculation.py │ │ ├── module_template.py │ │ └── ssh_task.py │ ├── module │ │ ├── base.py │ │ ├── core.py │ │ ├── git.py │ │ ├── github.py │ │ └── local.py │ ├── parser │ │ ├── base.py │ │ ├── conditional_value.py │ │ ├── config.py │ │ ├── function.py │ │ ├── reference.py │ │ ├── state.py │ │ └── token.py │ ├── source │ │ ├── base.py │ │ └── csv_file.py │ ├── task │ │ ├── base.py │ │ ├── command.py │ │ ├── remote_command.py │ │ ├── remote_script.py │ │ ├── script.py │ │ └── upload.py │ ├── validator │ │ ├── base.py │ │ ├── date_time.py │ │ ├── exists.py │ │ ├── number.py │ │ ├── string.py │ │ └── unique.py │ └── worker │ │ ├── base.py │ │ ├── docker.py │ │ └── kubernetes.py ├── profiles │ ├── display.yml │ ├── migrate.yml │ ├── test.yml │ └── test │ │ ├── base.yml │ │ ├── cache.yml │ │ ├── config.yml │ │ ├── data │ │ ├── base.yml │ │ ├── basic.yml │ │ ├── plugin.yml │ │ └── remove.yml │ │ ├── dataset.yml │ │ ├── dependency.yml │ │ ├── failure.yml │ │ ├── group.yml │ │ ├── host.yml │ │ ├── log.yml │ │ ├── module.yml │ │ ├── notification.yml │ │ ├── schedule.yml │ │ ├── state.yml │ │ ├── task.yml │ │ └── user.yml ├── requirements.client.txt ├── requirements.local.txt ├── requirements.server.txt ├── scripts │ ├── celery-flower.sh │ ├── cli.sh │ ├── client.sh │ ├── command.sh │ ├── config │ │ ├── api.sh │ │ ├── controller.sh │ │ ├── scheduler.sh │ │ └── worker.sh │ ├── controller.sh │ ├── data.sh │ ├── gateway.sh │ ├── install.sh │ ├── scheduler.sh │ ├── wait.sh │ └── worker.sh ├── services │ ├── celery.py │ ├── cli │ │ └── settings.py │ ├── command │ │ ├── settings.py │ │ └── urls.py │ ├── controller │ │ └── settings.py │ ├── data │ │ ├── settings.py │ │ └── urls.py │ ├── tasks │ │ └── settings.py │ └── wsgi.py ├── settings │ ├── __init__.py │ ├── app.py │ ├── client.py │ ├── config.py │ ├── core.py │ ├── full.py │ ├── install.py │ ├── roles.py │ └── tasks.py ├── spec │ ├── base │ │ ├── command.yml │ │ └── data.yml │ ├── channels.yml │ ├── commands │ │ ├── archiver.yml │ │ ├── cache.yml │ │ ├── calculate.yml │ │ ├── config.yml │ │ ├── controller.yml │ │ ├── database.yml │ │ ├── dataset.yml │ │ ├── gpu.yml │ │ ├── group.yml │ │ ├── host.yml │ │ ├── import.yml │ │ ├── log.yml │ │ ├── message.yml │ │ ├── module.yml │ │ ├── notification.yml │ │ ├── platform.yml │ │ ├── scaling.yml │ │ ├── schedule.yml │ │ ├── service.yml │ │ ├── state.yml │ │ ├── template.yml │ │ └── user.yml │ ├── data │ │ ├── cache.yml │ │ ├── config.yml │ │ ├── dataset.yml │ │ ├── group.yml │ │ ├── host.yml │ │ ├── log.yml │ │ ├── module.yml │ │ ├── notification.yml │ │ ├── scaling.yml │ │ ├── schedule.yml │ │ ├── state.yml │ │ └── user.yml │ ├── encryption.yml │ ├── mixins │ │ ├── command.yml │ │ ├── data.yml │ │ └── plugin.yml │ ├── plugins │ │ ├── calculation.yml │ │ ├── config.yml │ │ ├── data_processor.yml │ │ ├── dataset.yml │ │ ├── encryption.yml │ │ ├── field_processor.yml │ │ ├── formatter.yml │ │ ├── function.yml │ │ ├── group.yml │ │ ├── module.yml │ │ ├── parser.yml │ │ ├── source.yml │ │ ├── task.yml │ │ ├── user.yml │ │ ├── validator.yml │ │ └── worker.yml │ ├── roles.yml │ ├── services.yml │ └── workers.yml ├── systems │ ├── api │ │ ├── auth.py │ │ ├── command │ │ │ ├── auth.py │ │ │ ├── codecs.py │ │ │ ├── renderers.py │ │ │ ├── response.py │ │ │ ├── routers.py │ │ │ ├── schema.py │ │ │ └── views.py │ │ ├── data │ │ │ ├── auth.py │ │ │ ├── fields.py │ │ │ ├── filter │ │ │ │ ├── backends.py │ │ │ │ ├── filters.py │ │ │ │ └── related.py │ │ │ ├── filters.py │ │ │ ├── pagination.py │ │ │ ├── parsers.py │ │ │ ├── renderers.py │ │ │ ├── response.py │ │ │ ├── routers.py │ │ │ ├── schema.py │ │ │ ├── serializers.py │ │ │ └── views.py │ │ ├── encoders.py │ │ ├── renderers.py │ │ ├── response.py │ │ └── views.py │ ├── cache │ │ └── middleware.py │ ├── celery │ │ ├── app.py │ │ ├── registry.py │ │ ├── scheduler.py │ │ ├── task.py │ │ └── worker.py │ ├── client │ │ └── cli │ │ │ ├── args.py │ │ │ ├── client.py │ │ │ ├── commands │ │ │ ├── action.py │ │ │ ├── base.py │ │ │ ├── help.py │ │ │ ├── router.py │ │ │ ├── test.py │ │ │ └── version.py │ │ │ ├── errors.py │ │ │ └── index.py │ ├── commands │ │ ├── action.py │ │ ├── agent.py │ │ ├── args.py │ │ ├── base.py │ │ ├── calculator.py │ │ ├── cli.py │ │ ├── exec.py │ │ ├── factory │ │ │ ├── helpers.py │ │ │ ├── operations │ │ │ │ ├── clear.py │ │ │ │ ├── get.py │ │ │ │ ├── list.py │ │ │ │ ├── remove.py │ │ │ │ └── save.py │ │ │ └── resource.py │ │ ├── help.py │ │ ├── importer.py │ │ ├── index.py │ │ ├── messages.py │ │ ├── mixins │ │ │ ├── base.py │ │ │ ├── exec.py │ │ │ ├── meta.py │ │ │ ├── query.py │ │ │ ├── relations.py │ │ │ └── renderer.py │ │ ├── options.py │ │ ├── processor.py │ │ ├── profile.py │ │ ├── router.py │ │ └── schema.py │ ├── db │ │ ├── backends │ │ │ └── postgresql │ │ │ │ └── base.py │ │ ├── manager.py │ │ └── router.py │ ├── encryption │ │ └── cipher.py │ ├── index │ │ ├── component.py │ │ ├── django.py │ │ └── module.py │ ├── indexer.py │ ├── kubernetes │ │ ├── agent.py │ │ ├── base.py │ │ ├── cluster.py │ │ ├── config.py │ │ └── worker.py │ ├── manage │ │ ├── cluster.py │ │ ├── communication.py │ │ ├── runtime.py │ │ ├── service.py │ │ ├── task.py │ │ └── template.py │ ├── manager.py │ ├── models │ │ ├── aggregates.py │ │ ├── base.py │ │ ├── dataset.py │ │ ├── errors.py │ │ ├── facade.py │ │ ├── fields.py │ │ ├── index.py │ │ ├── mixins │ │ │ ├── annotations.py │ │ │ ├── fields.py │ │ │ ├── filters.py │ │ │ ├── query.py │ │ │ ├── relations.py │ │ │ ├── render.py │ │ │ └── update.py │ │ ├── overrides.py │ │ └── parsers │ │ │ ├── base.py │ │ │ ├── data_processors.py │ │ │ ├── field_processors.py │ │ │ ├── fields.py │ │ │ ├── filters.py │ │ │ ├── function.py │ │ │ └── order.py │ └── plugins │ │ ├── base.py │ │ ├── index.py │ │ └── parser.py ├── tasks │ ├── utility.yml │ └── zimagi.yml ├── templates │ ├── data │ │ └── model │ │ │ ├── command.yml │ │ │ ├── data.yml │ │ │ ├── index.yml │ │ │ └── plugin.yml │ ├── field │ │ ├── big_integer │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── binary │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── boolean │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── date │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── datetime │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── dict │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── duration │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── float │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── foreign_key │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── integer │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── list │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── many_to_many │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── string │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ ├── text │ │ │ ├── index.yml │ │ │ └── spec.yml │ │ └── url │ │ │ ├── index.yml │ │ │ └── spec.yml │ ├── functions │ │ ├── core_class.py │ │ ├── core_list.py │ │ └── core_text.py │ ├── module │ │ └── standard │ │ │ ├── .gitignore │ │ │ ├── LICENSE │ │ │ ├── django.py │ │ │ ├── index.yml │ │ │ ├── install.sh │ │ │ ├── requirements.txt │ │ │ └── zimagi.yml │ └── user │ │ └── role │ │ ├── index.yml │ │ └── spec.yml ├── tests │ ├── base.py │ ├── command │ │ └── base.py │ ├── command_local.py │ ├── data │ │ ├── config.yml │ │ ├── group.yml │ │ └── host.yml │ ├── mixins │ │ └── assertions.py │ └── sdk_python │ │ ├── base.py │ │ ├── data │ │ ├── base.py │ │ └── test_group.py │ │ ├── init │ │ ├── test_create.py │ │ ├── test_schedule.py │ │ ├── test_schema.py │ │ └── test_status.py │ │ └── runner.py ├── utility │ ├── data.py │ ├── dataframe.py │ ├── display.py │ ├── filesystem.py │ ├── git.py │ ├── mutex.py │ ├── nvidia.py │ ├── parallel.py │ ├── project.py │ ├── python.py │ ├── query.py │ ├── request.py │ ├── runtime.py │ ├── shell.py │ ├── ssh.py │ ├── temp.py │ ├── terminal.py │ ├── text.py │ └── time.py ├── zimagi-cli.py ├── zimagi-client.py └── zimagi-install.py ├── clean ├── compose.db.yaml ├── compose.network.yaml ├── compose.nvidia.local.yaml ├── compose.nvidia.test.yaml ├── compose.standard.local.yaml ├── compose.standard.test.yaml ├── docker ├── Dockerfile.cli ├── Dockerfile.server ├── build_client_image.sh ├── deploy_cli_image.sh ├── deploy_cli_manifest.sh ├── deploy_server_image.sh ├── deploy_server_manifest.sh ├── packages.core.txt └── packages.server.txt ├── docs ├── Makefile ├── _static │ ├── css │ │ └── override.css │ ├── favicon.ico │ ├── images │ │ ├── zimagi-architecture.png │ │ ├── zimagi-components.png │ │ ├── zimagi-flow.png │ │ └── zimagi-logo.png │ └── logo.png ├── _templates │ ├── breadcrumbs.html │ └── layout.html ├── archive │ ├── architecture.rst │ ├── prerequisites_and_environment_setup │ │ └── prereqs.rst │ ├── reference_and_core_resources │ │ └── core_resources.rst │ ├── tutorials │ │ └── tutorials.rst │ └── use_cases │ │ └── use_cases.rst ├── concepts │ ├── components.rst │ ├── design_architecture │ │ ├── commands │ │ │ ├── data.rst │ │ │ ├── environment.rst │ │ │ ├── help.rst │ │ │ ├── orchestration.rst │ │ │ ├── readme.rst │ │ │ └── service.rst │ │ ├── concepts.rst │ │ ├── data │ │ │ ├── configuration.rst │ │ │ ├── environment.rst │ │ │ ├── group.rst │ │ │ ├── host.rst │ │ │ ├── log.rst │ │ │ ├── module.rst │ │ │ ├── notification.rst │ │ │ ├── readme.rst │ │ │ ├── schedule.rst │ │ │ ├── state.rst │ │ │ └── user.rst │ │ ├── extending.rst │ │ ├── interface.rst │ │ ├── orchestration │ │ │ ├── config.rst │ │ │ ├── destroy.rst │ │ │ ├── group.rst │ │ │ ├── profile.rst │ │ │ ├── readme.rst │ │ │ └── run.rst │ │ ├── plugins │ │ │ ├── module.rst │ │ │ ├── readme.rst │ │ │ └── task.rst │ │ ├── readme.rst │ │ ├── services │ │ │ ├── cli.rst │ │ │ ├── command.rst │ │ │ ├── data.rst │ │ │ ├── readme.rst │ │ │ ├── scheduler.rst │ │ │ └── worker.rst │ │ ├── systems.rst │ │ └── technologies.rst │ └── specs │ │ ├── kix.6og5zbx0ctt.png │ │ ├── kix.7ft3utqvnd8j.png │ │ ├── kix.fws49h5c5sbl.png │ │ ├── kix.tpls2u9qapt1.png │ │ ├── kix.u09pexpvp3a8.png │ │ ├── kix.vsnbgztpsc6x.png │ │ └── kix.yk9yetlaxfoj.png ├── conf.py ├── creating_a_module │ ├── command_line_comparison.rst │ ├── commands.rst │ ├── create_module.rst │ ├── data_import.rst │ ├── data_model.rst │ ├── index.rst │ ├── initializing.rst │ └── module_functionality.rst ├── deploy.sh ├── getting_started │ ├── commands.rst │ ├── readme.rst │ ├── use_module.rst │ └── vagrant.rst ├── outline.md ├── overview │ ├── about.rst │ ├── contributing.rst │ └── why.rst ├── readme.rst └── requirements.txt ├── env ├── public.api ├── public.api.encrypted ├── public.default └── secret.example ├── package-lock.json ├── package.json ├── package ├── README.md ├── bin │ └── zimagi ├── deploy.sh ├── requirements.txt ├── setup.py └── zimagi │ ├── __init__.py │ ├── auth.py │ ├── client.py │ ├── codecs.py │ ├── collection.py │ ├── command │ ├── __init__.py │ ├── client.py │ ├── codecs.py │ ├── messages.py │ ├── response.py │ ├── schema.py │ └── transports.py │ ├── data │ ├── __init__.py │ ├── client.py │ ├── codecs.py │ └── transports.py │ ├── datetime.py │ ├── encryption.py │ ├── exceptions.py │ ├── parallel.py │ ├── settings.py │ ├── transports.py │ └── utility.py ├── pyproject.toml ├── reactor ├── build │ ├── client.sh │ ├── server.nvidia.sh │ ├── server.sh │ └── shared.sh ├── commands │ └── zimagi.sh ├── hooks.sh ├── initialize.sh ├── requirements.txt └── utilities │ └── env.sh ├── setup.cfg ├── start └── zimagi /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.{py,rst,ini}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{html,css,scss,json,yml,xml}] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [Makefile] 23 | indent_style = tab 24 | 25 | [default.conf] 26 | indent_style = space 27 | indent_size = 2 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac files 2 | .DS_Store 3 | 4 | # Eclipse project files 5 | /.project 6 | /.pydevproject 7 | /.buildpath 8 | /.settings 9 | 10 | # VSCode files 11 | /.vscode 12 | *.code-workspace 13 | 14 | # Python files 15 | __pycache__ 16 | 17 | # Package files 18 | /package/build/ 19 | /package/dist/ 20 | /package/*.egg-info 21 | /package/VERSION 22 | 23 | # Application files 24 | /.env 25 | /env/generated 26 | /env/secret 27 | /data/ 28 | /lib/ 29 | /docs/_build/ 30 | 31 | # Formatting related files 32 | /node_modules/ 33 | 34 | # External libraries 35 | /bin/ 36 | /charts/ 37 | 38 | # Extra files 39 | /tmp/ 40 | *.log 41 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | app/templates 2 | docs 3 | data 4 | env 5 | lib 6 | node_modules 7 | .circleci 8 | -------------------------------------------------------------------------------- /.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | /** @type {import('prettier').Config} */ 2 | module.exports = { 3 | printWidth: 120, 4 | semi: true, 5 | singleQuote: true, 6 | tabWidth: 2, 7 | trailingComma: 'es5', 8 | useTabs: false, 9 | 10 | plugins: [], 11 | overrides: [], 12 | }; 13 | -------------------------------------------------------------------------------- /app/VERSION: -------------------------------------------------------------------------------- 1 | 0.14.3 2 | -------------------------------------------------------------------------------- /app/commands/agent/archiver.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Agent 2 | from utility.data import Collection 3 | 4 | 5 | class Archiver(Agent("archiver")): 6 | processes = ("record_scaling_event",) 7 | 8 | def record_scaling_event(self): 9 | for package in self.listen("worker:scaling", state_key="core_archiver"): 10 | message = Collection(**package.message) 11 | self.save_instance( 12 | self._scaling_event, 13 | None, 14 | fields={ 15 | "created": package.time, 16 | "command": message.command, 17 | "worker_type": message.worker_type, 18 | "worker_max_count": message.worker_max_count, 19 | "worker_count": message.worker_count, 20 | "task_count": message.task_count, 21 | "workers_created": message.workers_created, 22 | }, 23 | ) 24 | -------------------------------------------------------------------------------- /app/commands/agent/controller.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from services.celery import app 3 | from systems.commands.index import Agent 4 | 5 | 6 | class Controller(Agent("controller")): 7 | def exec(self): 8 | self.manager.reset_spec() 9 | 10 | for agent in self.manager.collect_agents(): 11 | worker = self.get_provider( 12 | "worker", 13 | settings.WORKER_PROVIDER, 14 | app, 15 | worker_type=agent.spec.get("worker_type", "default"), 16 | command_name=" ".join(agent.command), 17 | command_options=agent.spec.get("options", {}), 18 | ) 19 | if self._check_agent_schedule(agent.spec): 20 | config_name = self.manager._get_agent_scale_config(agent.command) 21 | agent_count = self.get_config(config_name, 0) 22 | worker.scale_agents(agent_count) 23 | else: 24 | worker.scale_agents(0) 25 | -------------------------------------------------------------------------------- /app/commands/cache/clear.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.cache import caches 3 | from systems.commands.index import Command 4 | 5 | 6 | class Clear(Command("cache.clear")): 7 | def exec(self): 8 | caches[settings.CACHE_MIDDLEWARE_ALIAS].clear() 9 | caches[settings.CACHE_MIDDLEWARE_ALIAS].close() 10 | -------------------------------------------------------------------------------- /app/commands/calculate.py: -------------------------------------------------------------------------------- 1 | from systems.commands.calculator import Calculator 2 | from systems.commands.index import Command 3 | 4 | 5 | class Calculate(Command("calculate")): 6 | def exec(self): 7 | Calculator(self, display_only=self.show_spec, reset=self.reset).run( 8 | required_names=self.calculation_names, 9 | required_tags=self.tags, 10 | ignore_requirements=self.ignore_requirements, 11 | field_values=self.field_values, 12 | ) 13 | -------------------------------------------------------------------------------- /app/commands/db/backup.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Backup(Command("db.backup")): 5 | def exec(self): 6 | self.create_snapshot() 7 | -------------------------------------------------------------------------------- /app/commands/db/clean.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Clean(Command("db.clean")): 5 | def exec(self): 6 | self.clean_snapshots(self.keep_num) 7 | -------------------------------------------------------------------------------- /app/commands/db/restore.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Restore(Command("db.restore")): 5 | def exec(self): 6 | self.restore_snapshot(self.snapshot_name) 7 | -------------------------------------------------------------------------------- /app/commands/db/snapshots.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Snapshots(Command("db.snapshots")): 5 | def exec(self): 6 | self.info(" Zimagi Snapshots") 7 | self.info("") 8 | 9 | for snapshot_file in self.get_snapshots(): 10 | self.info(f" * {self.key_color(snapshot_file)}") 11 | self.info("") 12 | -------------------------------------------------------------------------------- /app/commands/destroy.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Destroy(Command("destroy")): 5 | def exec(self): 6 | self.module.provider.destroy_profile( 7 | self.profile_key, 8 | config=self.profile_config_fields, 9 | components=self.profile_components, 10 | display_only=self.display_only, 11 | ignore_missing=self.ignore_missing, 12 | ) 13 | -------------------------------------------------------------------------------- /app/commands/gpu.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | from utility.nvidia import Nvidia 3 | 4 | 5 | class Gpu(Command("gpu")): 6 | def exec(self): 7 | nvidia = Nvidia(self) 8 | 9 | self.sh("nvidia-smi") 10 | 11 | for device_index in range(nvidia.device_count): 12 | device = nvidia.get_device_info(device_index) 13 | device_info = device.export() 14 | device_info.pop("name") 15 | 16 | self.data(device.name, device_info, "gpu_info") 17 | -------------------------------------------------------------------------------- /app/commands/group/children.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Children(Command("group.children")): 5 | def exec(self): 6 | self.exec_local( 7 | "group save", {"group_key": self.group_key, "group_provider_name": self.group_provider_name, "verbosity": 0} 8 | ) 9 | parent = self._group.retrieve(self.group_key) 10 | for group in self.group_child_keys: 11 | self._group.store(group, {"provider_type": parent.provider_type, "parent": parent}, command=self) 12 | self.success(f"Successfully saved group {parent.name}") 13 | -------------------------------------------------------------------------------- /app/commands/import.py: -------------------------------------------------------------------------------- 1 | from systems.commands.importer import Importer 2 | from systems.commands.index import Command 3 | 4 | 5 | class Import(Command("import")): 6 | def exec(self): 7 | Importer(self, display_only=self.show_spec).run( 8 | required_names=self.import_names, 9 | required_tags=self.tags, 10 | ignore_requirements=self.ignore_requirements, 11 | field_values=self.field_values, 12 | ) 13 | -------------------------------------------------------------------------------- /app/commands/info.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Info(Command("info")): 5 | def exec(self): 6 | self.notice("Platform Information") 7 | self.table( 8 | [ 9 | [self.key_color("Version"), self.value_color(self.get_version())], 10 | ], 11 | "platform_info", 12 | ) 13 | self.info("") 14 | 15 | if self._host.count(): 16 | self.notice("Remote Hosts") 17 | self.table(self.render_list(self._host), "hosts") 18 | self.info("") 19 | 20 | self.notice("Installed Modules") 21 | self.table(self.render_list(self._module), "modules") 22 | self.info("") 23 | -------------------------------------------------------------------------------- /app/commands/log/abort.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Abort(Command("log.abort")): 5 | def exec(self): 6 | def abort_command(log_key): 7 | self.publish_abort(log_key) 8 | self.wait_for_tasks(log_key) 9 | self.success(f"Task {log_key} successfully aborted") 10 | 11 | self.run_list(self.log_keys, abort_command) 12 | -------------------------------------------------------------------------------- /app/commands/log/clean.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Clean(Command("log.clean")): 5 | def exec(self): 6 | self.clean_logs(log_days=self.log_days, message_days=self.log_message_days) 7 | self.success("Log entries successfully cleaned") 8 | -------------------------------------------------------------------------------- /app/commands/log/rerun.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from systems.commands.index import Command 4 | 5 | 6 | class Rerun(Command("log.rerun")): 7 | def exec(self): 8 | def rerun_command(log_key): 9 | log = self._log.retrieve(log_key) 10 | if log: 11 | options = copy.deepcopy(log.config) 12 | rerun_key = self.exec_local(log.command, options) 13 | self.success(f"Task {log.command}:{log_key} was successfully rerun: {rerun_key}") 14 | else: 15 | self.error(f"Log key {log_key} does not exist") 16 | 17 | self.run_list(self.log_keys, rerun_command) 18 | -------------------------------------------------------------------------------- /app/commands/mixins/config.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import CommandMixin 2 | 3 | 4 | class ConfigMixin(CommandMixin("config")): 5 | def get_config(self, name, default=None, required=False): 6 | if not name: 7 | return default 8 | 9 | config = self.get_instance(self._config, name, required=required, cache=False) 10 | if config is None: 11 | return default 12 | 13 | return config.value 14 | -------------------------------------------------------------------------------- /app/commands/mixins/message.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from systems.commands.index import CommandMixin 4 | from utility.data import clean_dict 5 | 6 | 7 | class MessageMixin(CommandMixin("message")): 8 | def check_channel_permission(self): 9 | communication_spec = copy.deepcopy(self.manager.get_spec("channels", {})) 10 | 11 | if self.communication_channel not in communication_spec: 12 | self.error(f"Communication channel {self.communication_channel} does not exist") 13 | 14 | communication_spec = communication_spec[self.communication_channel] 15 | if not communication_spec: 16 | communication_spec = {} 17 | 18 | roles = list(clean_dict(communication_spec, False).keys()) 19 | if not roles: 20 | return True 21 | 22 | return self.active_user.groups.filter(name__in=roles).exists() 23 | -------------------------------------------------------------------------------- /app/commands/module/add.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Add(Command("module.add")): 5 | def parse(self): 6 | super().parse() 7 | self.parse_scope(self._module) 8 | self.parse_relations(self._module) 9 | 10 | self.parse_module_fields( 11 | True, help_callback=self.get_provider("module", "help").field_help, exclude_fields=["remote"] 12 | ) 13 | 14 | def exec(self): 15 | self.set_scope(self._module) 16 | self.module_fields["remote"] = self.remote 17 | self.module_provider.create(None, {**self.module_fields, **self.get_relations(self._module)}) 18 | -------------------------------------------------------------------------------- /app/commands/module/create.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Create(Command("module.create")): 5 | def parse(self): 6 | super().parse() 7 | self.parse_scope(self._module) 8 | self.parse_relations(self._module) 9 | 10 | def exec(self): 11 | self.set_scope(self._module) 12 | self.module_provider.create( 13 | self.module_key, 14 | { 15 | "template_package": self.module_template, 16 | "template_fields": self.template_fields, 17 | **self.get_relations(self._module), 18 | }, 19 | ) 20 | -------------------------------------------------------------------------------- /app/commands/module/init.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Init(Command("module.init")): 5 | def bootstrap_ensure(self): 6 | return False 7 | 8 | def exec(self): 9 | self.ensure_resources(reinit=True, data_types=self.data_types, force=self.force) 10 | -------------------------------------------------------------------------------- /app/commands/module/install.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Install(Command("module.install")): 5 | def exec(self): 6 | self.info("Installing module requirements...") 7 | self.manager.install_scripts(self, self.verbosity == 3) 8 | self.manager.install_requirements(self, self.verbosity == 3) 9 | self.success("Successfully installed module requirements") 10 | -------------------------------------------------------------------------------- /app/commands/notification/clear.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Clear(Command("notification.clear")): 5 | def exec(self): 6 | self._notification.clear() 7 | self.success("Successfully cleared all command notification preferences") 8 | -------------------------------------------------------------------------------- /app/commands/notification/remove.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Remove(Command("notification.remove")): 5 | def exec(self): 6 | command = self.notify_command 7 | instance, created = self._notification.store(command) 8 | 9 | for group in self.notify_groups: 10 | if self.notify_failure: 11 | instance.failure_groups.filter(group=group).delete() 12 | self.success(f"Group {group.name} unsubscribed from {command} failure notifications") 13 | else: 14 | instance.groups.filter(group=group).delete() 15 | self.success(f"Group {group.name} unsubscribed from {command} notifications") 16 | -------------------------------------------------------------------------------- /app/commands/notification/save.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Save(Command("notification.save")): 5 | def exec(self): 6 | command = self.notify_command 7 | instance, created = self._notification.store(command) 8 | 9 | for group in self.notify_groups: 10 | if self.notify_failure: 11 | instance.failure_groups.get_or_create(group=group) 12 | self.success(f"Group {group.name} subscribed to {command} failure notifications") 13 | else: 14 | instance.groups.get_or_create(group=group) 15 | self.success(f"Group {group.name} subscribed to {command} notifications") 16 | -------------------------------------------------------------------------------- /app/commands/run.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Run(Command("run")): 5 | def exec(self): 6 | self.module.provider.run_profile( 7 | self.profile_key, 8 | config=self.profile_config_fields, 9 | components=self.profile_components, 10 | display_only=self.display_only, 11 | test=self.test, 12 | ignore_missing=self.ignore_missing, 13 | ) 14 | -------------------------------------------------------------------------------- /app/commands/send.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | from systems.manage.communication import channel_communication_key 3 | from utility.data import dump_json, normalize_value 4 | from utility.time import Time 5 | 6 | 7 | class Send(Command("send")): 8 | def exec(self): 9 | if not self.check_channel_permission(): 10 | self.error(f"You do not have permission to access the {self.communication_channel} channel") 11 | 12 | connection = self.manager.communication_connection() 13 | if connection: 14 | data = { 15 | "user": self.active_user.name, 16 | "time": Time().now_string, 17 | "message": normalize_value(self.communication_message, parse_json=True), 18 | } 19 | connection.publish(channel_communication_key(self.communication_channel), dump_json(data, indent=2)) 20 | self.success(f"Message sent to channel {self.communication_channel}: {self.communication_message}") 21 | -------------------------------------------------------------------------------- /app/commands/service/follow.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Follow(Command("service.follow")): 5 | def exec(self): 6 | for package in self.listen(self.channel, state_key=self.state_key, timeout=self.timeout, block_sec=1): 7 | self.info("=" * self.display_width) 8 | self.data("Time", package.time, "time") 9 | self.data("Sender", package.sender, "sender") 10 | self.data("Message", package.message, "message") 11 | -------------------------------------------------------------------------------- /app/commands/service/lock/clear.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | from utility.mutex import Mutex 3 | 4 | 5 | class Clear(Command("service.lock.clear")): 6 | def exec(self): 7 | Mutex.clear(*self.keys) 8 | -------------------------------------------------------------------------------- /app/commands/service/lock/set.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | from utility.mutex import Mutex 3 | 4 | 5 | class Set(Command("service.lock.set")): 6 | def exec(self): 7 | Mutex.set(self.key, self.expires if self.expires else None) 8 | -------------------------------------------------------------------------------- /app/commands/service/lock/wait.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | from utility.mutex import Mutex 3 | 4 | 5 | class Wait(Command("service.lock.wait")): 6 | def exec(self): 7 | success = Mutex.wait(*self.keys, timeout=self.timeout, interval=self.interval) 8 | if self.raise_error and not success: 9 | self.error(f"Lock timeout waiting for keys: {", ".join(self.keys)}") 10 | -------------------------------------------------------------------------------- /app/commands/task.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Task(Command("task")): 5 | def exec(self): 6 | self.module.provider.exec_task(self.task_key, self.task_fields) 7 | -------------------------------------------------------------------------------- /app/commands/template/generate.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Generate(Command("template.generate")): 5 | def exec(self): 6 | self.provision_template(self.module, self.module_template, self.template_fields, display_only=self.display_only) 7 | -------------------------------------------------------------------------------- /app/commands/user/rotate.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Rotate(Command("user.rotate")): 5 | def exec(self): 6 | user = self.user if self.user_key else self.active_user 7 | token = self._user.generate_token() 8 | 9 | user.set_password(token) 10 | user.save() 11 | 12 | self.silent_data("name", user.name) 13 | self.data(f"User {user.name} token:", token, "token") 14 | -------------------------------------------------------------------------------- /app/commands/version.py: -------------------------------------------------------------------------------- 1 | from systems.commands.index import Command 2 | 3 | 4 | class Version(Command("version")): 5 | def exec(self): 6 | self.table([["Server version", self.get_version()]], "version_info") 7 | -------------------------------------------------------------------------------- /app/components/config_store.py: -------------------------------------------------------------------------------- 1 | from systems.commands import profile 2 | 3 | 4 | class ProfileComponent(profile.BaseProfileComponent): 5 | def priority(self): 6 | return 0 7 | 8 | def run(self, name, value): 9 | self.exec( 10 | "config save", 11 | config_key=name, 12 | config_value_type=type(value).__name__, 13 | config_value=value, 14 | local=self.command.local, 15 | ) 16 | self.profile.config.set(name, value) 17 | 18 | def destroy(self, name, value): 19 | self.exec("config remove", config_key=name, force=True, local=self.command.local) 20 | -------------------------------------------------------------------------------- /app/components/post_destroy.py: -------------------------------------------------------------------------------- 1 | from .destroy import ProfileComponent as BaseProfileComponent 2 | 3 | 4 | class ProfileComponent(BaseProfileComponent): 5 | def priority(self): 6 | return 100 7 | -------------------------------------------------------------------------------- /app/components/post_run.py: -------------------------------------------------------------------------------- 1 | from .run import ProfileComponent as BaseProfileComponent 2 | 3 | 4 | class ProfileComponent(BaseProfileComponent): 5 | def priority(self): 6 | return 100 7 | -------------------------------------------------------------------------------- /app/components/pre_destroy.py: -------------------------------------------------------------------------------- 1 | from .destroy import ProfileComponent as BaseProfileComponent 2 | 3 | 4 | class ProfileComponent(BaseProfileComponent): 5 | def priority(self): 6 | return 0 7 | -------------------------------------------------------------------------------- /app/components/pre_run.py: -------------------------------------------------------------------------------- 1 | from .run import ProfileComponent as BaseProfileComponent 2 | 3 | 4 | class ProfileComponent(BaseProfileComponent): 5 | def priority(self): 6 | return 0 7 | -------------------------------------------------------------------------------- /app/components/roles.py: -------------------------------------------------------------------------------- 1 | from systems.commands import profile 2 | 3 | 4 | class ProfileComponent(profile.BaseProfileComponent): 5 | def priority(self): 6 | return 5 7 | 8 | def run(self, name, help): 9 | if not help: 10 | self.command.error(f"Role {name} help string must be provided") 11 | 12 | self.exec( 13 | "template generate", 14 | module_key=self.profile.module.instance.name, 15 | module_template="user/role", 16 | template_fields={"name": name.replace("_", "-"), "help": help.capitalize()}, 17 | local=True, 18 | ) 19 | -------------------------------------------------------------------------------- /app/data/base/name_resource.py: -------------------------------------------------------------------------------- 1 | from systems.models.index import BaseModel 2 | 3 | 4 | class NameResourceBase(BaseModel("name_resource")): 5 | def save(self, *args, **kwargs): 6 | self._prepare_save() 7 | super().save(*args, **kwargs) 8 | 9 | def get_id(self): 10 | return getattr(self, self.facade.pk) 11 | -------------------------------------------------------------------------------- /app/data/cache/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/cache/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/cache/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/cache/models.py -------------------------------------------------------------------------------- /app/data/config/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/config/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/config/models.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from systems.models.index import Model, ModelFacade 3 | from utility.data import format_value 4 | 5 | 6 | class ConfigFacade(ModelFacade("config")): 7 | def ensure(self, command, reinit, force): 8 | if settings.CLI_EXEC or settings.SCHEDULER_INIT: 9 | terminal_width = command.display_width 10 | 11 | if not reinit: 12 | command.notice("\n".join(["Loading Zimagi system configurations", "-" * terminal_width])) 13 | command.notice("-" * terminal_width) 14 | 15 | 16 | class Config(Model("config")): 17 | def save(self, *args, **kwargs): 18 | self.value = format_value(self.value_type, self.value) 19 | super().save(*args, **kwargs) 20 | -------------------------------------------------------------------------------- /app/data/dataset/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/dataset/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/dataset/models.py: -------------------------------------------------------------------------------- 1 | from systems.models.index import ModelFacade 2 | 3 | 4 | class DataSetFacade(ModelFacade("dataset")): 5 | def get_field_data_display(self, instance, value, short): 6 | data = instance.provider.load() 7 | return "\n" + self.notice_color(data) + "\n" 8 | -------------------------------------------------------------------------------- /app/data/group/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/group/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/host/migrations/0002_alter_host_token_alter_host_user.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.13 on 2025-05-15 11:21 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("host", "0001_initial"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="host", 15 | name="token", 16 | field=models.CharField(max_length=256, null=True), 17 | ), 18 | migrations.AlterField( 19 | model_name="host", 20 | name="user", 21 | field=models.CharField(max_length=150, null=True), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /app/data/host/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/host/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/log/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/log/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/mixins/provider.py: -------------------------------------------------------------------------------- 1 | from systems.models.index import ModelMixin, ModelMixinFacade 2 | 3 | 4 | class ProviderMixinFacade(ModelMixinFacade("provider")): 5 | @property 6 | def provider_name(self): 7 | if getattr(self.meta, "provider_name", None): 8 | return self.meta.provider_name 9 | return None 10 | 11 | 12 | class ProviderMixin(ModelMixin("provider")): 13 | def initialize(self, command, facade=None, **options): 14 | if not super().initialize(command, **options): 15 | return False 16 | 17 | provider_name = self.facade.provider_name 18 | if provider_name: 19 | self.provider = command.get_provider(provider_name, self.provider_type, instance=self, facade=facade) 20 | return True 21 | -------------------------------------------------------------------------------- /app/data/mixins/resource.py: -------------------------------------------------------------------------------- 1 | from django.utils.timezone import now 2 | from systems.models.index import ModelMixin 3 | 4 | 5 | class ResourceMixin(ModelMixin("resource")): 6 | def _set_created_time(self): 7 | if self.created is None: 8 | self.created = now() 9 | 10 | def _prepare_save(self): 11 | self._set_created_time() 12 | 13 | filters = {} 14 | self.facade._check_scope(filters) 15 | for field, value in filters.items(): 16 | if value is not None: 17 | setattr(self, field, value) 18 | -------------------------------------------------------------------------------- /app/data/module/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/module/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/notification/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/notification/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/notification/models.py: -------------------------------------------------------------------------------- 1 | from systems.models.index import ModelFacade 2 | 3 | 4 | class NotificationFacade(ModelFacade("notification")): 5 | def get_field_group_names_display(self, instance, value, short): 6 | display = [] 7 | for record in instance.groups.all(): 8 | display.append(record.group.name) 9 | return super().get_field_group_names_display(instance, "\n".join(display) + "\n", short) 10 | 11 | def get_field_failure_group_names_display(self, instance, value, short): 12 | display = [] 13 | for record in instance.failure_groups.all(): 14 | display.append(record.group.name) 15 | return super().get_field_failure_group_names_display(instance, "\n".join(display) + "\n", short) 16 | -------------------------------------------------------------------------------- /app/data/scaling_event/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/scaling_event/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/scaling_event/models.py: -------------------------------------------------------------------------------- 1 | from django.utils.timezone import now 2 | from systems.models.index import Model, ModelFacade 3 | from utility.data import create_token 4 | 5 | 6 | class ScalingEventFacade(ModelFacade("scaling_event")): 7 | pass 8 | 9 | 10 | class ScalingEvent(Model("scaling_event")): 11 | 12 | def save(self, *args, **kwargs): 13 | if not self.name: 14 | self.name = f"{now().strftime("%Y%m%d%H%M%S")}{create_token(5)}x" 15 | super().save(*args, **kwargs) 16 | -------------------------------------------------------------------------------- /app/data/schedule/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/schedule/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/state/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/state/migrations/__init__.py -------------------------------------------------------------------------------- /app/data/state/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/state/models.py -------------------------------------------------------------------------------- /app/data/user/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/data/user/migrations/__init__.py -------------------------------------------------------------------------------- /app/help/en/cache.yml: -------------------------------------------------------------------------------- 1 | cache: 2 | overview: | 3 | Manage zimagi cache 4 | 5 | description: | 6 | TODO 7 | 8 | clear: 9 | overview: | 10 | Clear the Zimagi page cache 11 | 12 | description: | 13 | TODO 14 | -------------------------------------------------------------------------------- /app/help/en/calculate.yml: -------------------------------------------------------------------------------- 1 | calculate: 2 | overview: | 3 | Calculate data objects from specifications 4 | 5 | description: | 6 | TODO 7 | -------------------------------------------------------------------------------- /app/help/en/db.yml: -------------------------------------------------------------------------------- 1 | backup: 2 | overview: | 3 | Dump current Zimagi environment to saved snapshot 4 | 5 | description: | 6 | TODO 7 | 8 | restore: 9 | overview: | 10 | Restore Zimagi environment from saved snapshot 11 | 12 | description: | 13 | TODO 14 | 15 | db: 16 | overview: | 17 | Manage Zimagi databases 18 | 19 | description: | 20 | Manage Zimagi databases 21 | 22 | Default database is SQLite, but PostgreSQL can be started in container 23 | 24 | pull: 25 | overview: | 26 | Download and install remote environment database on client 27 | 28 | description: | 29 | TODO 30 | 31 | push: 32 | overview: | 33 | Transfer and install local environment database on server 34 | 35 | description: | 36 | TODO 37 | -------------------------------------------------------------------------------- /app/help/en/group.yml: -------------------------------------------------------------------------------- 1 | group: 2 | overview: | 3 | Manage environment groups 4 | 5 | description: | 6 | TODO 7 | 8 | list: 9 | overview: | 10 | List environment groups 11 | 12 | description: | 13 | TODO 14 | 15 | get: 16 | overview: | 17 | get environment group information 18 | 19 | description: | 20 | TODO 21 | 22 | save: 23 | overview: | 24 | save an environment group 25 | 26 | description: | 27 | TODO 28 | 29 | children: 30 | overview: | 31 | add children to a new or existing environment group 32 | 33 | description: | 34 | TODO 35 | 36 | remove: 37 | overview: | 38 | remove environment group 39 | 40 | description: | 41 | TODO 42 | 43 | clear: 44 | overview: | 45 | clear environment groups 46 | 47 | description: | 48 | TODO 49 | -------------------------------------------------------------------------------- /app/help/en/import.yml: -------------------------------------------------------------------------------- 1 | import: 2 | overview: | 3 | Import data objects from specifications 4 | 5 | description: | 6 | Makes all data objects defined by provided specifications available. 7 | Requires ``import_names`` - The names of one or more specifications. 8 | -------------------------------------------------------------------------------- /app/help/en/log.yml: -------------------------------------------------------------------------------- 1 | log: 2 | overview: | 3 | Manage command log entries 4 | 5 | description: | 6 | TODO 7 | 8 | list: 9 | overview: | 10 | List command log entries 11 | 12 | description: | 13 | TODO 14 | 15 | get: 16 | overview: | 17 | Get information for a command execution 18 | 19 | description: | 20 | TODO 21 | 22 | remove: 23 | overview: | 24 | Remove an existing command log entry 25 | 26 | description: | 27 | TODO 28 | 29 | clear: 30 | overview: | 31 | Clear all existing command log entries 32 | 33 | description: | 34 | TODO 35 | -------------------------------------------------------------------------------- /app/help/en/notification.yml: -------------------------------------------------------------------------------- 1 | notification: 2 | overview: | 3 | Manage command notifications 4 | 5 | description: | 6 | Allows the user to list, save, and delete all notifications for the current active environment. 7 | 8 | list: 9 | overview: | 10 | List command notifications in current environment 11 | 12 | description: | 13 | TODO 14 | 15 | save: 16 | overview: | 17 | Save a command notification in current environment 18 | 19 | description: | 20 | TODO 21 | 22 | remove: 23 | overview: | 24 | Remove an existing command notification 25 | 26 | description: | 27 | TODO 28 | 29 | clear: 30 | overview: | 31 | Clear all existing command notifications 32 | 33 | description: | 34 | TODO 35 | -------------------------------------------------------------------------------- /app/help/en/schedule.yml: -------------------------------------------------------------------------------- 1 | schedule: 2 | overview: | 3 | Manage command schedules 4 | 5 | description: | 6 | Schedule commands let the user set commands to be carried out by workers, remove commands, and see all existing commands. 7 | 8 | list: 9 | overview: | 10 | List command schedules in current environment 11 | 12 | description: | 13 | Takes``field`` ``subfield`` -- a list of fields and subfields to display. 14 | 15 | get: 16 | overview: | 17 | Get environment command schedule 18 | 19 | description: | 20 | Takes ``scheduled_task_name`` - the name of the scheduled tasks. 21 | 22 | remove: 23 | overview: | 24 | Remove an existing command schedule 25 | 26 | description: | 27 | Takes ``scheduled_task_name`` - the name of the scheduled tasks. 28 | 29 | clear: 30 | overview: | 31 | Clear all existing command schedules 32 | 33 | description: | 34 | Deletes all of the existing command schedules within the current active environment. 35 | -------------------------------------------------------------------------------- /app/help/en/state.yml: -------------------------------------------------------------------------------- 1 | state: 2 | overview: | 3 | Manage environment states 4 | 5 | description: | 6 | The State command lets you manage the environment states. You can list all environment states, get a single state, remove a state, or clear all existing states. Takes 'subcommands' - the subcommand to perform on the environment states. 7 | 8 | list: 9 | overview: | 10 | List states in current environment 11 | 12 | description: | 13 | Takes `instance_search_query` - one or more search queries. 14 | 15 | get: 16 | overview: | 17 | Get environment state value 18 | 19 | description: | 20 | Takes `state_name` - the name of a state you want to get the value of. 21 | 22 | remove: 23 | overview: | 24 | Remove an existing environment state 25 | 26 | description: | 27 | Takes `state_name` - the name of a state you want to remove. 28 | 29 | clear: 30 | overview: | 31 | Clear all existing environment states 32 | 33 | description: | 34 | Takes `instance_search_query` - one or more search queries. 35 | -------------------------------------------------------------------------------- /app/plugins/calculation/addition.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("calculation", "addition")): 5 | def calc(self, p): 6 | return (p.b + p.a) if self.check(p.a, p.b) else None 7 | -------------------------------------------------------------------------------- /app/plugins/calculation/cov.py: -------------------------------------------------------------------------------- 1 | from statistics import mean, stdev 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("calculation", "cov")): 7 | def calc(self, p): 8 | data = self.prepare_list(p.a) 9 | return stdev(data) / mean(data) 10 | -------------------------------------------------------------------------------- /app/plugins/calculation/division.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("calculation", "division")): 5 | def calc(self, p): 6 | return (p.a / p.b) if self.check(p.a, p.b) and p.b != 0 else None 7 | -------------------------------------------------------------------------------- /app/plugins/calculation/functions/date.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | 4 | def date(date_str, format="%Y-%m-%d"): 5 | return datetime.strptime(date_str, format) 6 | 7 | 8 | def time(time_str, format="%Y-%m-%dT%H:%M:%S"): 9 | return datetime.strptime(time_str, format) 10 | 11 | 12 | def years(years): 13 | return timedelta(weeks=(years * 52)) 14 | 15 | 16 | def months(months): 17 | return timedelta(weeks=(months * 4.33333333333)) 18 | 19 | 20 | def weeks(weeks): 21 | return timedelta(weeks=weeks) 22 | 23 | 24 | def days(days): 25 | return timedelta(days=days) 26 | 27 | 28 | def hours(hours): 29 | return timedelta(hours=hours) 30 | 31 | 32 | def minutes(min): 33 | return timedelta(minutes=min) 34 | 35 | 36 | def seconds(sec): 37 | return timedelta(seconds=sec) 38 | -------------------------------------------------------------------------------- /app/plugins/calculation/min_max_scale.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("calculation", "min_max_scale")): 5 | def calc(self, p): 6 | values = self.prepare_list(p.a) 7 | low = min(values) 8 | range = max(values) - low 9 | return (values[-1] - low) / range if range != 0 else None 10 | -------------------------------------------------------------------------------- /app/plugins/calculation/multiplication.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("calculation", "multiplication")): 5 | def calc(self, p): 6 | return (p.a * p.b) if self.check(p.a, p.b) else None 7 | -------------------------------------------------------------------------------- /app/plugins/calculation/pchange.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("calculation", "pchange")): 5 | def calc(self, p): 6 | return ((p.a - p.b) / p.b) if self.check(p.a, p.b) and p.b != 0 else None 7 | -------------------------------------------------------------------------------- /app/plugins/calculation/stdev.py: -------------------------------------------------------------------------------- 1 | from statistics import stdev 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("calculation", "stdev")): 7 | def calc(self, p): 8 | return stdev(self.prepare_list(p.a)) 9 | -------------------------------------------------------------------------------- /app/plugins/calculation/subtraction.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("calculation", "subtraction")): 5 | def calc(self, p): 6 | return (p.b - p.a) if self.check(p.a, p.b) else None 7 | -------------------------------------------------------------------------------- /app/plugins/calculation/zscore.py: -------------------------------------------------------------------------------- 1 | from statistics import mean, stdev 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("calculation", "zscore")): 7 | def calc(self, p): 8 | values = self.prepare_list(p.a) 9 | std_dev = stdev(values) 10 | return (values[-1] - mean(values)) / std_dev if std_dev != 0 else None 11 | -------------------------------------------------------------------------------- /app/plugins/data_processor/base.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BasePlugin 2 | 3 | 4 | class BaseProvider(BasePlugin("data_processor")): 5 | def exec(self, dataset): 6 | # Override in subclass 7 | return dataset 8 | -------------------------------------------------------------------------------- /app/plugins/data_processor/drop_duplicates.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("data_processor", "drop_duplicates")): 5 | def exec(self, dataset): 6 | return dataset.drop_duplicates() 7 | -------------------------------------------------------------------------------- /app/plugins/data_processor/drop_na.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("data_processor", "drop_na")): 5 | def exec(self, dataset): 6 | return dataset.dropna() 7 | -------------------------------------------------------------------------------- /app/plugins/data_processor/shuffle.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("data_processor", "shuffle")): 5 | def exec(self, dataset): 6 | return dataset.sample(frac=1) 7 | -------------------------------------------------------------------------------- /app/plugins/data_processor/sort.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("data_processor", "sort")): 5 | def exec(self, dataset, *fields): 6 | columns = [] 7 | ascending = [] 8 | 9 | for field in fields: 10 | field_ascending = True 11 | 12 | if field[0] == "~" or field[0] == "-": 13 | field = field[1:] 14 | field_ascending = False 15 | 16 | columns.append(field) 17 | ascending.append(field_ascending) 18 | 19 | dataset.sort_values(by=columns, inplace=True, ascending=ascending) 20 | return dataset 21 | -------------------------------------------------------------------------------- /app/plugins/dataset/collection.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("dataset", "collection")): 5 | pass 6 | -------------------------------------------------------------------------------- /app/plugins/encryption/aes256_user.py: -------------------------------------------------------------------------------- 1 | from systems.encryption.cipher import EncryptionError 2 | from systems.plugins.index import BaseProvider 3 | 4 | 5 | class Provider(BaseProvider("encryption", "aes256_user")): 6 | def initialize(self, options): 7 | if options.get("user", None): 8 | user_facade = self.manager.index.get_facade_index()["user"] 9 | user = user_facade.retrieve(options["user"]) 10 | if not user: 11 | raise EncryptionError("User {} does not exist".format(options["user"])) 12 | 13 | options["key"] = user.encryption_key 14 | -------------------------------------------------------------------------------- /app/plugins/field_processor/base.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BasePlugin 2 | 3 | 4 | class BaseProvider(BasePlugin("field_processor")): 5 | def exec(self, dataset, field_data): 6 | # Override in subclass 7 | return field_data 8 | -------------------------------------------------------------------------------- /app/plugins/field_processor/bool_to_number.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("field_processor", "bool_to_number")): 5 | def exec(self, dataset, field_data): 6 | return field_data.map({True: 1, False: 0}) 7 | -------------------------------------------------------------------------------- /app/plugins/field_processor/combined_text.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.data import load_json 3 | 4 | 5 | class Provider(BaseProvider("field_processor", "combined_text")): 6 | def exec(self, dataset, field_data, append=None, separator=","): 7 | if append: 8 | if isinstance(append, str): 9 | append = load_json(append) if append[0] == "[" else append.split(",") 10 | 11 | for field in append: 12 | field_data = field_data.str.cat(dataset[field].astype(str), sep=separator) 13 | return field_data 14 | -------------------------------------------------------------------------------- /app/plugins/formatter/base.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BasePlugin 2 | 3 | 4 | class BaseProvider(BasePlugin("formatter")): 5 | def __init__(self, type, name, command, config): 6 | super().__init__(type, name, command) 7 | self.config = config 8 | 9 | def format(self, value, record): 10 | # Override in subclass. 11 | return value 12 | 13 | def format_value(self, value, record, provider, **config): 14 | if "id" not in config: 15 | config["id"] = self.field_id 16 | 17 | return self.command.get_provider("formatter", provider, config).format(value, record) 18 | 19 | def error(self, message): 20 | self.command.error(f"Formatter {self.name} {self.field_id} failed: {message}") 21 | -------------------------------------------------------------------------------- /app/plugins/formatter/capitalize.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("formatter", "capitalize")): 7 | def format(self, value, record): 8 | value = super().format(value, record) 9 | if value is not None: 10 | if self.field_words: 11 | value = " ".join([item.capitalize() for item in re.split(r"\s+", value)]) 12 | else: 13 | value = value.capitalize() 14 | return value 15 | -------------------------------------------------------------------------------- /app/plugins/formatter/date.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("formatter", "date")): 7 | def format(self, value, record): 8 | if isinstance(value, float): 9 | value = int(value) 10 | try: 11 | value = datetime.datetime.strptime(str(value), self.field_format) 12 | except ValueError as e: 13 | self.error(f"Value {value} is not a valid date according to pattern: {self.field_format}") 14 | 15 | return value.date() 16 | -------------------------------------------------------------------------------- /app/plugins/formatter/date_time.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("formatter", "date_time")): 7 | def format(self, value, record): 8 | if isinstance(value, float): 9 | value = int(value) 10 | try: 11 | value = datetime.datetime.strptime(str(value), self.field_format) 12 | except ValueError as e: 13 | self.error(f"Value {value} is not a valid date time according to pattern: {self.field_format}") 14 | 15 | return value 16 | -------------------------------------------------------------------------------- /app/plugins/formatter/integer.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("formatter", "integer")): 5 | def format(self, value, record): 6 | value = super().format(value, record) 7 | return value if value is None else int(value) 8 | -------------------------------------------------------------------------------- /app/plugins/formatter/joiner.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("formatter", "joiner")): 5 | def format(self, value, record): 6 | if isinstance(value, (list, tuple)): 7 | value = self.field_join.join([str(elem) for elem in value]) 8 | return value 9 | -------------------------------------------------------------------------------- /app/plugins/formatter/lower.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("formatter", "lower")): 5 | def format(self, value, record): 6 | value = super().format(value, record) 7 | return value.lower() if value else None 8 | -------------------------------------------------------------------------------- /app/plugins/formatter/number.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from systems.plugins.index import BaseProvider 4 | from utility.data import number 5 | 6 | 7 | class Provider(BaseProvider("formatter", "number")): 8 | def format(self, value, record): 9 | try: 10 | if isinstance(value, str): 11 | value = float(value) 12 | except Exception: 13 | return None 14 | 15 | if value is None or math.isnan(value): 16 | return None 17 | return number(value) 18 | -------------------------------------------------------------------------------- /app/plugins/formatter/remove_suffix.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("formatter", "remove_suffix")): 5 | def format(self, value, record): 6 | value = super().format(value, record) 7 | if value is not None and self.field_suffix: 8 | value = value.removesuffix(self.field_suffix) 9 | return value 10 | -------------------------------------------------------------------------------- /app/plugins/formatter/string.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("formatter", "string")): 7 | def format(self, value, record): 8 | if not value or (not isinstance(value, str) and math.isnan(value)): 9 | return None 10 | return str(value) 11 | -------------------------------------------------------------------------------- /app/plugins/formatter/title.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("formatter", "title")): 5 | def format(self, value, record): 6 | value = super().format(value, record) 7 | return value.title() if value else None 8 | -------------------------------------------------------------------------------- /app/plugins/formatter/upper.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("formatter", "upper")): 5 | def format(self, value, record): 6 | value = super().format(value, record) 7 | return value.upper() if value else None 8 | -------------------------------------------------------------------------------- /app/plugins/function/base.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BasePlugin 2 | 3 | 4 | class BaseProvider(BasePlugin("function")): 5 | def exec(self, *parameters): 6 | # Override in subclass 7 | return None 8 | -------------------------------------------------------------------------------- /app/plugins/function/calculations.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("function", "calculations")): 7 | def exec(self, pattern): 8 | calculation_names = [] 9 | for name in self.manager.get_spec("calculation").keys(): 10 | if re.match(rf"{pattern}", name): 11 | calculation_names.append(name) 12 | return calculation_names 13 | -------------------------------------------------------------------------------- /app/plugins/function/capitalize.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "capitalize")): 5 | def exec(self, value): 6 | return value.capitalize() 7 | -------------------------------------------------------------------------------- /app/plugins/function/data_atomic_fields.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_atomic_fields")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return facade.atomic_fields 8 | -------------------------------------------------------------------------------- /app/plugins/function/data_dynamic_fields.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_dynamic_fields")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return facade.dynamic_fields 8 | -------------------------------------------------------------------------------- /app/plugins/function/data_id.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_id")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return facade.pk 8 | -------------------------------------------------------------------------------- /app/plugins/function/data_key.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_key")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return facade.key() 8 | -------------------------------------------------------------------------------- /app/plugins/function/data_query_fields.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_query_fields")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return facade.query_fields 8 | -------------------------------------------------------------------------------- /app/plugins/function/data_relation_fields.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_relation_fields")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return list(facade.get_extra_relations().keys()) 8 | -------------------------------------------------------------------------------- /app/plugins/function/data_reverse_relation_fields.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_reverse_relation_fields")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return list(facade.get_reverse_relations().keys()) 8 | -------------------------------------------------------------------------------- /app/plugins/function/data_scope_fields.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "data_scope_fields")): 5 | def exec(self, data_type): 6 | facade = self.command.facade(data_type) 7 | return facade.scope_fields 8 | -------------------------------------------------------------------------------- /app/plugins/function/default.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.data import normalize_value 3 | 4 | 5 | class Provider(BaseProvider("function", "default")): 6 | def exec(self, data, default): 7 | value = normalize_value(data) 8 | return default if value is None else value 9 | -------------------------------------------------------------------------------- /app/plugins/function/filter.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "filter")): 5 | def exec(self, data, **filters): 6 | filtered_data = {} 7 | 8 | for key, value in data.items(): 9 | add_key = True 10 | 11 | for filter_param, filter_value in filters.items(): 12 | if isinstance(value, dict) and filter_param in value: 13 | if value[filter_param] != filter_value: 14 | add_key = False 15 | elif not isinstance(filter_value, bool) or filter_value is True: 16 | add_key = False 17 | 18 | if add_key: 19 | filtered_data[key] = value 20 | 21 | return filtered_data 22 | -------------------------------------------------------------------------------- /app/plugins/function/flatten.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "flatten")): 5 | def exec(self, *elements): 6 | values = [] 7 | 8 | def flatten(value): 9 | if isinstance(value, (list, tuple)): 10 | for item in value: 11 | flatten(item) 12 | else: 13 | values.append(value) 14 | 15 | for element in elements: 16 | flatten(element) 17 | 18 | return values 19 | -------------------------------------------------------------------------------- /app/plugins/function/join.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.data import ensure_list 3 | 4 | 5 | class Provider(BaseProvider("function", "join")): 6 | def exec(self, *elements): 7 | values = [] 8 | 9 | for element in elements: 10 | values.extend(ensure_list(element)) 11 | 12 | return values 13 | -------------------------------------------------------------------------------- /app/plugins/function/keys.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "keys")): 5 | def exec(self, data, prefix=None): 6 | keys = list(data.keys()) 7 | if prefix is None: 8 | return keys 9 | 10 | for index, key in enumerate(keys): 11 | keys[index] = f"{prefix}{key}" 12 | 13 | return keys 14 | -------------------------------------------------------------------------------- /app/plugins/function/lstrip.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "lstrip")): 5 | def exec(self, value, prefix=None): 6 | value = value.strip() 7 | if prefix and value.startswith(prefix): 8 | value = value[len(prefix) :] 9 | return value 10 | -------------------------------------------------------------------------------- /app/plugins/function/mock_data.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.filesystem import load_yaml 3 | 4 | 5 | class Provider(BaseProvider("function", "mock_data")): 6 | def exec(self, type): 7 | data_file = self.manager.index.get_module_file(f"tests/data/{type}.yml") 8 | return load_yaml(data_file) 9 | -------------------------------------------------------------------------------- /app/plugins/function/normalize.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.data import normalize_value 3 | 4 | 5 | class Provider(BaseProvider("function", "normalize")): 6 | def exec(self, data): 7 | return normalize_value(data) 8 | -------------------------------------------------------------------------------- /app/plugins/function/prefix.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.data import ensure_list 3 | 4 | 5 | class Provider(BaseProvider("function", "prefix")): 6 | def exec(self, values, prefix=None, keys=None): 7 | if isinstance(values, dict): 8 | values = list(values.keys()) 9 | else: 10 | values = ensure_list(values) 11 | 12 | if prefix is None: 13 | return values 14 | 15 | for index, value in enumerate(values): 16 | if keys and isinstance(value, (list, tuple, dict)): 17 | for key in keys.split("."): 18 | if isinstance(value, (list, tuple)): 19 | value = value[int(key)] 20 | else: 21 | value = value[key] 22 | 23 | values[index] = f"{prefix}{value}" 24 | return values 25 | -------------------------------------------------------------------------------- /app/plugins/function/random_keys.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("function", "random_keys")): 7 | def exec(self, dict_value, limit=None): 8 | keys = list(dict_value.keys()) 9 | random.shuffle(keys) 10 | 11 | if limit: 12 | return keys[: min(int(limit), len(keys))] 13 | return keys 14 | -------------------------------------------------------------------------------- /app/plugins/function/random_values.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from systems.plugins.index import BaseProvider 4 | from utility.data import ensure_list 5 | 6 | 7 | class Provider(BaseProvider("function", "random_values")): 8 | def exec(self, list_value, limit=None): 9 | values = ensure_list(list_value) 10 | random.shuffle(values) 11 | 12 | if limit: 13 | return values[: min(int(limit), len(values))] 14 | return values 15 | -------------------------------------------------------------------------------- /app/plugins/function/rstrip.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "rstrip")): 5 | def exec(self, value, suffix=None): 6 | value = value.strip() 7 | if suffix and value.endswith(suffix): 8 | value = value[: -len(suffix)] 9 | return value 10 | -------------------------------------------------------------------------------- /app/plugins/function/split.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("function", "split")): 7 | def exec(self, str_value, split_value=","): 8 | return re.split(split_value, str_value) 9 | -------------------------------------------------------------------------------- /app/plugins/function/substitute.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "substitute")): 5 | def exec(self, value, search, replacement): 6 | return value.replace(search, replacement) 7 | -------------------------------------------------------------------------------- /app/plugins/function/time.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.time import Time 3 | 4 | 5 | class Provider(BaseProvider("function", "time")): 6 | def exec(self, format="%Y-%m-%dT%H:%M:%S"): 7 | return Time().now.strftime(format) 8 | -------------------------------------------------------------------------------- /app/plugins/function/time_range.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | from utility.time import Time 3 | 4 | 5 | class Provider(BaseProvider("function", "time_range")): 6 | def exec(self, start_time, end_time, unit_type="days"): 7 | return Time().generate(start_time, end_time, unit_type) 8 | -------------------------------------------------------------------------------- /app/plugins/function/value.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "value")): 5 | def exec(self, data, keys, default=None): 6 | if isinstance(keys, str): 7 | keys = keys.split(".") 8 | 9 | last_index = len(keys) - 1 10 | for index, key in enumerate(keys): 11 | if index == last_index: 12 | data = data.get(key, default) 13 | else: 14 | data = data.get(key, {}) 15 | 16 | return data 17 | -------------------------------------------------------------------------------- /app/plugins/function/values.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("function", "values")): 5 | def exec(self, data, keys=None): 6 | if keys is None: 7 | return list(data.values()) 8 | 9 | def get_value(inner_data, inner_keys): 10 | last_index = len(inner_keys) - 1 11 | for index, inner_key in enumerate(inner_keys): 12 | if index == last_index: 13 | inner_data = inner_data.get(inner_key, None) 14 | else: 15 | inner_data = inner_data.get(inner_key, {}) 16 | return inner_data 17 | 18 | keys = keys.split(".") 19 | values = [] 20 | if isinstance(data, (list, tuple)): 21 | for index, value in enumerate(data): 22 | values.append(get_value(value, keys)) 23 | elif isinstance(data, dict): 24 | for key, value in data.items(): 25 | values.append(get_value(value, keys)) 26 | 27 | return values 28 | -------------------------------------------------------------------------------- /app/plugins/mixins/cli_task.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import ProviderMixin 2 | from utility.data import env_value 3 | 4 | 5 | class CLITaskMixin(ProviderMixin("cli_task")): 6 | def _env_vars(self, params): 7 | return self.command.options.interpolate(env_value(self._merge_options(self.field_env, params.pop("env", {})))) 8 | -------------------------------------------------------------------------------- /app/plugins/mixins/list_calculation.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import ProviderMixin 2 | 3 | 4 | class ListCalculationMixin(ProviderMixin("list_calculation")): 5 | def prepare_list(self, list_data): 6 | if not isinstance(list_data, (list, tuple)): 7 | self.abort("Calculation requires a list parameter") 8 | 9 | list_data = [value for value in list(list_data) if value is not None] 10 | 11 | if len(list_data) == 0 or (self.field_min_values is not None and len(list_data) < self.field_min_values): 12 | self.set_null() 13 | 14 | return reversed(list_data) if self.field_reverse else list_data 15 | -------------------------------------------------------------------------------- /app/plugins/mixins/module_template.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from systems.plugins.index import ProviderMixin 4 | 5 | 6 | class ModuleTemplateMixin(ProviderMixin("module_template")): 7 | def _provision_template(self, instance, test=False): 8 | if self.field_use_template: 9 | template_fields = self.field_template_fields if self.field_template_fields else {} 10 | template_fields["module_name"] = instance.name 11 | 12 | if not os.path.isfile(os.path.join(self.module_path(instance.name, False), "zimagi.yml")): 13 | self.command.provision_template( 14 | instance, os.path.join("module", self.field_template_package), template_fields, display_only=test 15 | ) 16 | -------------------------------------------------------------------------------- /app/plugins/mixins/ssh_task.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import ProviderMixin 2 | 3 | 4 | class SSHTaskMixin(ProviderMixin("ssh_task")): 5 | def _get_ssh(self, env=None): 6 | if not env: 7 | env = {} 8 | 9 | return self.command.ssh( 10 | self.field_host, 11 | self.field_user, 12 | password=self.field_password, 13 | key=self.field_private_key, 14 | timeout=self.field_timeout, 15 | port=self.field_port, 16 | env=env, 17 | ) 18 | 19 | def _ssh_exec(self, command, args=None, options=None, env=None, sudo=False, ssh=None): 20 | if not args: 21 | args = [] 22 | if not options: 23 | options = {} 24 | 25 | if ssh is None: 26 | ssh = self._get_ssh(env) 27 | 28 | if sudo: 29 | ssh.sudo(command, *args, **options) 30 | else: 31 | ssh.exec(command, *args, **options) 32 | -------------------------------------------------------------------------------- /app/plugins/module/core.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from systems.plugins.index import BaseProvider 3 | 4 | 5 | class Provider(BaseProvider("module", "core")): 6 | def module_path(self, name, ensure=True): 7 | return settings.APP_DIR 8 | -------------------------------------------------------------------------------- /app/plugins/module/local.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | from django.conf import settings 4 | from systems.plugins.index import BaseProvider 5 | from utility.filesystem import remove_dir 6 | 7 | 8 | class Provider(BaseProvider("module", "local")): 9 | def initialize_instance(self, instance, created): 10 | instance.remote = None 11 | instance.reference = "development" 12 | 13 | def store_related(self, instance, created, test): 14 | if created: 15 | self._provision_template(instance, test) 16 | 17 | def finalize_instance(self, instance): 18 | if settings.DISABLE_MODULE_SYNC: 19 | return 20 | 21 | def finalize(): 22 | module_path = self.module_path(instance.name) 23 | remove_dir(pathlib.Path(module_path)) 24 | 25 | self.run_exclusive(f"local-finalize-{instance.name}", finalize) 26 | -------------------------------------------------------------------------------- /app/plugins/source/csv_file.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("source", "csv_file")): 5 | def load(self): 6 | return self.load_csv_data_from_file( 7 | self.field_file, 8 | self.import_columns, 9 | archive_file=self.field_archive_file, 10 | separator=self.field_separator, 11 | data_type=self.field_data_type, 12 | header=self.field_header, 13 | ) 14 | -------------------------------------------------------------------------------- /app/plugins/task/command.py: -------------------------------------------------------------------------------- 1 | import shlex 2 | 3 | from systems.plugins.index import BaseProvider 4 | from utility.shell import ShellError 5 | 6 | 7 | class Provider(BaseProvider("task", "command")): 8 | def execute(self, results, params): 9 | env = self._env_vars(params) 10 | stdin = params.pop("input", self.field_input) 11 | cwd = params.pop("cwd", self.field_cwd) 12 | display = params.pop("display", self.field_display) 13 | options = self._merge_options(self.field_options, params, self.field_lock) 14 | 15 | command = self._interpolate(self.field_command, options) 16 | success = self.command.sh( 17 | shlex.split(command[0]), input=stdin, display=display, env=env, cwd=cwd, sudo=self.field_sudo 18 | ) 19 | if not success: 20 | raise ShellError(f"Shell command failed: {command[0]}") 21 | -------------------------------------------------------------------------------- /app/plugins/task/remote_command.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BaseProvider 2 | 3 | 4 | class Provider(BaseProvider("task", "remote_command")): 5 | def execute(self, results, params): 6 | env = self._env_vars(params) 7 | options = self._merge_options(self.field_options, params, self.field_lock) 8 | command = self._interpolate(self.field_command, options) 9 | 10 | self._ssh_exec(command, env=env, sudo=self.field_sudo) 11 | -------------------------------------------------------------------------------- /app/plugins/task/remote_script.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from systems.plugins.index import BaseProvider 4 | from utility.data import ensure_list 5 | 6 | 7 | class Provider(BaseProvider("task", "remote_script")): 8 | def execute(self, results, params): 9 | script_path = self.get_path(self.field_script) 10 | 11 | if not os.path.exists(script_path): 12 | self.command.error(f"Remote script task provider file {script_path} does not exist") 13 | 14 | script_base, script_ext = os.path.splitext(script_path) 15 | temp_path = f"/tmp/{self.generate_name(24)}{script_ext}" 16 | 17 | env = self._env_vars(params) 18 | options = self._merge_options(self.field_options, params, self.field_lock) 19 | args = ensure_list(self.field_args, []) 20 | 21 | ssh = self._get_ssh(env) 22 | ssh.upload(script_path, temp_path, mode=755) 23 | try: 24 | self._ssh_exec(temp_path, self._interpolate(args, options), sudo=self.field_sudo, ssh=ssh) 25 | finally: 26 | ssh.exec("rm -f", temp_path) 27 | -------------------------------------------------------------------------------- /app/plugins/task/script.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from systems.plugins.index import BaseProvider 4 | from utility.data import ensure_list 5 | from utility.shell import ShellError 6 | 7 | 8 | class Provider(BaseProvider("task", "script")): 9 | def execute(self, results, params): 10 | script_path = self.get_path(self.field_script) 11 | 12 | if not os.path.exists(script_path): 13 | self.command.error(f"Script task provider file {script_path} does not exist") 14 | 15 | env = self._env_vars(params) 16 | stdin = params.pop("input", self.field_input) 17 | cwd = params.pop("cwd", self.field_cwd) 18 | display = params.pop("display", self.field_display) 19 | options = self._merge_options(self.field_options, params, self.field_lock) 20 | 21 | command = [script_path] + self._interpolate(ensure_list(self.field_args), options) 22 | success = self.command.sh(command, input=stdin, display=display, env=env, cwd=cwd, sudo=self.field_sudo) 23 | if not success: 24 | raise ShellError(f"Shell script failed: {command}") 25 | -------------------------------------------------------------------------------- /app/plugins/task/upload.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("task", "upload")): 7 | def execute(self, results, params): 8 | file_path = self.get_path(self.field_file) 9 | 10 | if not os.path.exists(file_path): 11 | self.command.error(f"Upload task provider file {file_path} does not exist") 12 | 13 | ssh = self._get_ssh() 14 | ssh.upload(file_path, self.field_remote_path, mode=self.field_mode, owner=self.field_owner, group=self.field_group) 15 | -------------------------------------------------------------------------------- /app/plugins/validator/base.py: -------------------------------------------------------------------------------- 1 | from systems.plugins.index import BasePlugin 2 | 3 | 4 | class BaseProvider(BasePlugin("validator")): 5 | def __init__(self, type, name, command, config): 6 | super().__init__(type, name, command) 7 | self.config = config 8 | 9 | def validate(self, value, record): 10 | # Override in subclass. 11 | return True 12 | 13 | def warning(self, message): 14 | self.command.warning(f"Validator {self.name} {self.field_id} failed: {message}") 15 | -------------------------------------------------------------------------------- /app/plugins/validator/date_time.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from systems.plugins.index import BaseProvider 4 | from utility.data import ensure_list 5 | 6 | 7 | class Provider(BaseProvider("validator", "date_time")): 8 | def validate(self, value, record): 9 | if not self.field_empty and not value: 10 | self.warning("Empty strings not allowed") 11 | return False 12 | 13 | if value: 14 | if isinstance(value, float): 15 | value = int(value) 16 | 17 | for date_format in ensure_list(self.field_format): 18 | try: 19 | datetime.datetime.strptime(str(value), date_format) 20 | return True 21 | except ValueError as e: 22 | pass 23 | 24 | self.warning(f"Value {value} is not a valid date time according to pattern: {self.field_format}") 25 | return False 26 | return True 27 | -------------------------------------------------------------------------------- /app/plugins/validator/number.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from systems.plugins.index import BaseProvider 4 | from utility.data import number 5 | 6 | 7 | class Provider(BaseProvider("validator", "number")): 8 | def validate(self, value, record): 9 | try: 10 | value = number(value) 11 | except (ValueError, TypeError): 12 | self.warning(f"Value {value} is not a number") 13 | return False 14 | 15 | if not self.field_nan and math.isnan(value): 16 | self.warning("Value can not be NaN") 17 | return False 18 | 19 | if self.field_min is not None: 20 | if value < self.field_min: 21 | self.warning(f"Value {value} is below minimum allowed: {self.field_min}") 22 | return False 23 | 24 | if self.field_max is not None: 25 | if value > self.field_max: 26 | self.warning(f"Value {value} is above maximum allowed: {self.field_max}") 27 | return False 28 | 29 | return True 30 | -------------------------------------------------------------------------------- /app/plugins/validator/string.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("validator", "string")): 7 | def validate(self, value, record): 8 | if not isinstance(value, str): 9 | self.warning(f"Value {value} is not a string") 10 | return False 11 | 12 | if not self.field_empty and len(value) == 0: 13 | self.warning("Empty strings not allowed") 14 | return False 15 | 16 | if self.field_pattern: 17 | pattern = re.compile(self.field_pattern) 18 | if not pattern.match(value): 19 | self.warning(f"Value {value} does not match pattern: {self.field_pattern}") 20 | return False 21 | 22 | return True 23 | -------------------------------------------------------------------------------- /app/plugins/worker/kubernetes.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from systems.plugins.index import BaseProvider 4 | 5 | 6 | class Provider(BaseProvider("worker", "kubernetes")): 7 | @property 8 | def cluster(self): 9 | return self.manager.cluster 10 | 11 | def scale_agents(self, count): 12 | self.cluster.scale_agent( 13 | self.field_worker_type, self.agent_name.replace("_", "-"), re.split(r"\s+", self.field_command_name), count 14 | ) 15 | 16 | def get_worker_count(self): 17 | return len(self.cluster.get_active_workers(self.field_worker_type)) 18 | 19 | def start_worker(self, name): 20 | self.cluster.create_worker(self.field_worker_type, name.replace("_", "-")) 21 | -------------------------------------------------------------------------------- /app/profiles/migrate.yml: -------------------------------------------------------------------------------- 1 | run: 2 | pause: 3 | _task: sleep 4 | number: 5 5 | 6 | build_migrations: 7 | _requires: pause 8 | _task: build_migrations 9 | 10 | migrate: 11 | _requires: build_migrations 12 | _task: migrate 13 | -------------------------------------------------------------------------------- /app/profiles/test/base.yml: -------------------------------------------------------------------------------- 1 | config: 2 | verbosity: 0 3 | -------------------------------------------------------------------------------- /app/profiles/test/cache.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/base 3 | 4 | run: 5 | cache_clear: 6 | _command: cache clear 7 | verbosity: '@verbosity' 8 | -------------------------------------------------------------------------------- /app/profiles/test/config.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/plugin 3 | 4 | config: 5 | data_type: config 6 | command_base: config 7 | 8 | run: 9 | 'save_<>': 10 | _foreach: '@instances' 11 | _command: '@command_base save' 12 | verbosity: '@verbosity' 13 | config_key: '<>' 14 | config_provider_name: '<>' 15 | config_value_type: '<>' 16 | config_value: '<>' 17 | groups_keys: '<>' 18 | 19 | 'find_<>': 20 | _requires: 'save_<>' 21 | _foreach: '@instances' 22 | _command: '@command_base list' 23 | verbosity: '@verbosity' 24 | instance_search_query: 25 | - 'name=<>' 26 | - 'provider_type=#default(<>, base)' 27 | - 'value_type=<>' 28 | - "?> <> ? 'groups__name__in=<>' | null" 29 | -------------------------------------------------------------------------------- /app/profiles/test/data/base.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/base 3 | 4 | config: 5 | data_type: null 6 | command_base: '@data_type' 7 | remove_count: 2 8 | 9 | pre_run: 10 | instances: 11 | _config: '#mock_data(@data_type)' 12 | 13 | fields: 14 | _config: '#data_atomic_fields(@data_type)' 15 | 16 | scope: 17 | _config: '#data_scope_fields(@data_type)' 18 | 19 | relations: 20 | _config: '#data_relation_fields(@data_type)' 21 | 22 | reverse_relations: 23 | _config: '#data_reverse_relation_fields(@data_type)' 24 | 25 | query_fields: 26 | _requires: [fields, scope, relations, reverse_relations] 27 | _config: '#flatten(@fields, @scope, @relations, @reverse_relations)' 28 | 29 | pre_destroy: 30 | instances: 31 | _config: '#mock_data(@data_type)' 32 | -------------------------------------------------------------------------------- /app/profiles/test/data/basic.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/base 3 | 4 | run: 5 | 'get_<>': 6 | _requires: 'save_<>' 7 | _foreach: '@instances' 8 | _command: '@command_base get' 9 | verbosity: '@verbosity' 10 | '@{data_type}_key': '<>' 11 | 12 | 'get_fields_<>': 13 | _requires: 'save_<>' 14 | _foreach: '@instances' 15 | _command: '@command_base get' 16 | verbosity: '@verbosity' 17 | '@{data_type}_key': '<>' 18 | field_names: '@query_fields' 19 | 20 | list_fields: 21 | _requires: '#prefix(@instances, save_)' 22 | _command: '@command_base list' 23 | verbosity: '@verbosity' 24 | field_names: '@query_fields' 25 | -------------------------------------------------------------------------------- /app/profiles/test/data/plugin.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/basic 3 | 4 | run: 5 | check_null_provider: 6 | _requires: '#prefix(@instances, save_)' 7 | _command: '@command_base list' 8 | verbosity: '@verbosity' 9 | instance_search_query: 10 | - 'provider_type=null' 11 | reverse_status: true 12 | -------------------------------------------------------------------------------- /app/profiles/test/data/remove.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/base 3 | 4 | destroy: 5 | 'remove_<>': 6 | _foreach: '#[remove_keys]random_keys(@instances, @remove_count)' 7 | _command: '@command_base remove' 8 | verbosity: '@verbosity' 9 | '@{data_type}_key': '<>' 10 | force: true 11 | 12 | 'get_removed_<>': 13 | _requires: 'remove_<>' 14 | _foreach: '@remove_keys' 15 | _command: '@command_base get' 16 | verbosity: '@verbosity' 17 | '@{data_type}_key': '<>' 18 | reverse_status: true 19 | 20 | clear: 21 | _requires: '#prefix(@remove_keys, get_removed_)' 22 | _command: '@command_base clear' 23 | verbosity: '@verbosity' 24 | force: true 25 | instance_search_query: 26 | - '#data_key(@data_type)__startswith=test__' 27 | 28 | list_cleared: 29 | _requires: clear 30 | _command: '@command_base list' 31 | verbosity: '@verbosity' 32 | instance_search_query: 33 | - '#data_key(@data_type)__startswith=test__' 34 | reverse_status: true 35 | -------------------------------------------------------------------------------- /app/profiles/test/dataset.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/plugin 3 | -------------------------------------------------------------------------------- /app/profiles/test/dependency.yml: -------------------------------------------------------------------------------- 1 | run: 2 | a: 3 | _config: 'A' 4 | b: 5 | _requires: [a] 6 | _config: 'B(@a)' 7 | c: 8 | _config: 'C' 9 | d: 10 | _requires: [b, c] 11 | _config: 'D(@b,@c)' 12 | e: 13 | _requires: [d] 14 | _config: 'E(@d)' 15 | f: 16 | _requires: [b, e] 17 | _config: 'F(@b,@e)' 18 | g: 19 | _requires: [a, c, d] 20 | _config: 'G(@a,@c,@d)' 21 | h: 22 | _requires: [f] 23 | _config: 'H(@f)' 24 | i: 25 | _requires: [f, g, h] 26 | _config: 'I(@f,@g,@h)' 27 | j: 28 | _requires: [c, i] 29 | _config: 'J(@c,@i)' 30 | 31 | test: 32 | _requires: [j] 33 | _task: test_equal 34 | _module: core 35 | first: '@j' 36 | second: 'J(C,I(F(B(A),E(D(B(A),C))),G(A,C,D(B(A),C)),H(F(B(A),E(D(B(A),C))))))' 37 | 38 | display: 39 | _requires: [j] 40 | _task: echo 41 | _module: core 42 | text: 'Final: @j' 43 | -------------------------------------------------------------------------------- /app/profiles/test/failure.yml: -------------------------------------------------------------------------------- 1 | _echo: &echo 2 | _task: echo 3 | _module: core 4 | 5 | run: 6 | test1: 7 | <<: *echo 8 | text: 'Starting out (1)' 9 | 10 | test2: 11 | _requires: [test1] 12 | _task: test_equal 13 | _module: core 14 | first: not 15 | second: equal 16 | 17 | test3: 18 | <<: *echo 19 | _requires: [test2] 20 | text: 'This should not be executed (3)' 21 | 22 | test4: 23 | <<: *echo 24 | _requires: [test2] 25 | text: 'This should not be executed (4)' 26 | 27 | test5: 28 | <<: *echo 29 | _requires: [test3, test4] 30 | text: 'This should not be executed (5)' 31 | -------------------------------------------------------------------------------- /app/profiles/test/host.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/basic 3 | 4 | config: 5 | data_type: host 6 | command_base: host 7 | 8 | run: 9 | 'save_<>': 10 | _foreach: '@instances' 11 | _command: '@command_base save' 12 | verbosity: '@verbosity' 13 | '@{data_type}_key': '<>' 14 | '@{data_type}_fields': 15 | host: '<>' 16 | command_port: '<>' 17 | data_port: '<>' 18 | user: '<>' 19 | token: '<>' 20 | encryption_key: '<>' 21 | 22 | 'find_<>': 23 | _requires: 'save_<>' 24 | _foreach: '@instances' 25 | _command: '@command_base list' 26 | verbosity: '@verbosity' 27 | instance_search_query: 28 | - 'name=<>' 29 | - 'host=<>' 30 | - 'command_port=<>' 31 | - 'data_port=<>' 32 | - 'user=<>' 33 | -------------------------------------------------------------------------------- /app/profiles/test/log.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/base 3 | 4 | config: 5 | log_fields: 6 | - name 7 | - command 8 | - config 9 | - user__name 10 | - status 11 | - task_id 12 | - schedule__name 13 | - created 14 | - updated 15 | 16 | pre_run: 17 | log_entries: 18 | _config: '&log(status__in=success,failed)::@log_fields' 19 | 20 | run: 21 | 'get_log_<>': 22 | _foreach: '#[log_sample]random_values(@log_entries, 20)' 23 | _command: log get 24 | verbosity: '@verbosity' 25 | log_key: '<>' 26 | 27 | list_fields: 28 | _command: log list 29 | verbosity: '@verbosity' 30 | log_order: '~created' 31 | log_limit: 20 32 | field_names: '@log_fields' 33 | 34 | 'find_<>': 35 | _foreach: '@log_sample' 36 | _command: log list 37 | verbosity: '@verbosity' 38 | instance_search_query: 39 | - 'name=<>' 40 | - 'command=<>' 41 | - 'user__name=<>' 42 | - 'status=<>' 43 | - 'task_id=<>' 44 | - 'schedule__name=<>' 45 | -------------------------------------------------------------------------------- /app/profiles/test/module.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/plugin 3 | -------------------------------------------------------------------------------- /app/profiles/test/notification.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/basic 3 | -------------------------------------------------------------------------------- /app/profiles/test/schedule.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/basic 3 | -------------------------------------------------------------------------------- /app/profiles/test/state.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/basic 3 | -------------------------------------------------------------------------------- /app/profiles/test/task.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/base 3 | 4 | run: 5 | echo_hello_world: 6 | _task: echo 7 | text: 'hello world!' 8 | 9 | sleep_10: 10 | _task: sleep 11 | number: 10 12 | unit: s 13 | -------------------------------------------------------------------------------- /app/profiles/test/user.yml: -------------------------------------------------------------------------------- 1 | parents: 2 | - test/data/plugin 3 | -------------------------------------------------------------------------------- /app/requirements.client.txt: -------------------------------------------------------------------------------- 1 | # Terminal utilities 2 | terminaltables==3.1.10 3 | colorful==0.5.6 4 | 5 | # Core application 6 | django==5.2.1 7 | 8 | # Text and data handling 9 | pyyaml==6.0.2 10 | oyaml==1.0 11 | pandas==2.2.3 12 | 13 | # API capabilities 14 | openapi-spec-validator==0.7.1 15 | -------------------------------------------------------------------------------- /app/requirements.local.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Development dependencies 3 | # 4 | 5 | # Code quality 6 | # ------------------------------------------------------------------------------ 7 | flake8==7.2.0 # https://github.com/PyCQA/flake8 8 | flake8-isort==6.1.2 # https://github.com/gforcada/flake8-isort 9 | black==25.1.0 # https://github.com/psf/black 10 | djlint==1.36.4 # https://github.com/Riverside-Healthcare/djLint 11 | pylint-django==2.6.1 # https://github.com/PyCQA/pylint-django 12 | pylint-celery==0.3 # https://github.com/PyCQA/pylint-celery 13 | pre-commit==4.2.0 # https://github.com/pre-commit/pre-commit 14 | -------------------------------------------------------------------------------- /app/scripts/celery-flower.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | cd /usr/local/share/zimagi 5 | #------------------------------------------------------------------------------- 6 | 7 | if [ ! -z "$ZIMAGI_POSTGRES_HOST" -a ! -z "$ZIMAGI_POSTGRES_PORT" ]; then 8 | ./scripts/wait.sh --hosts="$ZIMAGI_POSTGRES_HOST" --port="$ZIMAGI_POSTGRES_PORT" --timeout=60 9 | fi 10 | if [ ! -z "$ZIMAGI_REDIS_HOST" -a ! -z "$ZIMAGI_REDIS_PORT" ]; then 11 | ./scripts/wait.sh --hosts="$ZIMAGI_REDIS_HOST" --port="$ZIMAGI_REDIS_PORT" --timeout=60 12 | fi 13 | 14 | echo "> Starting Celery Flower" 15 | celery --app=settings flower --port=5000 16 | -------------------------------------------------------------------------------- /app/scripts/cli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | cd /usr/local/share/zimagi 5 | #------------------------------------------------------------------------------- 6 | 7 | # Execute CLI command 8 | ./zimagi-cli.py "$@" 9 | -------------------------------------------------------------------------------- /app/scripts/client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | cd /usr/local/share/zimagi 5 | #------------------------------------------------------------------------------- 6 | 7 | # Execute client command 8 | ./zimagi-client.py "$@" 9 | -------------------------------------------------------------------------------- /app/scripts/command.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | if [[ "${ZIMAGI_AUTO_UPDATE^^}" == "TRUE" ]]; then 4 | echo "> Starting file watcher" 5 | watchmedo auto-restart \ 6 | --directory=./ \ 7 | --directory=/usr/local/lib/zimagi \ 8 | --pattern="*.py;*.sh" \ 9 | --recursive \ 10 | --signal SIGTERM \ 11 | --debug-force-polling \ 12 | --interval 1 \ 13 | -- zimagi-gateway api command 14 | else 15 | zimagi-gateway api command 16 | fi 17 | -------------------------------------------------------------------------------- /app/scripts/config/api.sh: -------------------------------------------------------------------------------- 1 | export ZIMAGI_SERVICE_PROCESS=( 2 | "gunicorn" 3 | "services.wsgi:application" 4 | "--limit-request-field_size=0" 5 | "--limit-request-fields=0" 6 | "--limit-request-line=0" 7 | "--log-level=${ZIMAGI_LOG_LEVEL:-info}" 8 | "--access-logformat=[gunicorn] %(h)s '%(t)s' %(U)s '%(q)s' %(s)s %(b)s '%(f)s' '%(a)s'" 9 | "--access-logfile=-" # STDOUT 10 | "--error-logfile=-" # STDERR 11 | "--timeout=${ZIMAGI_SERVER_TIMEOUT:-14400}" 12 | "--worker-class=gevent" 13 | "--workers=${ZIMAGI_SERVER_WORKERS:-4}" 14 | "--worker-connections=${ZIMAGI_SERVER_CONNECTIONS:-100}" 15 | "--backlog=${ZIMAGI_SERVER_MAX_PENDING_CONNECTIONS:-3000}" 16 | "--bind=0.0.0.0:5000" 17 | ) 18 | -------------------------------------------------------------------------------- /app/scripts/config/controller.sh: -------------------------------------------------------------------------------- 1 | export ZIMAGI_SERVICE_PROCESS=( 2 | "./zimagi-cli.py" 3 | "agent" 4 | "controller" 5 | ) 6 | -------------------------------------------------------------------------------- /app/scripts/config/scheduler.sh: -------------------------------------------------------------------------------- 1 | export ZIMAGI_SERVICE_PROCESS=( 2 | "celery" 3 | "--app=settings" 4 | "beat" 5 | "--scheduler=systems.celery.scheduler:CeleryScheduler" 6 | "--loglevel=${ZIMAGI_LOG_LEVEL:-info}" 7 | "--pidfile=/var/local/zimagi/scheduler.pid" 8 | ) 9 | -------------------------------------------------------------------------------- /app/scripts/config/worker.sh: -------------------------------------------------------------------------------- 1 | export ZIMAGI_QUEUE_COMMANDS=True 2 | 3 | WORKER_QUEUE="default" 4 | if [ ! -z "$ZIMAGI_WORKER_TYPE" ]; then 5 | WORKER_QUEUE="${ZIMAGI_WORKER_TYPE}" 6 | fi 7 | 8 | export ZIMAGI_SERVICE_PROCESS=( 9 | "celery" 10 | "--app=settings" 11 | "worker" 12 | "--task-events" 13 | "--optimization=fair" 14 | "--pool=solo" 15 | "--concurrency=1" 16 | "--max-tasks-per-child=${ZIMAGI_WORKER_TASKS_PER_PROCESS:-100}" 17 | "--loglevel=${ZIMAGI_LOG_LEVEL:-info}" 18 | "--queues=${WORKER_QUEUE}" 19 | ) 20 | -------------------------------------------------------------------------------- /app/scripts/controller.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | if [[ "${ZIMAGI_AUTO_UPDATE^^}" == "TRUE" ]]; then 4 | echo "> Starting file watcher" 5 | watchmedo auto-restart \ 6 | --directory=./ \ 7 | --directory=/usr/local/lib/zimagi \ 8 | --pattern="*.py;*.sh" \ 9 | --recursive \ 10 | --signal SIGTERM \ 11 | --debug-force-polling \ 12 | --interval 1 \ 13 | -- zimagi-gateway controller 14 | else 15 | zimagi-gateway controller 16 | fi 17 | -------------------------------------------------------------------------------- /app/scripts/data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | if [[ "${ZIMAGI_AUTO_UPDATE^^}" == "TRUE" ]]; then 4 | echo "> Starting file watcher" 5 | watchmedo auto-restart \ 6 | --directory=./ \ 7 | --directory=/usr/local/lib/zimagi \ 8 | --pattern="*.py;*.sh" \ 9 | --recursive \ 10 | --signal SIGTERM \ 11 | --debug-force-polling \ 12 | --interval 1 \ 13 | -- zimagi-gateway api data 14 | else 15 | zimagi-gateway api data 16 | fi 17 | -------------------------------------------------------------------------------- /app/scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | cd /usr/local/share/zimagi 5 | 6 | export ZIMAGI_DEBUG=True 7 | export ZIMAGI_DISPLAY_COLOR=False 8 | #------------------------------------------------------------------------------- 9 | 10 | # Execute module install 11 | ./zimagi-install.py "$@" 12 | -------------------------------------------------------------------------------- /app/scripts/scheduler.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | if [[ "${ZIMAGI_AUTO_UPDATE^^}" == "TRUE" ]]; then 4 | echo "> Starting file watcher" 5 | watchmedo auto-restart \ 6 | --directory=./ \ 7 | --directory=/usr/local/lib/zimagi \ 8 | --pattern="*.py;*.sh" \ 9 | --recursive \ 10 | --signal SIGTERM \ 11 | --debug-force-polling \ 12 | --interval 1 \ 13 | -- zimagi-gateway scheduler tasks 14 | else 15 | zimagi-gateway scheduler tasks 16 | fi 17 | -------------------------------------------------------------------------------- /app/scripts/worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash --login 2 | #------------------------------------------------------------------------------- 3 | if [[ "${ZIMAGI_AUTO_UPDATE^^}" == "TRUE" ]]; then 4 | echo "> Starting file watcher" 5 | watchmedo auto-restart \ 6 | --directory=./ \ 7 | --directory=/usr/local/lib/zimagi \ 8 | --pattern="*.py;*.sh" \ 9 | --recursive \ 10 | --signal SIGTERM \ 11 | --debug-force-polling \ 12 | --interval 1 \ 13 | -- zimagi-gateway worker tasks 14 | else 15 | zimagi-gateway worker tasks 16 | fi 17 | -------------------------------------------------------------------------------- /app/services/cli/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | For the full list of settings and their values, see 3 | https://docs.djangoproject.com/en/4.1/ref/settings/ 4 | """ 5 | 6 | # ------------------------------------------------------------------------------- 7 | # Global settings 8 | 9 | # ------------------------------------------------------------------------------- 10 | # Core Django settings 11 | 12 | # ------------------------------------------------------------------------------- 13 | # Django Addons 14 | -------------------------------------------------------------------------------- /app/services/command/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | For the full list of settings and their values, see 3 | https://docs.djangoproject.com/en/4.1/ref/settings/ 4 | """ 5 | 6 | # ------------------------------------------------------------------------------- 7 | # Global settings 8 | 9 | # 10 | # Display configurations 11 | # 12 | DISPLAY_COLOR = False 13 | 14 | # ------------------------------------------------------------------------------- 15 | # Core Django settings 16 | 17 | # ------------------------------------------------------------------------------- 18 | # Django Addons 19 | 20 | # 21 | # API configuration 22 | # 23 | ROOT_URLCONF = "services.command.urls" 24 | 25 | REST_FRAMEWORK = { 26 | "UNAUTHENTICATED_USER": None, 27 | "DEFAULT_SCHEMA_CLASS": "systems.api.command.schema.CommandSchema", 28 | "DEFAULT_AUTHENTICATION_CLASSES": ["systems.api.command.auth.CommandAPITokenAuthentication"], 29 | "DEFAULT_PERMISSION_CLASSES": ["systems.api.command.auth.CommandPermission"], 30 | "DEFAULT_RENDERER_CLASSES": ["systems.api.renderers.JSONRenderer"], 31 | } 32 | -------------------------------------------------------------------------------- /app/services/command/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.urls import include, path, re_path 3 | from rest_framework import permissions 4 | from rest_framework.schemas import get_schema_view 5 | from systems.api import views as shared_views 6 | from systems.api.command import renderers, routers, schema 7 | 8 | status_view = shared_views.Status.as_view(permission_classes=[permissions.AllowAny], encryption=settings.ENCRYPT_COMMAND_API) 9 | 10 | urlpatterns = [ 11 | re_path(r"^status/?$", status_view), 12 | path("", include(routers.CommandAPIRouter().urls)), 13 | path( 14 | "", 15 | get_schema_view( 16 | title="Zimagi Command API", 17 | generator_class=schema.CommandSchemaGenerator, 18 | renderer_classes=[renderers.CommandSchemaJSONRenderer], 19 | permission_classes=[permissions.AllowAny], 20 | ), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /app/services/controller/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | For the full list of settings and their values, see 3 | https://docs.djangoproject.com/en/4.1/ref/settings/ 4 | """ 5 | 6 | # ------------------------------------------------------------------------------- 7 | # Global settings 8 | 9 | # ------------------------------------------------------------------------------- 10 | # Core Django settings 11 | 12 | # ------------------------------------------------------------------------------- 13 | # Django Addons 14 | -------------------------------------------------------------------------------- /app/services/tasks/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | For the full list of settings and their values, see 3 | https://docs.djangoproject.com/en/4.1/ref/settings/ 4 | """ 5 | 6 | # ------------------------------------------------------------------------------- 7 | # Global settings 8 | 9 | # 10 | # Display configurations 11 | # 12 | DISPLAY_COLOR = False 13 | 14 | # ------------------------------------------------------------------------------- 15 | # Core Django settings 16 | 17 | # ------------------------------------------------------------------------------- 18 | # Django Addons 19 | -------------------------------------------------------------------------------- /app/services/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | Zimagi WSGI config. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | from systems.models.overrides import * # noqa: F401, F403 14 | from utility.mutex import Mutex 15 | 16 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.full") 17 | application = get_wsgi_application() 18 | 19 | Mutex.set("startup_{}".format(os.environ["ZIMAGI_SERVICE"])) 20 | -------------------------------------------------------------------------------- /app/settings/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | import celery # noqa: F401 3 | 4 | add_celery_service = True 5 | 6 | except Exception: 7 | add_celery_service = False 8 | 9 | if add_celery_service: 10 | from services.celery import app as celery_app 11 | 12 | __all__ = ["celery_app"] 13 | -------------------------------------------------------------------------------- /app/settings/app.py: -------------------------------------------------------------------------------- 1 | from django.apps.config import AppConfig 2 | from django.conf import settings 3 | 4 | 5 | class AppInit(AppConfig): 6 | name = "zimagi" 7 | 8 | def ready(self): 9 | settings.MANAGER.index.generate() 10 | -------------------------------------------------------------------------------- /app/settings/roles.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | class RoleAccessError(Exception): 5 | pass 6 | 7 | 8 | class MetaRoles(type): 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | self.index = settings.MANAGER.index.roles 12 | 13 | def __getattr__(self, name): 14 | name = name.replace("_", "-") 15 | 16 | if name in self.index: 17 | return name 18 | else: 19 | raise RoleAccessError(f"Role {name} does not exist") 20 | 21 | def get_index(self): 22 | return self.index 23 | 24 | def get_help(self, name): 25 | return self.index[name] 26 | 27 | 28 | class Roles(metaclass=MetaRoles): 29 | pass 30 | -------------------------------------------------------------------------------- /app/settings/tasks.py: -------------------------------------------------------------------------------- 1 | from celery import shared_task 2 | 3 | 4 | @shared_task( 5 | bind=True, 6 | name="zimagi.command.exec", 7 | autoretry_for=(Exception,), 8 | retry_backoff=True, 9 | retry_backoff_max=(5 * 60), 10 | retry_jitter=True, 11 | ) 12 | def exec_command(self, command, **options): 13 | options.pop("schedule", None) 14 | options.pop("schedule_begin", None) 15 | options.pop("schedule_end", None) 16 | options.pop("async_exec", None) 17 | self.exec_command(command, options) 18 | 19 | 20 | @shared_task( 21 | bind=True, 22 | name="zimagi.notification.send", 23 | autoretry_for=(Exception,), 24 | retry_kwargs={"max_retries": 25}, 25 | retry_backoff=True, 26 | retry_backoff_max=(5 * 60), 27 | retry_jitter=True, 28 | ) 29 | def send_notification(self, recipient, subject, body, **options): 30 | self.send_notification(recipient, subject, body, **options) 31 | -------------------------------------------------------------------------------- /app/spec/base/data.yml: -------------------------------------------------------------------------------- 1 | data_base: 2 | name_resource: 3 | class: NameResourceBase 4 | mixins: [resource] 5 | id: name 6 | key: name 7 | fields: 8 | name: 9 | type: '@django.CharField' 10 | color: key 11 | options: 12 | primary_key: true 13 | max_length: 256 14 | meta: 15 | ordering: [name] 16 | 17 | id_resource: 18 | class: IdentifierResourceBase 19 | mixins: [resource] 20 | id_fields: [name] 21 | key: name 22 | fields: 23 | id: 24 | type: '@django.CharField' 25 | color: key 26 | options: 27 | primary_key: true 28 | max_length: 64 29 | editable: false 30 | name: 31 | type: '@django.CharField' 32 | color: key 33 | options: 34 | max_length: 256 35 | meta: 36 | ordering: [name] 37 | -------------------------------------------------------------------------------- /app/spec/channels.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | administration: 3 | admin: 4 | db-admin: 5 | schedule-admin: 6 | notification-admin: 7 | user-admin: 8 | config-admin: 9 | module-admin: 10 | data-admin: 11 | -------------------------------------------------------------------------------- /app/spec/commands/archiver.yml: -------------------------------------------------------------------------------- 1 | command: 2 | agent: 3 | archiver: 4 | base: agent 5 | mixins: [scaling_event] 6 | -------------------------------------------------------------------------------- /app/spec/commands/cache.yml: -------------------------------------------------------------------------------- 1 | command: 2 | cache: 3 | priority: 44 4 | clear: 5 | base: cache 6 | -------------------------------------------------------------------------------- /app/spec/commands/config.yml: -------------------------------------------------------------------------------- 1 | command: 2 | config: 3 | resource: config 4 | base_name: config 5 | base: config 6 | priority: 20 7 | options: 8 | save_fields: 9 | value_type: [config_value_type, '--type'] 10 | value: [config_value, true] 11 | -------------------------------------------------------------------------------- /app/spec/commands/controller.yml: -------------------------------------------------------------------------------- 1 | command: 2 | agent: 3 | controller: 4 | base: agent 5 | -------------------------------------------------------------------------------- /app/spec/commands/database.yml: -------------------------------------------------------------------------------- 1 | command: 2 | db: 3 | priority: 40 4 | 5 | snapshots: 6 | priority: 50 7 | base: db 8 | 9 | backup: 10 | priority: 55 11 | base: db 12 | 13 | restore: 14 | priority: 60 15 | base: db 16 | confirm: true 17 | parse: 18 | snapshot_name: 19 | force: 20 | 21 | clean: 22 | priority: 65 23 | base: db 24 | parameters: 25 | keep_num: 26 | parser: variable 27 | type: int 28 | default: '@settings.DB_SNAPSHOT_RENTENTION' 29 | optional: '--keep' 30 | help: 'Number of Zimagi backup snapshots to keep' 31 | value_label: NUM 32 | tags: [db] 33 | parse: 34 | keep_num: 35 | -------------------------------------------------------------------------------- /app/spec/commands/dataset.yml: -------------------------------------------------------------------------------- 1 | command: 2 | data: 3 | resource: dataset 4 | base_name: dataset 5 | base: dataset 6 | priority: 55 7 | -------------------------------------------------------------------------------- /app/spec/commands/gpu.yml: -------------------------------------------------------------------------------- 1 | command: 2 | gpu: 3 | base: gpu 4 | priority: 999 5 | -------------------------------------------------------------------------------- /app/spec/commands/group.yml: -------------------------------------------------------------------------------- 1 | command: 2 | group: 3 | resource: group 4 | base_name: group 5 | base: group 6 | priority: 10 7 | children: 8 | base: group 9 | parameters: 10 | group_child_keys: 11 | parser: variables 12 | type: str 13 | optional: false 14 | help: 'one or more child group keys' 15 | value_label: 'GROUP_KEY' 16 | tags: [group] 17 | parse: 18 | group_key: 19 | help_text: 'parent group key' 20 | group_child_keys: 21 | group_provider_name: 22 | -------------------------------------------------------------------------------- /app/spec/commands/host.yml: -------------------------------------------------------------------------------- 1 | command: 2 | host: 3 | resource: host 4 | base_name: host 5 | base: host 6 | priority: 1 7 | -------------------------------------------------------------------------------- /app/spec/commands/log.yml: -------------------------------------------------------------------------------- 1 | command: 2 | log: 3 | resource: log 4 | base_name: log 5 | base: log 6 | priority: 50 7 | options: 8 | allow_access: false 9 | get: 10 | base: log 11 | priority: 12 12 | parse: 13 | log_key: false 14 | poll_interval: 15 | abort: 16 | base: log 17 | priority: 50 18 | parse: 19 | log_keys: false 20 | rerun: 21 | base: log 22 | priority: 55 23 | parse: 24 | log_keys: false 25 | clean: 26 | base: log 27 | priority: 60 28 | parse: 29 | log_days: 30 | log_message_days: 31 | -------------------------------------------------------------------------------- /app/spec/commands/message.yml: -------------------------------------------------------------------------------- 1 | command: 2 | send: 3 | base: message 4 | priority: 940 5 | parse: 6 | communication_channel: 7 | communication_message: 8 | listen: 9 | base: message 10 | priority: 940 11 | parse: 12 | communication_channel: 13 | communication_timeout: 14 | -------------------------------------------------------------------------------- /app/spec/commands/notification.yml: -------------------------------------------------------------------------------- 1 | command: 2 | notification: 3 | resource: notification 4 | base_name: notification 5 | base: notification 6 | priority: 30 7 | options: 8 | allow_access: false 9 | allow_update: false 10 | allow_remove: false 11 | save: 12 | base: notification 13 | parse: 14 | group_provider_name: '--group-provider' 15 | notify_failure: 16 | notify_command: true 17 | notify_groups: true 18 | remove: 19 | base: notification 20 | parse: 21 | group_provider_name: '--group-provider' 22 | notify_failure: 23 | notify_command: true 24 | notify_groups: true 25 | clear: 26 | base: notification 27 | -------------------------------------------------------------------------------- /app/spec/commands/platform.yml: -------------------------------------------------------------------------------- 1 | command: 2 | info: 3 | base: platform 4 | priority: 5 5 | version: 6 | base: platform 7 | priority: -4 8 | test: 9 | base: platform 10 | priority: -3 11 | parameters: 12 | test_types: 13 | parser: variables 14 | type: str 15 | default: null 16 | optional: '--types' 17 | help: 'test types' 18 | value_label: TYPE 19 | tags: [test] 20 | test_tags: 21 | parser: variables 22 | type: str 23 | default: null 24 | optional: '--tags' 25 | help: 'test tags (only supported on some types)' 26 | value_label: TAG 27 | tags: [test] 28 | test_exclude_tags: 29 | parser: variables 30 | type: str 31 | default: null 32 | optional: '--exclude-tags' 33 | help: 'test excluded tags (only supported on some types)' 34 | value_label: TAG 35 | tags: [test] 36 | parse: 37 | - test_types 38 | - test_tags 39 | - test_exclude_tags 40 | -------------------------------------------------------------------------------- /app/spec/commands/scaling.yml: -------------------------------------------------------------------------------- 1 | command: 2 | scaling: 3 | base: scaling_event 4 | resource: scaling_event 5 | base_name: scaling 6 | priority: 55 7 | options: 8 | allow_access: false 9 | allow_update: false 10 | allow_remove: false 11 | -------------------------------------------------------------------------------- /app/spec/commands/schedule.yml: -------------------------------------------------------------------------------- 1 | command: 2 | schedule: 3 | resource: scheduled_task 4 | base_name: scheduled_task 5 | base: schedule 6 | priority: 30 7 | -------------------------------------------------------------------------------- /app/spec/commands/state.yml: -------------------------------------------------------------------------------- 1 | command: 2 | state: 3 | resource: state 4 | base_name: state 5 | base: state 6 | priority: 20 7 | options: 8 | save_fields: 9 | value: [state_value, false] 10 | -------------------------------------------------------------------------------- /app/spec/commands/template.yml: -------------------------------------------------------------------------------- 1 | command: 2 | template: 3 | priority: 26 4 | generate: 5 | base: module 6 | priority: 10 7 | parse: 8 | module_key: false 9 | module_template: false 10 | template_fields: true 11 | display_only: 12 | -------------------------------------------------------------------------------- /app/spec/commands/user.yml: -------------------------------------------------------------------------------- 1 | command: 2 | user: 3 | resource: user 4 | base_name: user 5 | base: user 6 | priority: 5 7 | rotate: 8 | base: user 9 | parse: 10 | user_key: true 11 | -------------------------------------------------------------------------------- /app/spec/data/cache.yml: -------------------------------------------------------------------------------- 1 | data: 2 | cache: 3 | class: Cache 4 | base: id_resource 5 | disable_ops: [update, destroy] 6 | roles: 7 | edit: [] 8 | view: [admin] 9 | packages: [] 10 | fields: 11 | name: 12 | type: '@django.CharField' 13 | color: key 14 | options: 15 | max_length: 2000 16 | requests: 17 | type: '@django.PositiveBigIntegerField' 18 | options: 19 | default: 0 20 | meta: 21 | ordering: ['-requests'] 22 | -------------------------------------------------------------------------------- /app/spec/data/config.yml: -------------------------------------------------------------------------------- 1 | data: 2 | config: 3 | class: Config 4 | base: name_resource 5 | mixins: [provider, group] 6 | roles: 7 | edit: [config-admin] 8 | view: [config-auditor] 9 | triggers: 10 | check: [config_ensure] 11 | fields: 12 | value: 13 | type: '@zimagi.DataField' 14 | options: 15 | 'null': true 16 | value_type: 17 | type: '@django.CharField' 18 | options: 19 | max_length: 150 20 | default: str 21 | meta: 22 | provider_name: config 23 | -------------------------------------------------------------------------------- /app/spec/data/dataset.yml: -------------------------------------------------------------------------------- 1 | data: 2 | dataset: 3 | class: DataSet 4 | base: name_resource 5 | mixins: [provider, group] 6 | roles: 7 | edit: [data-admin] 8 | view: [data-auditor] 9 | meta: 10 | dynamic_fields: [data] 11 | provider_name: dataset 12 | command_base: data 13 | -------------------------------------------------------------------------------- /app/spec/data/group.yml: -------------------------------------------------------------------------------- 1 | data_mixins: 2 | group: 3 | class: GroupMixin 4 | fields: 5 | groups: 6 | type: '@django.ManyToManyField' 7 | relation: group 8 | options: 9 | blank: true 10 | 11 | data: 12 | group: 13 | class: Group 14 | base: name_resource 15 | mixins: [provider] 16 | auto_create: true 17 | roles: 18 | edit: [user-admin, config-admin] 19 | view: [user-auditor, config-auditor] 20 | triggers: 21 | check: [group_ensure] 22 | save: [group_ensure] 23 | fields: 24 | parent: 25 | type: '@django.ForeignKey' 26 | relation: group 27 | options: 28 | 'null': true 29 | on_delete: '@django.SET_NULL' 30 | related_name: children 31 | meta: 32 | provider_name: group 33 | -------------------------------------------------------------------------------- /app/spec/data/host.yml: -------------------------------------------------------------------------------- 1 | data: 2 | host: 3 | class: Host 4 | base: name_resource 5 | roles: 6 | edit: false 7 | view: false 8 | packages: [host] 9 | api: false 10 | fields: 11 | host: 12 | type: '@django.URLField' 13 | command_port: 14 | type: '@django.IntegerField' 15 | options: 16 | 'null': true 17 | data_port: 18 | type: '@django.IntegerField' 19 | options: 20 | 'null': true 21 | user: 22 | type: '@django.CharField' 23 | color: relation 24 | options: 25 | 'null': true 26 | max_length: 150 27 | token: 28 | type: '@django.CharField' 29 | options: 30 | 'null': true 31 | max_length: 256 32 | encryption_key: 33 | type: '@django.CharField' 34 | options: 35 | 'null': true 36 | max_length: 256 37 | -------------------------------------------------------------------------------- /app/spec/data/module.yml: -------------------------------------------------------------------------------- 1 | data: 2 | module: 3 | class: Module 4 | base: name_resource 5 | mixins: [provider, group] 6 | roles: 7 | edit: [module-admin] 8 | view: [module-auditor] 9 | triggers: 10 | check: [module_ensure] 11 | save: [module_ensure] 12 | fields: 13 | remote: 14 | type: '@django.CharField' 15 | options: 16 | 'null': true 17 | max_length: 256 18 | reference: 19 | type: '@django.CharField' 20 | options: 21 | 'null': true 22 | max_length: 128 23 | meta: 24 | dynamic_fields: [status, version, compatibility] 25 | ordering: ['-provider_type', name] 26 | provider_name: module 27 | -------------------------------------------------------------------------------- /app/spec/data/scaling.yml: -------------------------------------------------------------------------------- 1 | data: 2 | scaling_event: 3 | class: ScalingEvent 4 | base: name_resource 5 | disable_ops: [update, destroy] 6 | roles: 7 | edit: [] 8 | view: [admin] 9 | packages: [] 10 | fields: 11 | command: 12 | type: '@django.CharField' 13 | options: 14 | max_length: 255 15 | worker_type: 16 | type: '@django.CharField' 17 | options: 18 | max_length: 255 19 | 20 | worker_max_count: 21 | type: '@django.IntegerField' 22 | options: 23 | 'null': true 24 | 25 | worker_count: 26 | type: '@django.IntegerField' 27 | options: 28 | 'null': true 29 | 30 | task_count: 31 | type: '@django.IntegerField' 32 | options: 33 | 'null': true 34 | 35 | workers_created: 36 | type: '@django.IntegerField' 37 | options: 38 | 'null': true 39 | 40 | meta: 41 | ordering: ['-created'] 42 | -------------------------------------------------------------------------------- /app/spec/data/state.yml: -------------------------------------------------------------------------------- 1 | data: 2 | state: 3 | class: State 4 | base: name_resource 5 | roles: 6 | edit: [config-admin] 7 | view: [config-auditor] 8 | fields: 9 | value: 10 | type: '@zimagi.DataField' 11 | options: 12 | 'null': true 13 | -------------------------------------------------------------------------------- /app/spec/encryption.yml: -------------------------------------------------------------------------------- 1 | encryption: 2 | user_api_key: 3 | provider: aes256 4 | command_api: 5 | provider: aes256_user 6 | data_api: 7 | provider: aes256_user 8 | -------------------------------------------------------------------------------- /app/spec/mixins/data.yml: -------------------------------------------------------------------------------- 1 | data_mixins: 2 | resource: 3 | class: ResourceMixin 4 | config: 5 | class: ConfigMixin 6 | fields: 7 | config: 8 | type: '@zimagi.DictionaryField' 9 | color: json 10 | options: 11 | system: true 12 | provider: 13 | class: ProviderMixin 14 | mixins: [config] 15 | fields: 16 | provider_type: 17 | type: '@django.CharField' 18 | options: 19 | default: base 20 | max_length: 128 21 | system: true 22 | variables: 23 | type: '@zimagi.DictionaryField' 24 | color: json 25 | options: 26 | system: true 27 | editable: false 28 | group: 29 | class: GroupMixin 30 | fields: 31 | groups: 32 | type: '@django.ManyToManyField' 33 | relation: group 34 | options: 35 | blank: true 36 | -------------------------------------------------------------------------------- /app/spec/plugins/config.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | config: 3 | base: data 4 | data: config 5 | -------------------------------------------------------------------------------- /app/spec/plugins/data_processor.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | data_processor: 3 | base: base 4 | interface: 5 | exec: 6 | params: 7 | dataset: 'pandas.DataFrame' 8 | options: dict 9 | returns: 'pandas.DataFrame' 10 | providers: 11 | drop_duplicates: 12 | drop_na: 13 | shuffle: 14 | sort: 15 | -------------------------------------------------------------------------------- /app/spec/plugins/field_processor.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | field_processor: 3 | base: base 4 | interface: 5 | exec: 6 | params: 7 | dataset: 'pandas.DataFrame' 8 | field_data: 'pandas.Series' 9 | options: dict 10 | returns: 'pandas.Series' 11 | providers: 12 | bool_to_number: 13 | combined_text: 14 | -------------------------------------------------------------------------------- /app/spec/plugins/function.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | function: 3 | base: base 4 | interface: 5 | exec: 6 | params: '*' 7 | returns: '*' 8 | providers: 9 | mock_data: 10 | 11 | data_id: 12 | data_key: 13 | data_atomic_fields: 14 | data_dynamic_fields: 15 | data_query_fields: 16 | data_scope_fields: 17 | data_relation_fields: 18 | data_reverse_relation_fields: 19 | 20 | calculations: 21 | 22 | flatten: 23 | keys: 24 | random_keys: 25 | random_values: 26 | filter: 27 | values: 28 | value: 29 | prefix: 30 | join: 31 | split: 32 | lstrip: 33 | rstrip: 34 | capitalize: 35 | substitute: 36 | normalize: 37 | default: 38 | time: 39 | time_range: 40 | -------------------------------------------------------------------------------- /app/spec/plugins/group.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | group: 3 | base: data 4 | data: group 5 | providers: 6 | role: 7 | classification: 8 | -------------------------------------------------------------------------------- /app/spec/plugins/parser.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | parser: 3 | base: base 4 | interface: 5 | initialize: 6 | params: 7 | reset: bool 8 | parse: 9 | params: 10 | value: str 11 | returns: '*' 12 | providers: 13 | conditional_value: 14 | weight: 1 15 | function: 16 | weight: 2 17 | token: 18 | weight: 5 19 | state: 20 | weight: 5 21 | config: 22 | weight: 5 23 | reference: 24 | weight: 95 25 | query: true 26 | -------------------------------------------------------------------------------- /app/spec/plugins/user.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | user: 3 | base: data 4 | data: user 5 | -------------------------------------------------------------------------------- /app/spec/plugins/worker.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | worker: 3 | base: base 4 | interface: 5 | ensure: 6 | 7 | requirement: 8 | worker_type: 9 | type: str 10 | help: 'Worker processor machine type' 11 | command_name: 12 | type: str 13 | help: 'Full command name to be executed by worker' 14 | command_options: 15 | type: dict 16 | help: 'Command options passed to the worker' 17 | 18 | providers: 19 | docker: 20 | kubernetes: 21 | -------------------------------------------------------------------------------- /app/spec/workers.yml: -------------------------------------------------------------------------------- 1 | workers: 2 | default: 3 | image: '@RUNTIME_IMAGE' 4 | docker_runtime: '@DOCKER_RUNTIME' 5 | kube_cpu_min: 500m 6 | kube_cpu_max: 1500m 7 | kube_memory_min: 2000Mi 8 | kube_memory_max: 2400Mi 9 | env: {} 10 | -------------------------------------------------------------------------------- /app/systems/api/command/auth.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from rest_framework import permissions 4 | from systems.api.auth import APITokenAuthentication 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class CommandAPITokenAuthentication(APITokenAuthentication): 10 | api_type = "command_api" 11 | 12 | 13 | class CommandPermission(permissions.BasePermission): 14 | def has_permission(self, request, view): 15 | return view.check_execute(request.user) 16 | -------------------------------------------------------------------------------- /app/systems/api/command/renderers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import renderers 2 | from systems.api.command.codecs import ZimagiJSONCodec 3 | from systems.api.encoders import SafeJSONEncoder 4 | 5 | 6 | class CommandSchemaJSONRenderer(renderers.BaseRenderer): 7 | media_type = "application/vnd.zimagi+json" 8 | 9 | def render(self, data, media_type=None, renderer_context=None): 10 | return ZimagiJSONCodec().encode(data, cls=SafeJSONEncoder, indent=int(renderer_context.get("indent", 2))) 11 | -------------------------------------------------------------------------------- /app/systems/api/command/response.py: -------------------------------------------------------------------------------- 1 | from systems.api import response as shared_responses 2 | 3 | 4 | class EncryptedResponse(shared_responses.EncryptedResponse): 5 | api_type = "command_api" 6 | -------------------------------------------------------------------------------- /app/systems/api/command/routers.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.conf import settings 4 | from django.urls import path 5 | from rest_framework import routers 6 | from systems.commands import exec, router 7 | 8 | from . import views 9 | 10 | 11 | class CommandAPIRouter(routers.BaseRouter): 12 | def get_urls(self): 13 | urls = [] 14 | 15 | def add_commands(command): 16 | for subcommand in command.get_subcommands(): 17 | if isinstance(subcommand, router.RouterCommand): 18 | add_commands(subcommand) 19 | 20 | elif isinstance(subcommand, exec.ExecCommand) and subcommand.api_enabled(): 21 | if settings.API_EXEC: 22 | subcommand.parse_base() 23 | 24 | name = subcommand.get_full_name() 25 | urls.append(path(re.sub(r"\s+", "/", name), views.Command.as_view(name=name, command=subcommand))) 26 | 27 | add_commands(settings.MANAGER.index.command_tree) 28 | return urls 29 | -------------------------------------------------------------------------------- /app/systems/api/data/parsers.py: -------------------------------------------------------------------------------- 1 | from rest_framework.exceptions import ParseError 2 | from rest_framework.parsers import BaseParser 3 | from systems.encryption.cipher import Cipher 4 | from utility.data import load_json 5 | 6 | 7 | class JSONParser(BaseParser): 8 | media_type = "application/json" 9 | 10 | def parse(self, stream, media_type, parser_context): 11 | request = parser_context["request"] 12 | cipher = Cipher.get("data_api", user=request.user.name if request.user else None) 13 | try: 14 | return load_json(cipher.decrypt(stream.read())) 15 | 16 | except ValueError as e: 17 | raise ParseError(f"JSON parse error: {e}") 18 | -------------------------------------------------------------------------------- /app/systems/api/data/renderers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import renderers 2 | from systems.api.encoders import SafeJSONEncoder 3 | from utility.data import dump_json 4 | 5 | 6 | class DataSchemaJSONRenderer(renderers.JSONOpenAPIRenderer): 7 | def render(self, data, media_type=None, renderer_context=None): 8 | indent = int(renderer_context.get("indent", 2)) 9 | return dump_json(data, cls=SafeJSONEncoder, indent=indent).encode("utf-8") 10 | -------------------------------------------------------------------------------- /app/systems/api/data/response.py: -------------------------------------------------------------------------------- 1 | from systems.api import response as shared_responses 2 | 3 | 4 | class EncryptedResponse(shared_responses.EncryptedResponse): 5 | api_type = "data_api" 6 | -------------------------------------------------------------------------------- /app/systems/api/encoders.py: -------------------------------------------------------------------------------- 1 | from rest_framework.utils import encoders 2 | 3 | 4 | class SafeJSONEncoder(encoders.JSONEncoder): 5 | def default(self, obj): 6 | try: 7 | return super().default(obj) 8 | except Exception as e: 9 | return str(obj) 10 | -------------------------------------------------------------------------------- /app/systems/api/renderers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import renderers 2 | 3 | from .encoders import SafeJSONEncoder 4 | 5 | 6 | class JSONRenderer(renderers.JSONRenderer): 7 | encoder_class = SafeJSONEncoder 8 | -------------------------------------------------------------------------------- /app/systems/api/response.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from rest_framework.response import Response 3 | from systems.encryption.cipher import Cipher 4 | 5 | 6 | class EncryptedResponse(Response): 7 | api_type = None 8 | 9 | def __init__(self, user=None, api_type=None, **kwargs): 10 | super().__init__(**kwargs) 11 | self.user = user 12 | 13 | if api_type: 14 | self.api_type = api_type 15 | 16 | @property 17 | def rendered_content(self): 18 | if not self.api_type or not getattr( 19 | settings, "ENCRYPT_{}_API".format(self.api_type.replace("_api", "").upper()), True 20 | ): 21 | return super().rendered_content 22 | return Cipher.get(self.api_type, user=self.user).encrypt(super().rendered_content) 23 | -------------------------------------------------------------------------------- /app/systems/celery/app.py: -------------------------------------------------------------------------------- 1 | from celery import Celery as BaseCelery 2 | 3 | 4 | class Celery(BaseCelery): 5 | registry_cls = "systems.celery.registry:CommandTaskRegistry" 6 | -------------------------------------------------------------------------------- /app/systems/celery/registry.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from importlib import import_module 3 | 4 | from celery._state import get_current_app 5 | from celery.app.registry import TaskRegistry 6 | 7 | 8 | class CommandTaskRegistry(TaskRegistry): 9 | def __getitem__(self, name): 10 | return copy.deepcopy(super().__getitem__(name)) 11 | 12 | def get(self, name, default=None): 13 | if default is not None and name not in self: 14 | return default 15 | return self.__getitem__(name) 16 | 17 | def get_original(self, name): 18 | return super().__getitem__(name) 19 | 20 | 21 | def _unpickle_task(name, module=None): 22 | if module: 23 | import_module(module) 24 | return get_current_app().tasks.get_original(name) 25 | -------------------------------------------------------------------------------- /app/systems/client/cli/commands/version.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | from zimagi.command.messages import TableMessage 4 | 5 | from .action import ActionCommand 6 | 7 | 8 | class VersionCommand(ActionCommand): 9 | 10 | def exec(self): 11 | TableMessage([["Client version", settings.VERSION]]).display() 12 | self.client.execute(self.name, **self.options) 13 | self.print("") 14 | 15 | def handle_message(self, message): 16 | if not message.system: 17 | super().handle_message(message) 18 | -------------------------------------------------------------------------------- /app/systems/client/cli/errors.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import CommandError 2 | 3 | 4 | class CommandNetworkError(CommandError): 5 | pass 6 | 7 | 8 | class CommandNotFoundError(CommandError): 9 | pass 10 | 11 | 12 | class CommandMessageError(CommandError): 13 | pass 14 | 15 | 16 | class CommandAbort(CommandError): 17 | pass 18 | -------------------------------------------------------------------------------- /app/systems/commands/calculator.py: -------------------------------------------------------------------------------- 1 | from systems.commands.processor import Processor 2 | 3 | 4 | class Calculator(Processor): 5 | def __init__(self, command, display_only=False, reset=False): 6 | super().__init__( 7 | command, 8 | "calculation", 9 | display_only=display_only, 10 | ) 11 | self.reset = reset 12 | 13 | def provider_process(self, name, spec): 14 | self.command.get_provider(self.plugin_key, spec[self.plugin_key], name, spec).process(self.reset) 15 | -------------------------------------------------------------------------------- /app/systems/commands/factory/helpers.py: -------------------------------------------------------------------------------- 1 | def get_value(value, default): 2 | return value if value is not None else default 3 | 4 | 5 | def get_facade(base_name): 6 | return f"_{base_name}" 7 | 8 | 9 | def get_joined_value(*args): 10 | return "_".join([x for x in args if x is not None]) 11 | 12 | 13 | def parse_field_names(command): 14 | command.parse_variables( 15 | "field_names", "--fields", str, "field names to display", value_label="FIELD_NAME", tags=["fields"] 16 | ) 17 | 18 | 19 | def get_field_names(command): 20 | return command.options.get("field_names") 21 | 22 | 23 | def parse_fields(command, fields): 24 | for name, info in fields.items(): 25 | getattr(command, f"parse_{info[0]}")(*info[1:], tags=["fields"]) 26 | 27 | 28 | def get_fields(command, fields): 29 | data = {} 30 | for name, info in fields.items(): 31 | data[name] = getattr(command, info[0]) 32 | return data 33 | -------------------------------------------------------------------------------- /app/systems/commands/importer.py: -------------------------------------------------------------------------------- 1 | from systems.commands.processor import Processor 2 | 3 | 4 | class Importer(Processor): 5 | def __init__(self, command, display_only=False): 6 | super().__init__(command, "import", plugin_key="source", display_only=display_only) 7 | -------------------------------------------------------------------------------- /app/systems/db/backends/postgresql/base.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.db import DEFAULT_DB_ALIAS 3 | from django.db.backends.postgresql import base as postgresql 4 | 5 | 6 | class DatabaseWrapper(postgresql.DatabaseWrapper): 7 | def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS): 8 | super().__init__(settings_dict, alias) 9 | 10 | def get_new_connection(self, conn_params): 11 | with settings.DB_LOCK: 12 | return super().get_new_connection(conn_params) 13 | -------------------------------------------------------------------------------- /app/systems/db/router.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | class DatabaseRouter: 5 | def db_for_read(self, model, **hints): 6 | return "default" 7 | 8 | def db_for_write(self, model, **hints): 9 | if "write" in settings.DATABASES: 10 | return "write" 11 | return "default" 12 | 13 | def allow_relation(self, obj1, obj2, **hints): 14 | return True 15 | 16 | def allow_migrate(self, db, app_label, model_name=None, **hints): 17 | return True 18 | -------------------------------------------------------------------------------- /app/systems/models/aggregates.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Aggregate 2 | 3 | 4 | class Concat(Aggregate): 5 | function = "STRING_AGG" 6 | template = "%(function)s(%(distinct)s%(expressions)s, '%(separator)s')" 7 | allow_distinct = True 8 | 9 | def __init__(self, expression, separator=",", **extra): 10 | super().__init__(expression, separator=separator, **extra) 11 | -------------------------------------------------------------------------------- /app/systems/models/errors.py: -------------------------------------------------------------------------------- 1 | class ParseError(Exception): 2 | pass 3 | 4 | 5 | class ProviderError(Exception): 6 | pass 7 | 8 | 9 | class ScopeError(Exception): 10 | pass 11 | 12 | 13 | class AccessError(Exception): 14 | pass 15 | 16 | 17 | class RestrictedError(Exception): 18 | pass 19 | 20 | 21 | class UpdateError(Exception): 22 | pass 23 | -------------------------------------------------------------------------------- /app/systems/models/fields.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from utility.data import serialize, unserialize 3 | 4 | 5 | class DataField(models.TextField): 6 | def to_python(self, value): 7 | return unserialize(value) 8 | 9 | def from_db_value(self, value, expression, connection): 10 | return self.to_python(value) 11 | 12 | def get_prep_value(self, value): 13 | return serialize(value) 14 | 15 | def value_from_object(self, obj): 16 | value = super().value_from_object(obj) 17 | return self.get_prep_value(value) 18 | 19 | def value_to_string(self, obj): 20 | return self.value_from_object(obj) 21 | 22 | 23 | class ListField(models.JSONField): 24 | def __init__(self, *args, **kwargs): 25 | kwargs["default"] = list 26 | kwargs["null"] = False 27 | super().__init__(*args, **kwargs) 28 | 29 | 30 | class DictionaryField(models.JSONField): 31 | def __init__(self, *args, **kwargs): 32 | kwargs["default"] = dict 33 | kwargs["null"] = False 34 | super().__init__(*args, **kwargs) 35 | -------------------------------------------------------------------------------- /app/systems/models/mixins/render.py: -------------------------------------------------------------------------------- 1 | from django.utils.timezone import localtime 2 | 3 | 4 | class ModelFacadeRenderMixin: 5 | def get_field_created_display(self, instance, value, short): 6 | return localtime(value).strftime("%Y-%m-%d %H:%M:%S %Z") 7 | 8 | def get_field_updated_display(self, instance, value, short): 9 | return localtime(value).strftime("%Y-%m-%d %H:%M:%S %Z") 10 | -------------------------------------------------------------------------------- /app/systems/models/parsers/fields.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseParser 4 | from .field_processors import FieldProcessor, FieldProcessorParser 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class FieldParser(FieldProcessorParser, BaseParser): 10 | base_parsers = ( 11 | "p_expression_number", 12 | "p_expression_field", 13 | "p_expression_db_function", 14 | "p_expression_paren", 15 | "p_expression_calculation", 16 | ) 17 | 18 | def p_expression_assignment(self, p): 19 | """ 20 | expression : NAME EQUALS expression 21 | """ 22 | if isinstance(p[3], FieldProcessor): 23 | p[3].name = p[1] 24 | p[0] = p[3] 25 | logger.debug(f"name = processor: {p[0]}") 26 | else: 27 | annotations = {p[1]: p[3]} 28 | 29 | if self.facade: 30 | self.facade.add_annotations(**annotations) 31 | 32 | p[0] = p[1] 33 | logger.debug(f"name = value: {p[0]} {annotations}") 34 | -------------------------------------------------------------------------------- /app/systems/models/parsers/filters.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseParser 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | class FilterParser(BaseParser): 9 | # 10 | # Parser rules 11 | # 12 | base_parsers = ( 13 | "p_expression_boolean", 14 | "p_expression_number", 15 | "p_expression_string", 16 | "p_expression_field", 17 | "p_expression_db_function", 18 | "p_expression_paren", 19 | "p_expression_calculation", 20 | ) 21 | -------------------------------------------------------------------------------- /app/systems/models/parsers/function.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.db.models import F 4 | 5 | from .base import BaseParser 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class FunctionParser(BaseParser): 11 | # 12 | # Parser evaluation 13 | # 14 | def process(self, statement): 15 | if isinstance(statement, F): 16 | return statement.name 17 | return statement 18 | 19 | # 20 | # Parser rules 21 | # 22 | base_parsers = ("p_expression_db_function",) 23 | -------------------------------------------------------------------------------- /app/systems/models/parsers/order.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.db.models import F 4 | 5 | from .base import BaseParser 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class OrderParser(BaseParser): 11 | # 12 | # Parser indexes 13 | # 14 | tokens = [*BaseParser.tokens, "DASH", "TILDE"] 15 | 16 | # 17 | # Token definitions (in order) 18 | # 19 | t_DASH = r"\-" 20 | t_TILDE = r"\~" 21 | 22 | # 23 | # Parser evaluation 24 | # 25 | def process(self, statement): 26 | if isinstance(statement, F): 27 | return statement.name 28 | return statement 29 | 30 | # 31 | # Parser rules 32 | # 33 | base_parsers = ("p_expression_field", "p_expression_db_function") 34 | 35 | def p_expression_field_negation(self, p): 36 | """ 37 | expression : DASH NAME 38 | | TILDE NAME 39 | | DASH db_function 40 | | TILDE db_function 41 | """ 42 | p[0] = f"-{p[2].name if isinstance(p[2], F) else p[2]}" 43 | logger.debug(f"[-|~]field: {p[0]}") 44 | -------------------------------------------------------------------------------- /app/systems/plugins/base.py: -------------------------------------------------------------------------------- 1 | class BasePluginMixin: 2 | @classmethod 3 | def generate(cls, plugin, generator): 4 | # Override in subclass if needed 5 | pass 6 | -------------------------------------------------------------------------------- /app/tasks/zimagi.yml: -------------------------------------------------------------------------------- 1 | test_migrations: 2 | provider: command 3 | command: zimagi makemigrations --dry-run 4 | 5 | build_migrations: 6 | provider: command 7 | command: zimagi makemigrations 8 | 9 | migrate: 10 | provider: command 11 | command: zimagi migrate 12 | -------------------------------------------------------------------------------- /app/templates/data/model/plugin.yml: -------------------------------------------------------------------------------- 1 | plugin: 2 | <{ name }>: 3 | base: data 4 | data: <{ name }> 5 | -------------------------------------------------------------------------------- /app/templates/field/big_integer/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of big integer field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this integer field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | primary_key: 18 | help: Whether or not this field is the primary key of the model 19 | default: false 20 | 21 | map: 22 | spec.yml: 23 | target: spec/data/<{ data_name }>.yml 24 | location: data.<{ data_name }>.fields.<{ field_name }> 25 | -------------------------------------------------------------------------------- /app/templates/field/big_integer/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.BigIntegerField" 2 | options: 3 | "null": <{ nullable if default is none and not primary_key else false }> 4 | default: <{ default if not primary_key else null }> 5 | editable: <{ editable }> 6 | primary_key: <{ primary_key }> 7 | system: <{ not editable }> 8 | -------------------------------------------------------------------------------- /app/templates/field/binary/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of binary field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this binary field can be NULL 10 | default: true 11 | max_length: 12 | help: Maximum length for this field 13 | default: 256 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | 18 | map: 19 | spec.yml: 20 | target: spec/data/<{ data_name }>.yml 21 | location: data.<{ data_name }>.fields.<{ field_name }> 22 | -------------------------------------------------------------------------------- /app/templates/field/binary/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.BinaryField" 2 | options: 3 | "null": <{ nullable }> 4 | max_length: <{ max_length }> 5 | editable: <{ editable }> 6 | system: <{ not editable }> 7 | -------------------------------------------------------------------------------- /app/templates/field/boolean/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of boolean field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this boolean field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | 18 | map: 19 | spec.yml: 20 | target: spec/data/<{ data_name }>.yml 21 | location: data.<{ data_name }>.fields.<{ field_name }> 22 | -------------------------------------------------------------------------------- /app/templates/field/boolean/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.BooleanField" 2 | options: 3 | "null": <{ nullable if default is none else false }> 4 | default: <{ default }> 5 | editable: <{ editable }> 6 | system: <{ not editable }> 7 | -------------------------------------------------------------------------------- /app/templates/field/date/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of date field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this date field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | primary_key: 18 | help: Whether or not this field is the primary key of the model 19 | default: false 20 | 21 | map: 22 | spec.yml: 23 | target: spec/data/<{ data_name }>.yml 24 | location: data.<{ data_name }>.fields.<{ field_name }> 25 | -------------------------------------------------------------------------------- /app/templates/field/date/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.DateField" 2 | options: 3 | "null": <{ nullable if default is none and not primary_key else false }> 4 | default: <{ default if not primary_key else null }> 5 | editable: <{ editable }> 6 | primary_key: <{ primary_key }> 7 | system: <{ not editable }> 8 | -------------------------------------------------------------------------------- /app/templates/field/datetime/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of datetime field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this datetime field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | primary_key: 18 | help: Whether or not this field is the primary key of the model 19 | default: false 20 | 21 | map: 22 | spec.yml: 23 | target: spec/data/<{ data_name }>.yml 24 | location: data.<{ data_name }>.fields.<{ field_name }> 25 | -------------------------------------------------------------------------------- /app/templates/field/datetime/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.DateTimeField" 2 | options: 3 | "null": <{ nullable if default is none and not primary_key else false }> 4 | default: <{ default if not primary_key else null }> 5 | editable: <{ editable }> 6 | primary_key: <{ primary_key }> 7 | system: <{ not editable }> 8 | -------------------------------------------------------------------------------- /app/templates/field/dict/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of dictionary field on data model 7 | required: true 8 | editable: 9 | help: Whether or not this field is editable by users 10 | default: true 11 | 12 | map: 13 | spec.yml: 14 | target: spec/data/<{ data_name }>.yml 15 | location: data.<{ data_name }>.fields.<{ field_name }> 16 | -------------------------------------------------------------------------------- /app/templates/field/dict/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@zimagi.DictionaryField" 2 | color: json 3 | options: 4 | editable: <{ editable }> 5 | system: <{ not editable }> 6 | -------------------------------------------------------------------------------- /app/templates/field/duration/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of duration field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this duration field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | 18 | map: 19 | spec.yml: 20 | target: spec/data/<{ data_name }>.yml 21 | location: data.<{ data_name }>.fields.<{ field_name }> 22 | -------------------------------------------------------------------------------- /app/templates/field/duration/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.DurationField" 2 | options: 3 | "null": <{ nullable if default is none else false }> 4 | default: <{ default }> 5 | editable: <{ editable }> 6 | system: <{ not editable }> 7 | -------------------------------------------------------------------------------- /app/templates/field/float/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of float field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this float field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | primary_key: 18 | help: Whether or not this field is the primary key of the model 19 | default: false 20 | 21 | map: 22 | spec.yml: 23 | target: spec/data/<{ data_name }>.yml 24 | location: data.<{ data_name }>.fields.<{ field_name }> 25 | -------------------------------------------------------------------------------- /app/templates/field/float/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.FloatField" 2 | options: 3 | "null": <{ nullable if default is none and not primary_key else false }> 4 | default: <{ default if not primary_key else null }> 5 | editable: <{ editable }> 6 | primary_key: <{ primary_key }> 7 | system: <{ not editable }> 8 | -------------------------------------------------------------------------------- /app/templates/field/foreign_key/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of foreign key field on data model 7 | required: true 8 | related_data_name: 9 | help: Name of data model relation 10 | required: true 11 | reverse_related_name: 12 | help: Name of data field for reverse relation lookup on attached data model 13 | default: null 14 | nullable: 15 | help: Whether or not this foreign key field can be NULL (false if default specified) 16 | default: true 17 | on_delete: 18 | help: "How to handle deletion of related data (options: SET_NULL, SET_DEFAULT, CASCADE, PROTECT, RESTRICT, DO_NOTHING)" 19 | default: SET_NULL 20 | editable: 21 | help: Whether or not this field is editable by users 22 | default: true 23 | 24 | map: 25 | spec.yml: 26 | target: spec/data/<{ data_name }>.yml 27 | location: data.<{ data_name }>.fields.<{ field_name }> 28 | -------------------------------------------------------------------------------- /app/templates/field/foreign_key/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.ForeignKey" 2 | relation: <{ related_data_name }> 3 | color: relation 4 | options: 5 | "null": <{ nullable }> 6 | on_delete: "@django.<{ on_delete }>" 7 | editable: <{ editable }> 8 | system: <{ not editable }> 9 | #%- if reverse_related_name %# 10 | related_name: "<{ reverse_related_name }>" 11 | #%- endif %# 12 | -------------------------------------------------------------------------------- /app/templates/field/integer/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of integer field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this integer field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | primary_key: 18 | help: Whether or not this field is the primary key of the model 19 | default: false 20 | 21 | map: 22 | spec.yml: 23 | target: spec/data/<{ data_name }>.yml 24 | location: data.<{ data_name }>.fields.<{ field_name }> 25 | -------------------------------------------------------------------------------- /app/templates/field/integer/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.IntegerField" 2 | options: 3 | "null": <{ nullable if default is none and not primary_key else false }> 4 | default: <{ default if not primary_key else null }> 5 | editable: <{ editable }> 6 | primary_key: <{ primary_key }> 7 | system: <{ not editable }> 8 | -------------------------------------------------------------------------------- /app/templates/field/list/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of list field on data model 7 | required: true 8 | editable: 9 | help: Whether or not this field is editable by users 10 | default: true 11 | 12 | map: 13 | spec.yml: 14 | target: spec/data/<{ data_name }>.yml 15 | location: data.<{ data_name }>.fields.<{ field_name }> 16 | -------------------------------------------------------------------------------- /app/templates/field/list/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@zimagi.ListField" 2 | color: json 3 | options: 4 | editable: <{ editable }> 5 | system: <{ not editable }> 6 | -------------------------------------------------------------------------------- /app/templates/field/many_to_many/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of many to many field on data model 7 | required: true 8 | related_data_name: 9 | help: Name of data model relation 10 | required: true 11 | reverse_related_name: 12 | help: Name of data field for reverse relation lookup on attached data model 13 | default: null 14 | 15 | map: 16 | spec.yml: 17 | target: spec/data/<{ data_name }>.yml 18 | location: data.<{ data_name }>.fields.<{ field_name }> 19 | -------------------------------------------------------------------------------- /app/templates/field/many_to_many/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.ManyToManyField" 2 | relation: <{ related_data_name }> 3 | color: relation 4 | options: 5 | blank: true 6 | #%- if reverse_related_name %# 7 | related_name: "<{ reverse_related_name }>" 8 | #%- endif %# 9 | -------------------------------------------------------------------------------- /app/templates/field/string/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of character field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this string field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | choices: 15 | help: Choices available for field value 16 | default: null 17 | max_length: 18 | help: Maximum character length for this field 19 | default: 256 20 | editable: 21 | help: Whether or not this field is editable by users 22 | default: true 23 | primary_key: 24 | help: Whether or not this field is the primary key of the model 25 | default: false 26 | 27 | map: 28 | spec.yml: 29 | target: spec/data/<{ data_name }>.yml 30 | location: data.<{ data_name }>.fields.<{ field_name }> 31 | -------------------------------------------------------------------------------- /app/templates/field/string/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.CharField" 2 | options: 3 | "null": <{ nullable if default is none and not primary_key else false }> 4 | default: <{ default if not primary_key else null }> 5 | choices: <{ choices }> 6 | max_length: <{ max_length }> 7 | editable: <{ editable }> 8 | primary_key: <{ primary_key }> 9 | system: <{ not editable }> 10 | -------------------------------------------------------------------------------- /app/templates/field/text/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of text field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this text field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | editable: 15 | help: Whether or not this field is editable by users 16 | default: true 17 | 18 | map: 19 | spec.yml: 20 | target: spec/data/<{ data_name }>.yml 21 | location: data.<{ data_name }>.fields.<{ field_name }> 22 | -------------------------------------------------------------------------------- /app/templates/field/text/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.TextField" 2 | options: 3 | "null": <{ nullable if default is none else false }> 4 | default: <{ default }> 5 | editable: <{ editable }> 6 | system: <{ not editable }> 7 | -------------------------------------------------------------------------------- /app/templates/field/url/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | data_name: 3 | help: Name of data model 4 | required: true 5 | field_name: 6 | help: Name of URL field on data model 7 | required: true 8 | nullable: 9 | help: Whether or not this URL field can be NULL (false if default specified) 10 | default: true 11 | default: 12 | help: Default value for this field 13 | default: null 14 | max_length: 15 | help: Maximum character length for this field 16 | default: 256 17 | editable: 18 | help: Whether or not this field is editable by users 19 | default: true 20 | 21 | map: 22 | spec.yml: 23 | target: spec/data/<{ data_name }>.yml 24 | location: data.<{ data_name }>.fields.<{ field_name }> 25 | -------------------------------------------------------------------------------- /app/templates/field/url/spec.yml: -------------------------------------------------------------------------------- 1 | type: "@django.URLField" 2 | options: 3 | "null": <{ nullable if default is none else false }> 4 | default: <{ default }> 5 | max_length: <{ max_length }> 6 | editable: <{ editable }> 7 | system: <{ not editable }> 8 | -------------------------------------------------------------------------------- /app/templates/functions/core_class.py: -------------------------------------------------------------------------------- 1 | # 2 | # Class related functions 3 | # 4 | 5 | 6 | def class_name(name): 7 | return name.replace("_", " ").title().replace(" ", "") 8 | -------------------------------------------------------------------------------- /app/templates/functions/core_list.py: -------------------------------------------------------------------------------- 1 | from utility import data 2 | 3 | # 4 | # List processing functions 5 | # 6 | 7 | 8 | def ensure_list(value): 9 | return data.ensure_list(value) 10 | -------------------------------------------------------------------------------- /app/templates/functions/core_text.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | # 4 | # Text processing functions 5 | # 6 | 7 | 8 | def split_text(text, pattern): 9 | return re.split(pattern, text) 10 | -------------------------------------------------------------------------------- /app/templates/module/standard/.gitignore: -------------------------------------------------------------------------------- 1 | # Python cache files 2 | __pycache__ 3 | 4 | # System files 5 | .zimagi* 6 | -------------------------------------------------------------------------------- /app/templates/module/standard/django.py: -------------------------------------------------------------------------------- 1 | from settings.config import Config 2 | 3 | # 4 | # Module environment configurations 5 | # 6 | -------------------------------------------------------------------------------- /app/templates/module/standard/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Install module related dependencies 4 | # 5 | set -e 6 | -------------------------------------------------------------------------------- /app/templates/module/standard/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/app/templates/module/standard/requirements.txt -------------------------------------------------------------------------------- /app/templates/module/standard/zimagi.yml: -------------------------------------------------------------------------------- 1 | name: <{ module_name }> 2 | 3 | version: 0.0.1 4 | compatibility: ">=0.11.0,<0.12" 5 | 6 | #%- if include_install_script %# 7 | scripts: scripts/install.sh 8 | #%- endif %# 9 | #%- if inlude_requirements %# 10 | requirements: requirements.txt 11 | #%- endif %# 12 | 13 | #%- if modules %# 14 | modules: 15 | #%- for module in modules %# 16 | - remote: <{ module.remote }> 17 | reference: <{ module.reference }> 18 | #%- endfor %# 19 | #%- endif %# 20 | -------------------------------------------------------------------------------- /app/templates/user/role/index.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | name: 3 | help: Name of Zimagi role (use dashes, not underscores) 4 | required: true 5 | help: 6 | help: Zimagi role help message 7 | required: true 8 | 9 | map: 10 | spec.yml: 11 | target: spec/roles.yml 12 | location: roles.<{ name }> 13 | -------------------------------------------------------------------------------- /app/templates/user/role/spec.yml: -------------------------------------------------------------------------------- 1 | "<{ help }>" 2 | -------------------------------------------------------------------------------- /app/tests/base.py: -------------------------------------------------------------------------------- 1 | class BaseTest: 2 | def __init__(self, command, tags=None, exclude_tags=None): 3 | self.command = command 4 | self.manager = command.manager 5 | self.tags = tags or [] 6 | self.exclude_tags = exclude_tags or [] 7 | 8 | def exec(self): 9 | raise NotImplementedError("Subclasses of BaseTest must implement exec method") 10 | -------------------------------------------------------------------------------- /app/tests/command_local.py: -------------------------------------------------------------------------------- 1 | from tests.command.base import BaseCommandTest 2 | 3 | 4 | class Test(BaseCommandTest): 5 | pass 6 | -------------------------------------------------------------------------------- /app/tests/data/config.yml: -------------------------------------------------------------------------------- 1 | test__1: 2 | provider_type: null 3 | value_type: dict 4 | value: 5 | test1: 1 6 | test2: 2 7 | test3: 3 8 | groups: [test__config1] 9 | test__2: 10 | provider_type: base 11 | value_type: int 12 | value: 25 13 | groups: null 14 | test__3: 15 | provider_type: base 16 | value_type: str 17 | value: 'hello world' 18 | groups: [test__config1, test__config2] 19 | test__4: 20 | provider_type: base 21 | value_type: list 22 | value: [test1, test2, test3, test4] 23 | groups: null 24 | test__5: 25 | provider_type: null 26 | value_type: bool 27 | value: True 28 | groups: [] 29 | test__6: 30 | provider_type: base 31 | value_type: float 32 | value: 54.76 33 | groups: [test__config1, test__config3] 34 | -------------------------------------------------------------------------------- /app/tests/data/group.yml: -------------------------------------------------------------------------------- 1 | test__1: 2 | provider_type: base 3 | parent: null 4 | test__2: 5 | provider_type: classification 6 | parent: null 7 | test__3: 8 | provider_type: role 9 | parent: null 10 | test__4: 11 | provider_type: role 12 | parent: test__3 13 | test__5: 14 | provider_type: classification 15 | parent: test__1 16 | -------------------------------------------------------------------------------- /app/tests/data/host.yml: -------------------------------------------------------------------------------- 1 | test__1: 2 | host: test1.example.com 3 | command_port: 5555 4 | data_port: 5870 5 | user: test1 6 | token: testing1 7 | encryption_key: 1234567890 8 | test__2: 9 | host: test2.example.com 10 | command_port: 5510 11 | data_port: 5263 12 | user: test2 13 | token: testing2 14 | encryption_key: 0987654321 15 | test__3: 16 | host: test3.example.com 17 | command_port: 5700 18 | data_port: 5480 19 | user: test3 20 | token: testing3 21 | encryption_key: zzzzz97zzz 22 | test__4: 23 | host: test4.example.com 24 | command_port: 5900 25 | data_port: 5001 26 | user: test4 27 | token: testing4 28 | encryption_key: 01010101010101 29 | test__5: 30 | host: test5.example.com 31 | command_port: 5777 32 | data_port: 5999 33 | user: test5 34 | token: testing5 35 | encryption_key: abcdefghijk 36 | -------------------------------------------------------------------------------- /app/tests/sdk_python/init/test_status.py: -------------------------------------------------------------------------------- 1 | from django.test import tag 2 | from tests.sdk_python.base import BaseTest 3 | 4 | SUCCESS_MESSAGE = "System check successful" 5 | 6 | 7 | @tag("init", "status") 8 | class StatusTest(BaseTest): 9 | @tag("data_status") 10 | def test_data_status(self): 11 | status_info = self.data_api.get_status() 12 | self.assertEqual(status_info["message"], SUCCESS_MESSAGE) 13 | 14 | @tag("command_status") 15 | def test_command_status(self): 16 | status_info = self.command_api.get_status() 17 | self.assertEqual(status_info["message"], SUCCESS_MESSAGE) 18 | -------------------------------------------------------------------------------- /app/tests/sdk_python/runner.py: -------------------------------------------------------------------------------- 1 | from django.test.runner import DiscoverRunner 2 | 3 | 4 | class TestRunner(DiscoverRunner): 5 | 6 | def run_tests(self, test_labels, extra_tests=None, **kwargs): 7 | self.setup_test_environment() 8 | suite = self.build_suite(test_labels) 9 | run_failed = False 10 | try: 11 | result = self.run_suite(suite) 12 | except Exception: 13 | run_failed = True 14 | raise 15 | finally: 16 | try: 17 | self.teardown_test_environment() 18 | except Exception: 19 | # Silence teardown exceptions if an exception was raised during 20 | # runs to avoid shadowing it. 21 | if not run_failed: 22 | raise 23 | self.time_keeper.print_results() 24 | return self.suite_result(suite, result) 25 | -------------------------------------------------------------------------------- /app/utility/project.py: -------------------------------------------------------------------------------- 1 | import os 2 | from contextlib import contextmanager 3 | 4 | from django.conf import settings 5 | 6 | from .filesystem import FileSystem 7 | 8 | 9 | @contextmanager 10 | def project_dir(type, name=None, base_path=None): 11 | project = ProjectDir(type, name, base_path) 12 | yield project 13 | 14 | 15 | @contextmanager 16 | def project_temp_dir(type, name=None, base_path=None): 17 | project = ProjectDir(type, name, base_path) 18 | try: 19 | yield project 20 | finally: 21 | project.delete() 22 | 23 | 24 | class ProjectDir(FileSystem): 25 | def __init__(self, type, name=None, base_path=None): 26 | self.type = type 27 | self.name = name 28 | 29 | if base_path is None: 30 | base_path = settings.MANAGER.file_path 31 | 32 | path_args = [base_path, self.type] 33 | if self.name: 34 | path_args.append(self.name) 35 | 36 | super().__init__(os.path.join(*path_args)) 37 | -------------------------------------------------------------------------------- /app/utility/request.py: -------------------------------------------------------------------------------- 1 | import ssl 2 | 3 | import requests 4 | import urllib3 5 | 6 | 7 | class CustomHttpAdapter(requests.adapters.HTTPAdapter): 8 | def __init__(self, ssl_context=None, **kwargs): 9 | self.ssl_context = ssl_context 10 | super().__init__(**kwargs) 11 | 12 | def init_poolmanager(self, connections, maxsize, block=False): 13 | self.poolmanager = urllib3.poolmanager.PoolManager( 14 | num_pools=connections, maxsize=maxsize, block=block, ssl_context=self.ssl_context 15 | ) 16 | 17 | 18 | def request_legacy_session(): 19 | ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) 20 | ctx.options |= 0x4 # OP_LEGACY_SERVER_CONNECT 21 | session = requests.session() 22 | session.mount("https://", CustomHttpAdapter(ctx)) 23 | return session 24 | -------------------------------------------------------------------------------- /app/zimagi-cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | from systems.commands import cli 7 | 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.full") 9 | cli.execute(sys.argv) 10 | -------------------------------------------------------------------------------- /app/zimagi-client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | from systems.client.cli import client 7 | 8 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.client") 9 | client.execute(sys.argv) 10 | -------------------------------------------------------------------------------- /app/zimagi-install.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | 4 | if __name__ == "__main__": 5 | from systems.commands import cli 6 | 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.install") 8 | cli.install() 9 | -------------------------------------------------------------------------------- /compose.db.yaml: -------------------------------------------------------------------------------- 1 | include: 2 | - ./compose.network.yaml 3 | 4 | volumes: 5 | postgres_data: {} 6 | redis_data: {} 7 | qdrant_data: {} 8 | 9 | services: 10 | postgresql: 11 | image: 'postgres:17.4' 12 | command: postgres -c 'max_connections=100' 13 | networks: 14 | - zimagi-net 15 | volumes: 16 | - 'postgres_data:/var/lib/postgresql/data' 17 | env_file: 18 | - ./env/public.${ZIMAGI_PROFILE} 19 | - ./env/secret 20 | ports: 21 | - '5432:5432' 22 | 23 | redis: 24 | image: 'redis:7.4.3' 25 | command: /bin/sh -c "redis-server --requirepass $$ZIMAGI_REDIS_PASSWORD" 26 | networks: 27 | - zimagi-net 28 | volumes: 29 | - 'redis_data:/data' 30 | env_file: 31 | - ./env/public.${ZIMAGI_PROFILE} 32 | - ./env/secret 33 | ports: 34 | - '6379:6379' 35 | -------------------------------------------------------------------------------- /compose.network.yaml: -------------------------------------------------------------------------------- 1 | networks: 2 | zimagi-net: 3 | name: zimagi-net 4 | driver: bridge 5 | -------------------------------------------------------------------------------- /docker/build_client_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | 5 | export __zimagi_docker_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | export __zimagi_dir="$(dirname "${__zimagi_docker_dir}")" 7 | 8 | export ZIMAGI_ENVIRONMENT="${1:-local}" 9 | #------------------------------------------------------------------------------- 10 | 11 | ZIMAGI_VERSION=$(cat "${__zimagi_dir}/app/VERSION") 12 | ZIMAGI_BUILD_IMAGE="zimagi/cli:${ZIMAGI_VERSION}" 13 | 14 | echo "Building Zimagi Client Docker image: ${ZIMAGI_BUILD_IMAGE}" 15 | docker build \ 16 | --file "${__zimagi_docker_dir}/Dockerfile.cli" \ 17 | --tag "$ZIMAGI_BUILD_IMAGE" \ 18 | --build-arg ZIMAGI_ENVIRONMENT="$ZIMAGI_ENVIRONMENT" \ 19 | "${__zimagi_dir}" 20 | -------------------------------------------------------------------------------- /docker/packages.core.txt: -------------------------------------------------------------------------------- 1 | # Internet utilities 2 | curl 3 | wget 4 | 5 | # Package management utilities 6 | apt-utils 7 | software-properties-common 8 | netbase 9 | lsb-release 10 | 11 | # Security packages 12 | apt-transport-https 13 | ca-certificates 14 | gnupg2 15 | sudo 16 | 17 | # Dependency building 18 | gcc 19 | g++ 20 | make 21 | cmake 22 | libssl-dev 23 | unzip 24 | 25 | # Language 26 | python3-venv 27 | python3-dev 28 | python3-pip 29 | -------------------------------------------------------------------------------- /docker/packages.server.txt: -------------------------------------------------------------------------------- 1 | # Utilities 2 | git 3 | openssh-client 4 | sshpass 5 | 6 | # Database support 7 | libpq-dev 8 | postgresql-client 9 | 10 | # Service clients 11 | docker-ce 12 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Minimal makefile for Sphinx documentation 3 | # 4 | SPHINXOPTS = 5 | SPHINXBUILD = sphinx-build 6 | SPHINXPROJ = Zimagi 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | help: 11 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 12 | 13 | .PHONY: help Makefile 14 | 15 | %: Makefile 16 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 17 | -------------------------------------------------------------------------------- /docs/_static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/_static/favicon.ico -------------------------------------------------------------------------------- /docs/_static/images/zimagi-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/_static/images/zimagi-architecture.png -------------------------------------------------------------------------------- /docs/_static/images/zimagi-components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/_static/images/zimagi-components.png -------------------------------------------------------------------------------- /docs/_static/images/zimagi-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/_static/images/zimagi-flow.png -------------------------------------------------------------------------------- /docs/_static/images/zimagi-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/_static/images/zimagi-logo.png -------------------------------------------------------------------------------- /docs/_static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/_static/logo.png -------------------------------------------------------------------------------- /docs/archive/prerequisites_and_environment_setup/prereqs.rst: -------------------------------------------------------------------------------- 1 | 2 | Zimagi Prerequisites 3 | ******************** 4 | -------------------------------------------------------------------------------- /docs/archive/reference_and_core_resources/core_resources.rst: -------------------------------------------------------------------------------- 1 | 2 | Zimagi Core Resources 3 | ********************* 4 | -------------------------------------------------------------------------------- /docs/archive/tutorials/tutorials.rst: -------------------------------------------------------------------------------- 1 | 2 | Zimagi Tutorials 3 | **************** 4 | -------------------------------------------------------------------------------- /docs/archive/use_cases/use_cases.rst: -------------------------------------------------------------------------------- 1 | 2 | Zimagi Use Cases 3 | **************** 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/commands/data.rst: -------------------------------------------------------------------------------- 1 | ############################### 2 | Zimagi Data Management Commands 3 | ############################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/commands/environment.rst: -------------------------------------------------------------------------------- 1 | ##################################################### 2 | Zimagi Environment and Connection Management Commands 3 | ##################################################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/commands/help.rst: -------------------------------------------------------------------------------- 1 | #################### 2 | Zimagi Help Commands 3 | #################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/commands/orchestration.rst: -------------------------------------------------------------------------------- 1 | ########################################## 2 | Zimagi Orchestration and Workflow Commands 3 | ########################################## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/commands/readme.rst: -------------------------------------------------------------------------------- 1 | ############### 2 | Zimagi Commands 3 | ############### 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Commands 8 | 9 | help 10 | environment 11 | service 12 | data 13 | orchestration 14 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/commands/service.rst: -------------------------------------------------------------------------------- 1 | ####################### 2 | Zimagi Service Commands 3 | ####################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/concepts.rst: -------------------------------------------------------------------------------- 1 | ############### 2 | Zimagi Concepts 3 | ############### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/configuration.rst: -------------------------------------------------------------------------------- 1 | ######################### 2 | Zimagi Configuration Data 3 | ######################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/environment.rst: -------------------------------------------------------------------------------- 1 | ####################### 2 | Zimagi Environment Data 3 | ####################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/group.rst: -------------------------------------------------------------------------------- 1 | ################# 2 | Zimagi Group Data 3 | ################# 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/host.rst: -------------------------------------------------------------------------------- 1 | ############################ 2 | Zimagi Environment Host Data 3 | ############################ 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/log.rst: -------------------------------------------------------------------------------- 1 | ############### 2 | Zimagi Log Data 3 | ############### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/module.rst: -------------------------------------------------------------------------------- 1 | ################## 2 | Zimagi Module Data 3 | ################## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/notification.rst: -------------------------------------------------------------------------------- 1 | ######################## 2 | Zimagi Notification Data 3 | ######################## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/readme.rst: -------------------------------------------------------------------------------- 1 | ################# 2 | Zimagi Data Types 3 | ################# 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Data Types 8 | 9 | environment 10 | host 11 | user 12 | group 13 | log 14 | module 15 | state 16 | configuration 17 | schedule 18 | notification 19 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/schedule.rst: -------------------------------------------------------------------------------- 1 | #################### 2 | Zimagi Schedule Data 3 | #################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/state.rst: -------------------------------------------------------------------------------- 1 | ################# 2 | Zimagi State Data 3 | ################# 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/data/user.rst: -------------------------------------------------------------------------------- 1 | ################ 2 | Zimagi User Data 3 | ################ 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/extending.rst: -------------------------------------------------------------------------------- 1 | ################ 2 | Extending Zimagi 3 | ################ 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/interface.rst: -------------------------------------------------------------------------------- 1 | ################ 2 | Zimagi Interface 3 | ################ 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/orchestration/config.rst: -------------------------------------------------------------------------------- 1 | ############################## 2 | Zimagi Configuration Component 3 | ############################## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/orchestration/destroy.rst: -------------------------------------------------------------------------------- 1 | ######################### 2 | Zimagi Destroy Components 3 | ######################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/orchestration/group.rst: -------------------------------------------------------------------------------- 1 | ###################### 2 | Zimagi Group Component 3 | ###################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/orchestration/profile.rst: -------------------------------------------------------------------------------- 1 | ######################## 2 | Zimagi Profile Component 3 | ######################## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/orchestration/readme.rst: -------------------------------------------------------------------------------- 1 | ############################### 2 | Zimagi Orchestration Components 3 | ############################### 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Orchestration Components 8 | 9 | config 10 | group 11 | run 12 | destroy 13 | profile 14 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/orchestration/run.rst: -------------------------------------------------------------------------------- 1 | ##################### 2 | Zimagi Run Components 3 | ##################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/plugins/module.rst: -------------------------------------------------------------------------------- 1 | #################### 2 | Zimagi Module Plugin 3 | #################### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/plugins/readme.rst: -------------------------------------------------------------------------------- 1 | ############## 2 | Zimagi Plugins 3 | ############## 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Plugins 8 | 9 | module 10 | task 11 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/plugins/task.rst: -------------------------------------------------------------------------------- 1 | ################## 2 | Zimagi Task Plugin 3 | ################## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/readme.rst: -------------------------------------------------------------------------------- 1 | ############################## 2 | Zimagi Design and Architecture 3 | ############################## 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Design and Architecture 8 | 9 | concepts 10 | technologies 11 | systems 12 | interface 13 | services/readme 14 | data/readme 15 | plugins/readme 16 | commands/readme 17 | orchestration/readme 18 | extending 19 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/services/cli.rst: -------------------------------------------------------------------------------- 1 | ############################# 2 | Zimagi Command Line Interface 3 | ############################# 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/services/command.rst: -------------------------------------------------------------------------------- 1 | ################## 2 | Zimagi Command API 3 | ################## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/services/data.rst: -------------------------------------------------------------------------------- 1 | ############### 2 | Zimagi Data API 3 | ############### 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/services/readme.rst: -------------------------------------------------------------------------------- 1 | ############### 2 | Zimagi Services 3 | ############### 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Services 8 | 9 | cli 10 | command 11 | data 12 | scheduler 13 | worker 14 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/services/scheduler.rst: -------------------------------------------------------------------------------- 1 | ################ 2 | Zimagi Scheduler 3 | ################ 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/services/worker.rst: -------------------------------------------------------------------------------- 1 | ############# 2 | Zimagi Worker 3 | ############# 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/systems.rst: -------------------------------------------------------------------------------- 1 | ############## 2 | Zimagi Systems 3 | ############## 4 | -------------------------------------------------------------------------------- /docs/concepts/design_architecture/technologies.rst: -------------------------------------------------------------------------------- 1 | ################### 2 | Zimagi Technologies 3 | ################### 4 | -------------------------------------------------------------------------------- /docs/concepts/specs/kix.6og5zbx0ctt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/concepts/specs/kix.6og5zbx0ctt.png -------------------------------------------------------------------------------- /docs/concepts/specs/kix.7ft3utqvnd8j.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/concepts/specs/kix.7ft3utqvnd8j.png -------------------------------------------------------------------------------- /docs/concepts/specs/kix.fws49h5c5sbl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/concepts/specs/kix.fws49h5c5sbl.png -------------------------------------------------------------------------------- /docs/concepts/specs/kix.tpls2u9qapt1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/concepts/specs/kix.tpls2u9qapt1.png -------------------------------------------------------------------------------- /docs/concepts/specs/kix.u09pexpvp3a8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/concepts/specs/kix.u09pexpvp3a8.png -------------------------------------------------------------------------------- /docs/concepts/specs/kix.vsnbgztpsc6x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/concepts/specs/kix.vsnbgztpsc6x.png -------------------------------------------------------------------------------- /docs/concepts/specs/kix.yk9yetlaxfoj.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/docs/concepts/specs/kix.yk9yetlaxfoj.png -------------------------------------------------------------------------------- /docs/creating_a_module/index.rst: -------------------------------------------------------------------------------- 1 | ########################## 2 | Developing a Zimagi Module 3 | ########################## 4 | 5 | These lessons walk you through creating a sample module. 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | :caption: Documentation 10 | 11 | create_module 12 | initializing 13 | module_functionality 14 | data_model 15 | data_import 16 | commands 17 | command_line_comparison 18 | 19 | * :ref:`genindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /docs/overview/contributing.rst: -------------------------------------------------------------------------------- 1 | ###################### 2 | Contributing to Zimagi 3 | ###################### 4 | 5 | Please feel free to fork and PR the Zimagi core project at 6 | https://github.com/zimagi/zimagi, or indeed any of the components within the 7 | Zimagi GitHub organization. 8 | 9 | If you would like to become a direct contributor and project member, please 10 | contact adrian.webb@zimagi.com. 11 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx==8.2.3 2 | sphinx_rtd_theme==3.0.2 3 | -------------------------------------------------------------------------------- /env/public.api: -------------------------------------------------------------------------------- 1 | # Common 2 | 3 | POSTGRES_DB=zimagi 4 | ZIMAGI_POSTGRES_DB=zimagi 5 | ZIMAGI_DB_MAX_CONNECTIONS=100 6 | 7 | ZIMAGI_PARALLEL=true 8 | ZIMAGI_THREAD_COUNT=20 9 | ZIMAGI_MUTEX_TTL_SECONDS=120 10 | 11 | ZIMAGI_DEBUG=true 12 | ZIMAGI_DEBUG_COMMAND_PROFILES=false 13 | ZIMAGI_LOG_LEVEL=warning 14 | ZIMAGI_QUEUE_COMMANDS=false 15 | 16 | ZIMAGI_SERVER_WORKERS=1 17 | ZIMAGI_SERVER_TIMEOUT=3600 18 | 19 | # Command API 20 | 21 | ZIMAGI_ENCRYPT_COMMAND_API=false 22 | 23 | # Data API 24 | 25 | ZIMAGI_ENCRYPT_DATA_API=false 26 | ZIMAGI_DISABLE_PAGE_CACHE=true 27 | ZIMAGI_PAGE_CACHE_SECONDS=60 28 | ZIMAGI_REST_PAGE_COUNT=50 29 | -------------------------------------------------------------------------------- /env/public.api.encrypted: -------------------------------------------------------------------------------- 1 | # Common 2 | 3 | POSTGRES_DB=zimagi 4 | ZIMAGI_POSTGRES_DB=zimagi 5 | ZIMAGI_DB_MAX_CONNECTIONS=100 6 | 7 | ZIMAGI_PARALLEL=true 8 | ZIMAGI_THREAD_COUNT=20 9 | ZIMAGI_MUTEX_TTL_SECONDS=120 10 | 11 | ZIMAGI_DEBUG=true 12 | ZIMAGI_DEBUG_COMMAND_PROFILES=false 13 | ZIMAGI_LOG_LEVEL=warning 14 | ZIMAGI_QUEUE_COMMANDS=false 15 | 16 | ZIMAGI_SERVER_WORKERS=1 17 | ZIMAGI_SERVER_TIMEOUT=3600 18 | 19 | # Command API 20 | 21 | ZIMAGI_ENCRYPT_COMMAND_API=true 22 | 23 | # Data API 24 | 25 | ZIMAGI_ENCRYPT_DATA_API=true 26 | ZIMAGI_DISABLE_PAGE_CACHE=true 27 | ZIMAGI_PAGE_CACHE_SECONDS=60 28 | ZIMAGI_REST_PAGE_COUNT=50 29 | -------------------------------------------------------------------------------- /env/public.default: -------------------------------------------------------------------------------- 1 | # Common 2 | 3 | POSTGRES_DB=zimagi 4 | ZIMAGI_POSTGRES_DB=zimagi 5 | ZIMAGI_DB_MAX_CONNECTIONS=100 6 | 7 | ZIMAGI_PARALLEL=true 8 | ZIMAGI_THREAD_COUNT=20 9 | ZIMAGI_MUTEX_TTL_SECONDS=120 10 | 11 | ZIMAGI_DEBUG=true 12 | ZIMAGI_DEBUG_COMMAND_PROFILES=false 13 | ZIMAGI_LOG_LEVEL=warning 14 | ZIMAGI_QUEUE_COMMANDS=true 15 | 16 | ZIMAGI_SERVER_WORKERS=1 17 | ZIMAGI_SERVER_TIMEOUT=3600 18 | 19 | # Command API 20 | 21 | ZIMAGI_ENCRYPT_COMMAND_API=false 22 | 23 | # Data API 24 | 25 | ZIMAGI_ENCRYPT_DATA_API=false 26 | ZIMAGI_DISABLE_PAGE_CACHE=true 27 | ZIMAGI_PAGE_CACHE_SECONDS=60 28 | ZIMAGI_REST_PAGE_COUNT=50 29 | 30 | # Flower 31 | 32 | FLOWER_UNAUTHENTICATED_API=true 33 | -------------------------------------------------------------------------------- /env/secret.example: -------------------------------------------------------------------------------- 1 | ZIMAGI_APP_NAME=zimagi 2 | 3 | ZIMAGI_SECRET_KEY=20181105 4 | ZIMAGI_ADMIN_USER=admin 5 | ZIMAGI_DEFAULT_ADMIN_TOKEN=999999999 6 | 7 | POSTGRES_USER=postgres 8 | POSTGRES_PASSWORD=999999999 9 | ZIMAGI_POSTGRES_USER=postgres 10 | ZIMAGI_POSTGRES_PASSWORD=999999999 11 | 12 | ZIMAGI_REDIS_PASSWORD=999999999 13 | 14 | ZIMAGI_EMAIL_HOST= 15 | ZIMAGI_EMAIL_HOST_USER= 16 | ZIMAGI_EMAIL_HOST_PASSWORD= 17 | ZIMAGI_EMAIL_PORT=587 18 | ZIMAGI_EMAIL_USE_TLS=true 19 | ZIMAGI_EMAIL_SUBJECT_PREFIX=' ' 20 | 21 | ZIMAGI_GITHUB_ORG= 22 | ZIMAGI_GITHUB_TOKEN= 23 | 24 | ZIMAGI_CLIENT_CACHE_LIFETIME=60 25 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "zimagi", 3 | "lockfileVersion": 3, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "name": "zimagi", 8 | "dependencies": { 9 | "prettier": "^3.5.3" 10 | }, 11 | "engines": { 12 | "node": "^18.17.1 || ^20.3.0 || >= 21.0.0" 13 | } 14 | }, 15 | "node_modules/prettier": { 16 | "version": "3.5.3", 17 | "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", 18 | "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", 19 | "bin": { 20 | "prettier": "bin/prettier.cjs" 21 | }, 22 | "engines": { 23 | "node": ">=14" 24 | }, 25 | "funding": { 26 | "url": "https://github.com/prettier/prettier?sponsor=1" 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "zimagi", 3 | "engines": { 4 | "node": "^18.17.1 || ^20.3.0 || >= 21.0.0" 5 | }, 6 | "scripts": { 7 | "check:prettier": "prettier --check .", 8 | "fix:prettier": "prettier -w ." 9 | }, 10 | "dependencies": { 11 | "prettier": "^3.5.3" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /package/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zimagi/core/c1d91311d3c758aaa35c10d378741f11f63de1ea/package/README.md -------------------------------------------------------------------------------- /package/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | 5 | SCRIPT_DIR="$(cd "$(dirname "$([ `readlink "$0"` ] && echo "`readlink "$0"`" || echo "$0")")"; pwd -P)" 6 | cd "$SCRIPT_DIR" 7 | #------------------------------------------------------------------------------- 8 | 9 | if [ -z "$PKG_PIP_TOKEN" ] 10 | then 11 | echo "PKG_PIP_TOKEN environment variable must be defined to deploy application" 12 | exit 1 13 | fi 14 | 15 | echo "Creating PyPi configuration" 16 | if [ ! -f ~/.pypirc ] 17 | then 18 | echo " 19 | [distutils] 20 | index-servers = pypi 21 | 22 | [pypi] 23 | username: __token__ 24 | password: $PKG_PIP_TOKEN 25 | " > ~/.pypirc 26 | fi 27 | chmod 600 ~/.pypirc 28 | 29 | echo "Installing pip build tools" 30 | python3 -m pip install --no-cache-dir --upgrade setuptools wheel twine 31 | 32 | echo "Building pip distribution" 33 | python3 setup.py sdist bdist_wheel --universal --owner=root --group=root 34 | 35 | echo "Distributing to PyPi repository" 36 | python3 -m twine upload --verbose dist/* 37 | -------------------------------------------------------------------------------- /package/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.* 2 | pycryptodome==3.23.* 3 | terminaltables==3.1.* 4 | numpy==2.2.* 5 | pandas==2.2.* 6 | tblib==3.1.* 7 | -------------------------------------------------------------------------------- /package/zimagi/__init__.py: -------------------------------------------------------------------------------- 1 | from .command import Client as CommandClient # noqa: F401 2 | from .command.messages import * # noqa: F401, F403 3 | from .command.response import * # noqa: F401, F403 4 | from .data import Client as DataClient # noqa: F401 5 | from .datetime import Time # noqa: F401 6 | from .exceptions import * # noqa: F401, F403 7 | 8 | time = Time("UTC") 9 | -------------------------------------------------------------------------------- /package/zimagi/auth.py: -------------------------------------------------------------------------------- 1 | from requests import auth 2 | 3 | 4 | class ClientTokenAuthentication(auth.AuthBase): 5 | def __init__(self, user, token, client=None): 6 | self.client = client 7 | self.user = user 8 | self.token = token 9 | self.encrypted = False 10 | 11 | def __call__(self, request): 12 | if not self.encrypted and self.client.cipher: 13 | self.token = self.client.cipher.encrypt(self.token).decode("utf-8") 14 | self.encrypted = True 15 | 16 | request.headers["Authorization"] = f"Token {self.user} {self.token}" 17 | return request 18 | -------------------------------------------------------------------------------- /package/zimagi/codecs.py: -------------------------------------------------------------------------------- 1 | from . import collection, exceptions, utility 2 | 3 | 4 | class JSONCodec: 5 | media_types = ["application/json"] 6 | 7 | def decode(self, bytestring, **options): 8 | def convert(data): 9 | if isinstance(data, dict): 10 | data = collection.RecursiveCollection(data) 11 | elif isinstance(data, (list, tuple)): 12 | for index, value in enumerate(data): 13 | data[index] = convert(value) 14 | return data 15 | 16 | try: 17 | return convert(utility.load_json(bytestring.decode("utf-8"))) 18 | 19 | except ValueError as error: 20 | raise exceptions.ParseError(f"Malformed JSON: {error}") 21 | -------------------------------------------------------------------------------- /package/zimagi/command/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import Client # noqa: F401 2 | from .messages import * # noqa: F401, F403 3 | from .response import * # noqa: F401, F403 4 | -------------------------------------------------------------------------------- /package/zimagi/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import Client # noqa: F401 2 | -------------------------------------------------------------------------------- /package/zimagi/exceptions.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import traceback 3 | 4 | 5 | def format_exception_info(): 6 | exc_type, exc_value, exc_tb = sys.exc_info() 7 | return traceback.format_exception(exc_type, exc_value, exc_tb) 8 | 9 | 10 | class ClientError(Exception): 11 | pass 12 | 13 | 14 | class ConnectionError(ClientError): 15 | pass 16 | 17 | 18 | class ParseError(ClientError): 19 | pass 20 | 21 | 22 | class ResponseError(ClientError): 23 | def __init__(self, message, code=None, result=None): 24 | super().__init__(message) 25 | self.code = code 26 | self.result = result or message 27 | -------------------------------------------------------------------------------- /package/zimagi/settings.py: -------------------------------------------------------------------------------- 1 | CACHE_DIR = None 2 | CACHE_LIFETIME = 86400 # 24 hours 3 | 4 | DEFAULT_HOST = "localhost" 5 | DEFAULT_COMMAND_PORT = 5123 6 | DEFAULT_DATA_PORT = 5323 7 | DEFAULT_VERIFY_CERT = False 8 | 9 | DEFAULT_USER = "admin" 10 | DEFAULT_TOKEN = "uy5c8xiahf93j2pl8s00e6nb32h87dn3" 11 | 12 | CONNECTION_RETRIES = 20 13 | CONNECTION_RETRY_WAIT = 3 14 | 15 | COMMAND_RAISE_ERROR = False 16 | 17 | PARALLEL = True 18 | THREAD_COUNT = 5 19 | 20 | LOG_LEVEL = "WARNING" 21 | -------------------------------------------------------------------------------- /reactor/build/client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | 5 | export __script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | source "${__script_dir}/shared.sh" 7 | #------------------------------------------------------------------------------- 8 | 9 | export ZIMAGI_DOCKER_RUNTIME="standard" 10 | 11 | client_build_args 12 | -------------------------------------------------------------------------------- /reactor/build/server.nvidia.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | 5 | export __script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | source "${__script_dir}/shared.sh" 7 | #------------------------------------------------------------------------------- 8 | 9 | export ZIMAGI_DOCKER_RUNTIME="nvidia" 10 | 11 | server_build_args 12 | -------------------------------------------------------------------------------- /reactor/build/server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #------------------------------------------------------------------------------- 3 | set -e 4 | 5 | export __script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 6 | source "${__script_dir}/shared.sh" 7 | #------------------------------------------------------------------------------- 8 | 9 | export ZIMAGI_DOCKER_RUNTIME="standard" 10 | 11 | server_build_args 12 | -------------------------------------------------------------------------------- /reactor/build/shared.sh: -------------------------------------------------------------------------------- 1 | 2 | function server_build_args() { 3 | zimagi_environment 4 | 5 | export DOCKER_BUILD_VARS=( 6 | "ZIMAGI_PARENT_IMAGE" 7 | "ZIMAGI_ENVIRONMENT=${__environment}" 8 | "ZIMAGI_USER_UID=$(id -u)" 9 | ) 10 | } 11 | 12 | function client_build_args() { 13 | zimagi_environment 14 | 15 | export DOCKER_BUILD_VARS=( 16 | "ZIMAGI_PARENT_IMAGE" 17 | "ZIMAGI_ENVIRONMENT=${__environment}" 18 | "ZIMAGI_USER_UID=$(id -u)" 19 | ) 20 | } 21 | -------------------------------------------------------------------------------- /reactor/commands/zimagi.sh: -------------------------------------------------------------------------------- 1 | # 2 | #========================================================================================= 3 | # Command 4 | # 5 | 6 | function zimagi_description () { 7 | render "Execute a Zimagi operation within the reactor environment context" 8 | export PASSTHROUGH="1" 9 | } 10 | 11 | function zimagi_command () { 12 | zimagi_environment 13 | 14 | ZIMAGI_ARGS=( 15 | "--rm" 16 | "--interactive" 17 | "--tty" 18 | "--network" "host" 19 | "--user" "$(id -u):zimagi" 20 | "--volume" "${__zimagi_app_dir}:/usr/local/share/zimagi" 21 | "--volume" "${__zimagi_package_dir}:/usr/local/share/zimagi-client" 22 | "--volume" "${__zimagi_data_dir}:/var/local/zimagi" 23 | ) 24 | while IFS= read -r variable; do 25 | ZIMAGI_ARGS=("${ZIMAGI_ARGS[@]}" "--env" "$variable") 26 | done <<< "$(env | grep -o "ZIMAGI_[_A-Z0-9]*")" 27 | 28 | ZIMAGI_ARGS=("${ZIMAGI_ARGS[@]}" "zimagi_client:dev" "${@}") 29 | 30 | docker run "${ZIMAGI_ARGS[@]}" 31 | } 32 | -------------------------------------------------------------------------------- /reactor/hooks.sh: -------------------------------------------------------------------------------- 1 | # 2 | #========================================================================================= 3 | # Project Hooks 4 | # 5 | 6 | function hook_modified () { 7 | for file in "${__zimagi_module_dir}"/*/*; do 8 | check_git_status "zimagi module" "$(basename "$file")" "$file" 9 | done 10 | } 11 | 12 | function hook_update () { 13 | info "Initializing Zimagi CLI ..." 14 | run_subcommand zimagi info 15 | run_subcommand zimagi host save "${__environment}" \ 16 | host="cmd.${ZIMAGI_DOMAIN}" \ 17 | command_port="443" \ 18 | user="${ZIMAGI_ADMIN_USER}" \ 19 | token="${ZIMAGI_DEFAULT_ADMIN_TOKEN}" \ 20 | encryption_key="${ZIMAGI_ADMIN_API_KEY}" 21 | } 22 | 23 | function hook_update_host () { 24 | info "Setting Zimagi CLI default host ${__environment} ..." 25 | run_subcommand zimagi config save option_platform_host "${__environment}" --local 26 | } 27 | -------------------------------------------------------------------------------- /reactor/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Development dependencies 3 | # 4 | 5 | # Code quality 6 | # ------------------------------------------------------------------------------ 7 | flake8==7.1.1 # https://github.com/PyCQA/flake8 8 | flake8-isort==6.1.1 # https://github.com/gforcada/flake8-isort 9 | black==24.10.0 # https://github.com/psf/black 10 | djlint==1.36.4 # https://github.com/Riverside-Healthcare/djLint 11 | pylint-django==2.6.1 # https://github.com/PyCQA/pylint-django 12 | pylint-celery==0.3 # https://github.com/PyCQA/pylint-celery 13 | pre-commit==4.0.1 # https://github.com/pre-commit/pre-commit 14 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # flake8 and pycodestyle don't support pyproject.toml 2 | # https://github.com/PyCQA/flake8/issues/234 3 | # https://github.com/PyCQA/pycodestyle/issues/813 4 | [flake8] 5 | max-line-length = 125 6 | exclude = .tox,.git,*/migrations/* 7 | extend-ignore = E203,F841 8 | 9 | [pycodestyle] 10 | max-line-length = 125 11 | exclude = .tox,.git,*/migrations/* 12 | --------------------------------------------------------------------------------