├── shakenfist ├── __init__.py ├── util │ ├── __init__.py │ ├── callstack.py │ ├── json.py │ └── access_tokens.py ├── client │ ├── __init__.py │ └── backup.py ├── protos │ ├── __init__.py │ ├── agent_pb2_grpc.py │ ├── common_pb2_grpc.py │ ├── nodelock_pb2_grpc.py │ ├── privexec_pb2_grpc.py │ ├── common_pb2.py │ ├── event_pb2.py │ └── nodelock_pb2.pyi ├── external_api │ ├── __init__.py │ ├── clusteroperation.py │ ├── util.py │ ├── snapshot.py │ └── admin.py ├── tests │ ├── schema │ │ ├── __init__.py │ │ └── operations │ │ │ ├── __init__.py │ │ │ └── test_baseclusteroperation.py │ ├── external_api │ │ └── __init__.py │ ├── files │ │ ├── dhcphosts.tmpl │ │ ├── dnshosts.tmpl │ │ ├── qemu-img-info │ │ ├── dhcp.tmpl │ │ ├── cirros-MD5SUMS-0.3.4 │ │ ├── ubuntu-MD5SUMS-groovy │ │ └── ubuntu-MD5SUMS-bionic │ ├── __init__.py │ ├── base.py │ ├── test_util_general.py │ ├── test_images.py │ ├── test_config.py │ ├── test_networkinterface.py │ └── test_baseobject.py ├── deploy │ ├── requirements.txt │ ├── shakenfist_ci │ │ ├── __init__.py │ │ ├── guest_ci_tests │ │ │ ├── __init__.py │ │ │ ├── files │ │ │ │ └── fibonacci.py │ │ │ ├── test_ubuntu.py │ │ │ ├── test_disks.py │ │ │ ├── test_multiple_nics.py │ │ │ └── test_boot.py │ │ ├── smoke_ci_tests │ │ │ ├── __init__.py │ │ │ ├── files │ │ │ │ └── fibonacci.py │ │ │ ├── test_disk_specs.py │ │ │ ├── test_auth.py │ │ │ └── test_events.py │ │ ├── cluster_ci_tests │ │ │ ├── __init__.py │ │ │ ├── files │ │ │ │ ├── console_scribbler_userdata │ │ │ │ └── writedata_userdata │ │ │ ├── test_api.py │ │ │ ├── test_uploads.py │ │ │ ├── test_nodes.py │ │ │ ├── test_auth.py │ │ │ ├── test_disk_specs.py │ │ │ ├── test_system_namespace.py │ │ │ ├── test_events.py │ │ │ ├── test_object_names.py │ │ │ ├── test_console_log.py │ │ │ └── test_upgrades.py │ │ └── AUTHORS │ ├── ansible │ │ ├── roles │ │ │ ├── database │ │ │ │ ├── meta │ │ │ │ │ └── main.yml │ │ │ │ └── tasks │ │ │ │ │ ├── main.yml │ │ │ │ │ └── bootstrap.yml │ │ │ ├── mariadb │ │ │ │ ├── meta │ │ │ │ │ └── main.yml │ │ │ │ ├── handlers │ │ │ │ │ └── main.yml │ │ │ │ └── tasks │ │ │ │ │ ├── main.yml │ │ │ │ │ └── bootstrap.yml │ │ │ ├── base │ │ │ │ ├── meta │ │ │ │ │ └── main.yml │ │ │ │ ├── templates │ │ │ │ │ ├── 00-sf-disable-dnsmasq.tmpl │ │ │ │ │ ├── 10-sf-ipv6.tmpl │ │ │ │ │ ├── sleep.tmpl │ │ │ │ │ ├── rsyslog-client-01-sf.conf │ │ │ │ │ ├── etc_hosts.tmpl │ │ │ │ │ ├── journald.tmpl │ │ │ │ │ └── config │ │ │ │ ├── tasks │ │ │ │ │ ├── main.yml │ │ │ │ │ ├── syslog.yml │ │ │ │ │ └── create_admin_namespace.yml │ │ │ │ └── defaults │ │ │ │ │ └── main.yml │ │ │ ├── network │ │ │ │ ├── meta │ │ │ │ │ └── main.yml │ │ │ │ ├── defaults │ │ │ │ │ └── main.yml │ │ │ │ └── tasks │ │ │ │ │ ├── main.yml │ │ │ │ │ ├── validate_mtus.yml │ │ │ │ │ └── bootstrap.yml │ │ │ ├── primary │ │ │ │ ├── meta │ │ │ │ │ └── main.yml │ │ │ │ ├── defaults │ │ │ │ │ └── main.yml │ │ │ │ ├── templates │ │ │ │ │ ├── ds_prometheus.yml │ │ │ │ │ ├── rsyslog-server-01-sf.conf │ │ │ │ │ ├── inventory.yaml │ │ │ │ │ └── etc_prometheus.yml │ │ │ │ ├── tasks │ │ │ │ │ ├── main.yml │ │ │ │ │ ├── apache2.yml │ │ │ │ │ ├── bootstrap.yml │ │ │ │ │ ├── etcd_config.yml │ │ │ │ │ └── grafana.yml │ │ │ │ └── files │ │ │ │ │ └── grafana │ │ │ │ │ ├── provisioning │ │ │ │ │ └── dashboards │ │ │ │ │ │ └── dashboards.yaml │ │ │ │ │ └── ldap.toml │ │ │ ├── hypervisor │ │ │ │ ├── defaults │ │ │ │ │ └── main.yml │ │ │ │ ├── meta │ │ │ │ │ └── main.yml │ │ │ │ └── tasks │ │ │ │ │ ├── validate_kvm.yml │ │ │ │ │ └── main.yml │ │ │ ├── pki_internal_ca │ │ │ │ ├── meta │ │ │ │ │ └── main.yml │ │ │ │ ├── templates │ │ │ │ │ └── ca_template.tmpl │ │ │ │ ├── tasks │ │ │ │ │ ├── main.yml │ │ │ │ │ ├── distribute_certificates.yml │ │ │ │ │ ├── bootstrap.yml │ │ │ │ │ └── host_certificate.yml │ │ │ │ └── defaults │ │ │ │ │ └── main.yml │ │ │ └── common │ │ │ │ └── handlers │ │ │ │ └── main.yml │ │ ├── files │ │ │ ├── sf.target │ │ │ ├── shakenfist.json │ │ │ ├── sfrc │ │ │ ├── sf-api.service │ │ │ └── apache-site-primary.conf │ │ ├── hosts │ │ └── tasks │ │ │ ├── build-wheel.yml │ │ │ └── distro-check.yml │ ├── cluster-ci.conf │ ├── guest-ci.conf │ ├── smoke-ci.conf │ ├── templates │ │ ├── dhcphosts.tmpl │ │ ├── dnshosts.tmpl │ │ └── dhcp.tmpl │ ├── install │ ├── ansiblemoduletests.sh │ ├── requirements.yml │ └── ansible_module_ci │ │ ├── 001.yml │ │ └── 002.yml ├── daemons │ ├── database │ │ └── __init__.py │ ├── network │ │ ├── mtus.py │ │ └── stray_nics.py │ ├── sentinel_last │ │ └── main.py │ └── sentinel_first │ │ └── main.py ├── operations │ └── clusteroperationmapping.py ├── schema │ ├── operations │ │ └── baseclusteroperation.py │ ├── external_view.py │ └── object_types.py └── upload.py ├── docs ├── CNAME ├── operator_guide │ ├── networking │ │ ├── sf-multinode-networking.png │ │ ├── sf-single-node-networking-no-instances.png │ │ ├── sf-single-node-networking-unfloat-instance.png │ │ ├── sf-single-node-networking-unfloat-instance-nat.png │ │ ├── sf-single-node-networking-two-instances-one-floating-ip.png │ │ └── sf-single-node-networking-two-instances-one-floating-ip-ingress.png │ ├── threads.md │ ├── locks.md │ ├── power_states.md │ ├── python_versions.md │ ├── authentication.md │ └── artifacts.md ├── developer_guide │ ├── api_reference │ │ ├── blob_checksums.md │ │ ├── upload.md │ │ ├── admin.md │ │ └── clusteroperations.md │ ├── updating_docs.md │ ├── release_process.md │ └── standards.md ├── components │ └── overview.md ├── user_guide │ ├── metadata.md │ ├── objects.md │ ├── authentication.md │ └── events.md ├── community.md └── manifesto.md ├── .stestr.conf ├── network-example.dia ├── network-example.png ├── examples ├── schema-v0_3_3.tgz └── configdrive-sample-ocata-plain.tgz ├── .coveragerc ├── .github └── workflows │ ├── ci-images-failure.md │ ├── scheduled-tests-failure.md │ ├── renovate.yml │ ├── refresh-website.yml │ ├── publish-website.yml │ ├── docs-tests.yml │ ├── codeql-analysis.yml │ ├── code-formatting.yml │ ├── ci-dependencies.yml │ └── pr-re-review.yml ├── README.md ├── AUTHORS ├── GOALS.md ├── DEVELOPER.md ├── protos ├── common.proto ├── nodelock.proto ├── _make_stubs.sh └── event.proto ├── .vscode └── settings.json ├── tools └── flake8wrap.sh ├── renovate.json ├── BRANCHES.rst ├── release.sh ├── tox.ini └── .gitignore /shakenfist/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | shakenfist.com -------------------------------------------------------------------------------- /shakenfist/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/client/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/protos/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/external_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/tests/schema/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/deploy/requirements.txt: -------------------------------------------------------------------------------- 1 | ansible -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/tests/external_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/tests/schema/operations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/guest_ci_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/smoke_ci_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.stestr.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=./shakenfist/tests 3 | top_dir=./ -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/AUTHORS: -------------------------------------------------------------------------------- 1 | Michael Still 2 | -------------------------------------------------------------------------------- /shakenfist/daemons/database/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Michael Still and contributors 2 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/database/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - role: common 3 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/mariadb/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - role: common 3 | -------------------------------------------------------------------------------- /network-example.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/network-example.dia -------------------------------------------------------------------------------- /network-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/network-example.png -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - role: common 3 | 4 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/templates/00-sf-disable-dnsmasq.tmpl: -------------------------------------------------------------------------------- 1 | disable dnsmasq.service -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/network/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - role: common 3 | 4 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - role: common 3 | 4 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/hypervisor/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # bootstrap 2 | template_path: "undefined" -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/hypervisor/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - role: common 3 | 4 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/network/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # bootstrap 2 | template_path: "undefined" -------------------------------------------------------------------------------- /shakenfist/deploy/cluster-ci.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=./shakenfist_ci/cluster_ci_tests 3 | top_dir=. -------------------------------------------------------------------------------- /shakenfist/deploy/guest-ci.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=./shakenfist_ci/guest_ci_tests 3 | top_dir=./ -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/pki_internal_ca/meta/main.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - role: common 3 | 4 | -------------------------------------------------------------------------------- /shakenfist/deploy/smoke-ci.conf: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | test_path=./shakenfist_ci/smoke_ci_tests 3 | top_dir=./ 4 | -------------------------------------------------------------------------------- /examples/schema-v0_3_3.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/examples/schema-v0_3_3.tgz -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/hypervisor/tasks/validate_kvm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check that we can run KVM 3 | shell: kvm-ok -------------------------------------------------------------------------------- /shakenfist/tests/files/dhcphosts.tmpl: -------------------------------------------------------------------------------- 1 | {%- for vm in instances %} 2 | {{vm.macaddr}},{{vm.name}},{{vm.ipv4}} 3 | {%- endfor %} -------------------------------------------------------------------------------- /shakenfist/deploy/templates/dhcphosts.tmpl: -------------------------------------------------------------------------------- 1 | {%- for vm in instances %} 2 | {{vm.macaddr}},{{vm.name}},{{vm.ipv4}} 3 | {%- endfor %} -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = shakenfist 4 | omit = shakenfist/tests/* 5 | 6 | [report] 7 | ignore_errors = True 8 | -------------------------------------------------------------------------------- /examples/configdrive-sample-ocata-plain.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/examples/configdrive-sample-ocata-plain.tgz -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/templates/10-sf-ipv6.tmpl: -------------------------------------------------------------------------------- 1 | net.ipv6.conf.all.disable_ipv6 = 1 2 | net.ipv6.conf.default.disable_ipv6 = 1 -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/files/sf.target: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Shaken Fist 3 | Requires=multi-user.target 4 | After=multi-user.target 5 | AllowIsolate=yes -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/mariadb/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Restart MariaDB 3 | service: 4 | name: mariadb 5 | state: restarted 6 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/templates/sleep.tmpl: -------------------------------------------------------------------------------- 1 | [Sleep] 2 | AllowSuspend=no 3 | AllowHibernation=no 4 | AllowSuspendThenHibernate=no 5 | AllowHybridSleep=no -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/pki_internal_ca/templates/ca_template.tmpl: -------------------------------------------------------------------------------- 1 | cn = Shaken Fist CA for {{deploy_name}} 2 | ca 3 | cert_signing_key 4 | expiration_days = 3650 -------------------------------------------------------------------------------- /docs/operator_guide/networking/sf-multinode-networking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/docs/operator_guide/networking/sf-multinode-networking.png -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/files/shakenfist.json: -------------------------------------------------------------------------------- 1 | { 2 | "namespace": "system", 3 | "key": "{{system_key}}", 4 | "apiurl": "{{hostvars['localhost']['api_url']}}" 5 | } -------------------------------------------------------------------------------- /docs/developer_guide/api_reference/blob_checksums.md: -------------------------------------------------------------------------------- 1 | # Blob Checksums (/blob_checksums/) 2 | 3 | Blob checksums are documented as part of [blobs](/developer_guide/api_reference/blobs). -------------------------------------------------------------------------------- /docs/developer_guide/api_reference/upload.md: -------------------------------------------------------------------------------- 1 | # Upload (/upload) 2 | 3 | Uploads are documented as part of the [artifacts API reference documentation](/developer_guide/api_reference/artifacts/#uploads). -------------------------------------------------------------------------------- /docs/operator_guide/networking/sf-single-node-networking-no-instances.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/docs/operator_guide/networking/sf-single-node-networking-no-instances.png -------------------------------------------------------------------------------- /shakenfist/tests/files/dnshosts.tmpl: -------------------------------------------------------------------------------- 1 | {%- for vm in instances %} 2 | {{vm.ipv4}} {{vm.name}} 3 | {%- endfor %} 4 | 5 | {%- for name, value in hosted_dns.items() %} 6 | {{value}} {{name}} 7 | {%- endfor %} -------------------------------------------------------------------------------- /shakenfist/deploy/templates/dnshosts.tmpl: -------------------------------------------------------------------------------- 1 | {%- for vm in instances %} 2 | {{vm.ipv4}} {{vm.name}} 3 | {%- endfor %} 4 | 5 | {%- for name, value in hosted_dns.items() %} 6 | {{value}} {{name}} 7 | {%- endfor %} -------------------------------------------------------------------------------- /docs/operator_guide/networking/sf-single-node-networking-unfloat-instance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/docs/operator_guide/networking/sf-single-node-networking-unfloat-instance.png -------------------------------------------------------------------------------- /docs/operator_guide/networking/sf-single-node-networking-unfloat-instance-nat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/docs/operator_guide/networking/sf-single-node-networking-unfloat-instance-nat.png -------------------------------------------------------------------------------- /docs/operator_guide/networking/sf-single-node-networking-two-instances-one-floating-ip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/docs/operator_guide/networking/sf-single-node-networking-two-instances-one-floating-ip.png -------------------------------------------------------------------------------- /shakenfist/deploy/install: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # A simple script to extract the installer and run it. 4 | 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 6 | cd ${DIR}/ansible 7 | python3 ./deploy.py $@ 8 | -------------------------------------------------------------------------------- /docs/operator_guide/networking/sf-single-node-networking-two-instances-one-floating-ip-ingress.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shakenfist/shakenfist/HEAD/docs/operator_guide/networking/sf-single-node-networking-two-instances-one-floating-ip-ingress.png -------------------------------------------------------------------------------- /.github/workflows/ci-images-failure.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Scheduled CI image build {{ env.SF_CI_NAME }} failed 3 | labels: ci-failure 4 | --- 5 | A scheduled CI image rebuild {{ env.SF_CI_NAME }} failed. 6 | 7 | The action run is {{ env.SF_ACTION_RUN }} . 8 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/files/console_scribbler_userdata: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | runcmd: 4 | - | 5 | for i in $(seq 200); do 6 | echo "fhdjkhfdsjhgjkdsfhkgsdjfkghjkdfshgjkhsdfjkghjksdfhgjkhsdfjkgjksd" > /dev/console 7 | done 8 | -------------------------------------------------------------------------------- /.github/workflows/scheduled-tests-failure.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Scheduled CI test failure for {{ env.SF_CI_NAME }} 3 | labels: ci-failure 4 | --- 5 | A scheduled CI test for {{ env.SF_BRANCH }} failed test {{ env.SF_CI_NAME }}. 6 | 7 | The action run is {{ env.SF_ACTION_RUN }} . 8 | -------------------------------------------------------------------------------- /shakenfist/tests/files/qemu-img-info: -------------------------------------------------------------------------------- 1 | foo 2 | image: /tmp/foo 3 | file format: qcow2 4 | virtual size: 112M (117440512 bytes) 5 | disk size: 16M 6 | cluster_size: 65536 7 | Format specific information: 8 | compat: 1.1 9 | lazy refcounts: false 10 | refcount bits: 16 11 | corrupt: false -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/guest_ci_tests/files/fibonacci.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # A silly little demo to calculate some Fibonacci sequence numbers. 3 | 4 | if __name__ == '__main__': 5 | f = [0, 1] 6 | 7 | for _ in range(998): 8 | f.append(f[-2] + f[-1]) 9 | 10 | print(f) 11 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/smoke_ci_tests/files/fibonacci.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # A silly little demo to calculate some Fibonacci sequence numbers. 3 | 4 | if __name__ == '__main__': 5 | f = [0, 1] 6 | 7 | for _ in range(998): 8 | f.append(f[-2] + f[-1]) 9 | 10 | print(f) 11 | -------------------------------------------------------------------------------- /shakenfist/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Logging for tests is configured via the SHAKENFIST_LOG_TO_STDOUT environment 2 | # variable in tox.ini. When set to '1', shakenfist_utilities.logs.setup() will 3 | # write to stdout instead of syslog, allowing stestr to capture logs and only 4 | # display them for failing tests. 5 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # grafana 2 | deploy_name: "sf" 3 | system_key: "unknown" 4 | 5 | # etcd_config 6 | etcd_host: "127.0.0.1" 7 | auth_secret: "unknown" 8 | ram_system_reservation: 5 9 | lowest_mtu: 1500 10 | dns_server: "8.8.8.8" 11 | http_proxy: "" 12 | extra_config: "{{ extra_config }}" -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/templates/ds_prometheus.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | orgId: 1 7 | url: http://{{node_mesh_ip}}:9090 8 | isDefault: true 9 | version: 1 10 | editable: false 11 | access: proxy 12 | jsonData: 13 | tlsSkipVerify: true -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/templates/rsyslog-server-01-sf.conf: -------------------------------------------------------------------------------- 1 | # provides UDP syslog reception 2 | module(load="imudp") 3 | input(type="imudp" port="514") 4 | 5 | # provides TCP syslog reception 6 | module(load="imtcp") 7 | input(type="imtcp" port="514") 8 | 9 | # Don't mangle multi-line log messages 10 | $EscapeControlCharactersOnReceive off -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Executing base role action {{ role_action }}." 4 | delegate_to: localhost 5 | run_once: true 6 | 7 | - include_tasks: "{{ role_action }}.yml" 8 | 9 | - debug: 10 | msg: "Completed base role action {{ role_action }}." 11 | delegate_to: localhost 12 | run_once: true -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/templates/rsyslog-client-01-sf.conf: -------------------------------------------------------------------------------- 1 | *.*;*.!=debug action(type="omfwd" target="{{syslog_target}}" port="514" protocol="tcp" 2 | action.resumeRetryCount="100" 3 | queue.type="linkedList" queue.size="10000") 4 | 5 | # Don't mangle multi-line log messages 6 | $EscapeControlCharactersOnReceive off -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/hosts: -------------------------------------------------------------------------------- 1 | # This file is only used when running ansible directly without deploy.py, 2 | # or as a fallback when no topology is provided. 3 | # 4 | # During normal deployment, deploy.py generates /etc/sf/ansible-hosts with 5 | # the full topology including all hosts, groups, and their variables. 6 | 7 | [local] 8 | localhost ansible_connection=local 9 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/network/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Executing network role action {{ role_action }}." 4 | delegate_to: localhost 5 | run_once: true 6 | 7 | - include_tasks: "{{ role_action }}.yml" 8 | 9 | - debug: 10 | msg: "Completed network role action {{ role_action }}." 11 | delegate_to: localhost 12 | run_once: true -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Executing primary role action {{ role_action }}." 4 | delegate_to: localhost 5 | run_once: true 6 | 7 | - include_tasks: "{{ role_action }}.yml" 8 | 9 | - debug: 10 | msg: "Completed primary role action {{ role_action }}." 11 | delegate_to: localhost 12 | run_once: true -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/files/sfrc: -------------------------------------------------------------------------------- 1 | # Command line hinting 2 | eval "$(_SF_CLIENT_COMPLETE=bash_source /srv/shakenfist/venv/bin/sf-client)" 3 | 4 | # Use the v3 etcd API 5 | export ETCDCTL_API=3 6 | 7 | # Client auth 8 | export SHAKENFIST_NAMESPACE="system" 9 | export SHAKENFIST_KEY="{{system_key}}" 10 | export SHAKENFIST_API_URL="{{hostvars['localhost']['api_url']}}" 11 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/database/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Executing database role action {{ role_action }}." 4 | delegate_to: localhost 5 | run_once: true 6 | 7 | - include_tasks: "{{ role_action }}.yml" 8 | 9 | - debug: 10 | msg: "Completed database role action {{ role_action }}." 11 | delegate_to: localhost 12 | run_once: true 13 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/hypervisor/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Executing hypervisor role action {{ role_action }}." 4 | delegate_to: localhost 5 | run_once: true 6 | 7 | - include_tasks: "{{ role_action }}.yml" 8 | 9 | - debug: 10 | msg: "Completed hypervisor role action {{ role_action }}." 11 | delegate_to: localhost 12 | run_once: true -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/mariadb/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Executing mariadb role action {{ role_action }}." 4 | delegate_to: localhost 5 | run_once: true 6 | 7 | - include_tasks: "{{ role_action }}.yml" 8 | 9 | - debug: 10 | msg: "Completed mariadb role action {{ role_action }}." 11 | delegate_to: localhost 12 | run_once: true 13 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Shared handlers 3 | 4 | - name: Reload systemd 5 | become: true 6 | systemd: 7 | daemon_reload: yes 8 | 9 | - name: Restart journald 10 | service: 11 | name: systemd-journald 12 | state: restarted 13 | 14 | - name: Restart logind 15 | service: 16 | name: systemd-logind 17 | state: restarted -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/pki_internal_ca/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Executing pki_internal_ca role action {{ role_action }}." 4 | delegate_to: localhost 5 | run_once: true 6 | 7 | - include_tasks: "{{ role_action }}.yml" 8 | 9 | - debug: 10 | msg: "Completed pki_internal_ca role action {{ role_action }}." 11 | delegate_to: localhost 12 | run_once: true -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/files/grafana/provisioning/dashboards/dashboards.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: "git configured dashboards" 5 | orgId: 1 6 | folder: "git" 7 | folderUid: "" 8 | type: file 9 | disableDeletion: false 10 | editable: true 11 | updateIntervalSeconds: 60 12 | options: 13 | path: /etc/grafana/provisioning/dashboards 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shaken Fist: Opinionated to the point of being impolite 2 | ![Python application](https://github.com/shakenfist/shakenfist/workflows/Python%20application/badge.svg) 3 | 4 | Package version 5 | 6 | 7 | **Documentation:** https://shakenfist.com/ 8 | **Source Code:** https://github.com/shakenfist/shakenfist -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/templates/etc_hosts.tmpl: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost 2 | 3 | # The following lines are desirable for IPv6 capable hosts 4 | ::1 ip6-localhost ip6-loopback 5 | fe00::0 ip6-localnet 6 | ff00::0 ip6-mcastprefix 7 | ff02::1 ip6-allnodes 8 | ff02::2 ip6-allrouters 9 | ff02::3 ip6-allhosts 10 | 11 | {% for svr in groups.allsf %} 12 | {{ hostvars[svr]['node_mesh_ip'] }} {{svr}} 13 | {% endfor %} 14 | {{ hostvars[groups['primary_node'][0]]['node_mesh_ip'] }} sf-primary -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/files/writedata_userdata: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | # runcmd is only run on first boot. bootcmd is run for every boot, but 4 | # happens earlier in the boot process. It is sufficient for this use case 5 | # though. 6 | bootcmd: 7 | - | 8 | dd if=/dev/random of=/randomdata bs=1m count=50 9 | echo "" > /dev/console 10 | echo "System booted ok" > /dev/console 11 | echo "" > /dev/console 12 | echo "System booted ok" > /dev/console 13 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/tasks/syslog.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Send syslog to the primary server, unless I am the primary server 3 | template: 4 | src: templates/rsyslog-client-01-sf.conf 5 | dest: /etc/rsyslog.d/01-sf.conf 6 | owner: root 7 | group: sudo 8 | mode: u=r,g=r,o= 9 | when: syslog_target != node_mesh_ip 10 | 11 | - name: Restart syslog 12 | service: 13 | name: rsyslog 14 | enabled: yes 15 | state: restarted 16 | when: syslog_target != node_mesh_ip -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/tasks/create_admin_namespace.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create an admin namespace called "system" with one key configured 3 | shell: | 4 | /srv/shakenfist/venv/bin/sf-ctl \ 5 | bootstrap-system-key deploy {{ system_key }} 6 | environment: 7 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 8 | SHAKENFIST_NODE_MESH_IP: "{{ node_mesh_ip }}" 9 | SHAKENFIST_EVENTLOG_NODE_IP: "{{ eventlog_node_ip }}" 10 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 11 | run_once: true -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/defaults/main.yml: -------------------------------------------------------------------------------- 1 | # shared 2 | system_key: "unknown" 3 | etcd_host: "127.0.0.1" 4 | node_name: "localhost" 5 | node_mesh_ip: "127.0.0.1" 6 | 7 | # bootstrap 8 | server_package: "shakenfist" 9 | agent_package: "shakenfist-agent" 10 | client_package: "shakenfist-client" 11 | pip_extra: "" 12 | 13 | # syslog 14 | syslog_target: "unknown" 15 | 16 | # config 17 | sf_config_template_path: "unknown" 18 | 19 | # register 20 | # ... 21 | 22 | # create_admin_namespace 23 | eventlog_node_ip: "127.0.0.1" -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Andrew McCallum 2 | Andy McCallum 3 | Brad Hards 4 | Jack Adamson <7891953+jackadamson@users.noreply.github.com> 5 | Jack Adamson 6 | John 7 | LudvikGalois 8 | Michael Still 9 | Mike Carden 10 | Robert `Probie' Offner 11 | mandoonandy 12 | shakenfist-bot <72302723+shakenfist-bot@users.noreply.github.com> 13 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_api.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from shakenfist_ci import base 3 | 4 | 5 | class TestOpenAPIJS(base.BaseNamespacedTestCase): 6 | def __init__(self, *args, **kwargs): 7 | kwargs['namespace_prefix'] = 'openapijs' 8 | super().__init__(*args, **kwargs) 9 | 10 | def test_ui_js(self): 11 | # Ensure we can fetch the UI Javascript 12 | r = requests.get( 13 | f'{self.test_client.base_url}/flasgger_static/swagger-ui.css') 14 | self.assertEqual(200, r.status_code) 15 | -------------------------------------------------------------------------------- /shakenfist/tests/base.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import testtools 4 | 5 | 6 | class ShakenFistTestCase(testtools.TestCase): 7 | def setUp(self): 8 | super().setUp() 9 | 10 | # Logging is configured in shakenfist/tests/__init__.py to write to 11 | # stdout, which stestr captures and only displays for failing tests. 12 | 13 | self.mock_add_event_multi = mock.patch( 14 | 'shakenfist.eventlog.add_event_multi') 15 | self.mock_add_event_multi.start() 16 | self.addCleanup(self.mock_add_event_multi.stop) 17 | -------------------------------------------------------------------------------- /.github/workflows/renovate.yml: -------------------------------------------------------------------------------- 1 | name: Renovate dependency updater 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: '0 * * * *' 6 | 7 | jobs: 8 | renovate: 9 | runs-on: [self-hosted, static] 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v4 13 | 14 | - name: Self-hosted Renovate 15 | uses: renovatebot/github-action@v41.0.22 16 | with: 17 | token: ${{ secrets.RENOVATE_TOKEN }} 18 | env: 19 | RENOVATE_AUTODISCOVER: "true" 20 | RENOVATE_AUTODISCOVER_FILTER: "shakenfist/shakenfist" -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/pki_internal_ca/defaults/main.yml: -------------------------------------------------------------------------------- 1 | role_action: "undefined" 2 | 3 | ca_path: "/srv/shakenfist/pki/CA" 4 | deploy_name: "sf" 5 | hostname: "undefined" 6 | 7 | ca_cert_path: "{{ ca_path }}/{{ deploy_name }}-ca-cert.pem" 8 | ca_key_path: "{{ ca_path }}/{{ deploy_name }}-ca-key.pem" 9 | ca_template_path: "{{ ca_path }}/{{ deploy_name }}-ca-template.info" 10 | 11 | host_cert_path: "{{ ca_path }}/{{ hostname }}-server-cert.pem" 12 | host_key_path: "{{ ca_path }}/{{ hostname }}-server-key.pem" 13 | host_template_path: "{{ ca_path }}/{{ hostname }}-server-template.pem" -------------------------------------------------------------------------------- /shakenfist/deploy/ansiblemoduletests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Simple node ansible module tests. 4 | 5 | function log { 6 | echo -e "$(date) $1" 7 | } 8 | 9 | # Install dependencies 10 | sudo apt-get install -y pwgen 11 | 12 | cd /home/debian/shakenfist/shakenfist/deploy/ansible_module_ci 13 | for scenario in *.yml; do 14 | echo 15 | echo 16 | log "=== Scenario ${scenario} ===" 17 | echo 18 | ansible-playbook ${scenario} 19 | if [ $? -gt 0 ]; then 20 | echo "TESTS FAILED." 21 | exit 1 22 | fi 23 | echo 24 | done 25 | 26 | # Done 27 | echo 28 | log "Test complete" -------------------------------------------------------------------------------- /docs/components/overview.md: -------------------------------------------------------------------------------- 1 | # What are the components of a Shaken Fist cluster? 2 | 3 | Shaken Fist is composed of a series of components. There is Shaken Fist itself, 4 | which provides the orchestration and APIs to handle compute and virtual 5 | networks. The majority of this website discusses Shaken Fist, and if it is 6 | not specified, then you should assume that Shaken Fist is the component 7 | providing functionality. 8 | 9 | [Kerbside](/components/kerbside/) is a SPICE protocol native VDI proxy 10 | responsible for providing rich VDI experiences to users of Shaken Fist. You 11 | can read more about Kerbside at the page linked above. -------------------------------------------------------------------------------- /.github/workflows/refresh-website.yml: -------------------------------------------------------------------------------- 1 | name: Refresh shakenfist.com daily 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '0 3 * * *' # Runs every day at 3am 7 | 8 | jobs: 9 | refresh: 10 | runs-on: [self-hosted, static] 11 | steps: 12 | - name: Trigger GitHub pages rebuild 13 | run: | 14 | curl --fail --request POST \ 15 | --url https://api.github.com/repos/${{ github.repository }}/pages/builds \ 16 | --header "Authorization: Bearer $USER_TOKEN" 17 | env: 18 | # You must create a personal token with repo access as GitHub does 19 | # not yet support server-to-server page builds. 20 | USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} 21 | -------------------------------------------------------------------------------- /docs/developer_guide/updating_docs.md: -------------------------------------------------------------------------------- 1 | # Updating These Docs 2 | 3 | Built using MkDocs: https://www.mkdocs.org/ 4 | Theme: https://squidfunk.github.io/mkdocs-material/customization/ 5 | 6 | ## Setup 7 | 8 | Install mkdocs and the material theme 9 | ```bash 10 | pip install mkdocs-material 11 | ``` 12 | 13 | ## Viewing Locally 14 | 15 | Start the live web-server with 16 | ```bash 17 | mkdocs serve 18 | ``` 19 | View at http://localhost:8000 20 | 21 | ## Deploying to GitHub Pages 22 | 23 | Build and deploy with 24 | ```bash 25 | mkdocs gh-deploy 26 | ``` 27 | This will push to the `gh-pages` branch of the current git remote. 28 | 29 | ## Navigation Bar 30 | 31 | The navigation bar is configured via the `mkdocs.yml` file in the repository root. -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/templates/journald.tmpl: -------------------------------------------------------------------------------- 1 | [Journal] 2 | #Storage=auto 3 | #Compress=yes 4 | #Seal=yes 5 | #SplitMode=uid 6 | #SyncIntervalSec=5m 7 | #RateLimitIntervalSec=30s 8 | #RateLimitBurst=10000 9 | #SystemMaxUse= 10 | #SystemKeepFree= 11 | SystemMaxFileSize=1000M 12 | #SystemMaxFiles=100 13 | #RuntimeMaxUse= 14 | #RuntimeKeepFree= 15 | #RuntimeMaxFileSize= 16 | #RuntimeMaxFiles=100 17 | #MaxRetentionSec= 18 | #MaxFileSec=1month 19 | #ForwardToSyslog=yes 20 | #ForwardToKMsg=no 21 | #ForwardToConsole=no 22 | #ForwardToWall=yes 23 | #TTYPath=/dev/console 24 | #MaxLevelStore=debug 25 | #MaxLevelSyslog=debug 26 | #MaxLevelKMsg=notice 27 | #MaxLevelConsole=info 28 | #MaxLevelWall=emerg 29 | #LineMax=48K 30 | #ReadKMsg=yes 31 | #Audit=no -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/tasks/apache2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install apache2 3 | apt: 4 | name: apache2 5 | state: latest 6 | register: apt_action 7 | retries: 100 8 | until: apt_action is success or ('Failed to lock apt for exclusive operation' not in apt_action.msg and '/var/lib/dpkg/lock' not in apt_action.msg) 9 | 10 | - name: Enable proxy modules for apache 11 | shell: a2enmod proxy proxy_http lbmethod_byrequests 12 | 13 | - name: Write apache site 14 | template: 15 | src: files/apache-site-primary.conf 16 | dest: /etc/apache2/sites-available/sf-example.conf 17 | owner: root 18 | group: root 19 | mode: ugo+r 20 | 21 | - name: Restart apache 22 | service: 23 | name: apache2 24 | enabled: yes 25 | state: restarted -------------------------------------------------------------------------------- /shakenfist/util/callstack.py: -------------------------------------------------------------------------------- 1 | import re 2 | import traceback 3 | 4 | 5 | FILENAME_RE = re.compile('.*/dist-packages/shakenfist/(.*)') 6 | 7 | 8 | def get_caller(offset=-2): 9 | # Determine the name of the calling method 10 | filename = traceback.extract_stack()[offset].filename 11 | f_match = FILENAME_RE.match(filename) 12 | if f_match: 13 | filename = f_match.group(1) 14 | return '{}:{}:{}()'.format(filename, 15 | traceback.extract_stack()[offset].lineno, 16 | traceback.extract_stack()[offset].name) 17 | 18 | 19 | def generate_traceback(offset=-2): 20 | stack = traceback.extract_stack() 21 | formatted = traceback.format_list(stack[:-offset]) 22 | return '\n%s'.join(formatted) 23 | -------------------------------------------------------------------------------- /GOALS.md: -------------------------------------------------------------------------------- 1 | # Current development goals 2 | 3 | ## Currently under way 4 | 5 | * Convert from etcd to mariadb for persistent storage to take advantage of secondary indices 6 | * Add mypy type hints 7 | * Use shakenfist.schema for all data storage requiring a schema -- etcd while it lasts, generation of mariadb schemas, REST API outputs 8 | * REST API input schema validation 9 | 10 | ## Partial, but not currently being progressed 11 | 12 | * Move privileged operations to privexec. Perhaps move all process executions. Perhaps have multiple privexec daemons with different access levels. 13 | * Opportunistically convert to f-strings 14 | 15 | ## Not yet started 16 | 17 | * ansible-lint for the deployment Ansible 18 | * mTLS with a private CA for gRPC services 19 | * SO_PASSCRED for UDS services -------------------------------------------------------------------------------- /docs/operator_guide/threads.md: -------------------------------------------------------------------------------- 1 | # Threading 2 | 3 | Shaken Fist adopted a threading model for concurrency in v0.8, instead of the 4 | previous process-centric approach. This was mainly done as we moved to using 5 | more and more gRPC APIs, and experienced issues with reliability with gRPC and 6 | our forking heavy model. 7 | 8 | At that time, logging was expanded to include the thread id of the relevant 9 | thread. For processes with a single thread, this id will be the same as the 10 | process id itself. For other threads, it is an effectively random number 11 | *which the linux kernel reserves the right to recycle*. This can be annoying 12 | when processing logs, but is outside our control. Unfortunately, Python thread 13 | ids also do not map the the ones that Linux shows you in `ps` and `top`. -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/network/tasks/validate_mtus.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Determine the mesh network interface MTU 3 | shell: ip link show 4 | register: ip_links 5 | 6 | - name: Log network interfaces 7 | debug: 8 | msg: "{{ip_links.stdout}}" 9 | 10 | - name: Determine the mesh network interface MTU 11 | shell: ip link show {{node_mesh_nic}} | grep mtu | sed -e 's/.*mtu //' -e 's/ .*//' 12 | register: node_mtu_complex 13 | 14 | - name: Extract default interface MTU 15 | set_fact: 16 | node_mtu: "{{node_mtu_complex.stdout}}" 17 | 18 | - name: Log node MTU 19 | debug: 20 | msg: "Node MTU is {{node_mtu}}" 21 | 22 | - name: Abort if default interface MTU is too low 23 | fail: 24 | msg: "Node MTU is too low." 25 | when: ignore_mtu != "1" and node_mtu|int < 2000 -------------------------------------------------------------------------------- /shakenfist/util/json.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | import json 3 | import uuid 4 | 5 | from shakenfist import baseobject 6 | 7 | # To avoid circular imports, util modules should only import a limited 8 | # set of shakenfist modules, mainly exceptions, and specific 9 | # other util modules. 10 | 11 | 12 | class JSONEncoderCustomTypes(json.JSONEncoder): 13 | def default(self, obj): 14 | if isinstance(obj, baseobject.State): 15 | return obj.obj_dict() 16 | if isinstance(obj, uuid.UUID): 17 | return str(obj) 18 | if isinstance(obj, Enum): 19 | return obj.name 20 | return json.JSONEncoder.default(self, obj) 21 | 22 | 23 | def json_dump(data): 24 | return json.dumps( 25 | data, indent=4, sort_keys=True, cls=JSONEncoderCustomTypes) 26 | -------------------------------------------------------------------------------- /DEVELOPER.md: -------------------------------------------------------------------------------- 1 | # Displaying the documentation locally 2 | 3 | Install mkdocs-material, and then run `mkdocs serve`, like this: 4 | 5 | ``` 6 | $ pip install mkdocs-material 7 | ... 8 | $ mkdocs serve 9 | INFO - Building documentation... 10 | INFO - Cleaning site directory 11 | INFO - Documentation built in 0.67 seconds 12 | INFO - [11:54:17] Watching paths for changes: 'docs', 'mkdocs.yml' 13 | INFO - [11:54:17] Serving on http://127.0.0.1:8000/ 14 | ``` 15 | 16 | # Finding commits made by a human 17 | 18 | Now that shakenfist-bot is making a lot of automated commits, its sometimes 19 | nice to be able to see only changes made by a human. I use this command line: 20 | 21 | ``` 22 | git log --no-merges --oneline --invert-grep --perl-regexp \ 23 | --author='^((?!shakenfist-bot).*)$' 24 | ``` -------------------------------------------------------------------------------- /protos/common.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package shakenfist.protos; 4 | 5 | message EnvironmentVariable { 6 | string name = 1; 7 | string value = 2; 8 | } 9 | 10 | message ExecuteRequest { 11 | enum IOPriority { 12 | NORMAL = 0; 13 | LOW = 1; 14 | HIGH = 2; 15 | } 16 | 17 | string command = 1; 18 | repeated EnvironmentVariable environment_variables = 2; 19 | string network_namespace = 3; 20 | IOPriority io_priority = 4; 21 | string working_directory = 5; 22 | string request_id = 6; 23 | string execution_id = 7; 24 | } 25 | 26 | message ExecuteReply { 27 | string stdout = 1; 28 | string stderr = 2; 29 | int32 exit_code = 3; 30 | string request_id = 4; 31 | string execution_id = 5; 32 | double execution_seconds = 6; 33 | } -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/files/sf-api.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Shaken Fist API daemon 3 | After=sf-privexec.service 4 | After=sf-nodelock.service 5 | After=sf-database.service 6 | Before=sf-sentinel-last.service 7 | PartOf=sf.target 8 | ReloadPropagatedFrom=sf.target 9 | 10 | [Service] 11 | Type=simple 12 | User=root 13 | Group=root 14 | 15 | EnvironmentFile=/etc/sf/config 16 | ExecStartPre=/bin/sh -c '/srv/shakenfist/venv/bin/sf-ctl verify-config' 17 | ExecStartPre=/bin/sh -c 'rm -f /run/sf/gunicorn.pid' 18 | ExecStart=/bin/sh -c '/srv/shakenfist/venv/bin/gunicorn --workers 5 --bind 0.0.0.0:13000 --log-syslog --log-syslog-prefix sf --timeout 300 --name "sf-api" --pid /run/sf/gunicorn.pid --preload shakenfist.external_api.app:app' 19 | 20 | Restart=on-failure 21 | RestartSec=5 22 | 23 | [Install] 24 | WantedBy=sf.target 25 | -------------------------------------------------------------------------------- /shakenfist/tests/test_util_general.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from shakenfist.tests import base 4 | from shakenfist.util import general as util_general 5 | 6 | 7 | class UtilUserAgent(base.ShakenFistTestCase): 8 | @mock.patch('cpuinfo.get_cpu_info', return_value={ 9 | 'arch_string_raw': 'x86_64', 10 | 'vendor_id_raw': 'GenuineIntel' 11 | }) 12 | @mock.patch('distro.name', return_value='Debian GNU/Linux 10 (buster)') 13 | @mock.patch('shakenfist.util.general.get_version', return_value='1.2.3') 14 | def test_user_agent(self, mock_version, mock_distro, mock_cpuinfo): 15 | ua = util_general.get_user_agent() 16 | self.assertEqual('Mozilla/5.0 (Debian GNU/Linux 10 (buster); ' 17 | 'GenuineIntel x86_64) Shaken Fist/1.2.3', 18 | ua) 19 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/tasks/build-wheel.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure unit tests pass 3 | shell: 4 | cmd: tox -epy3 5 | chdir: "{{repo_path}}" 6 | tags: unittest 7 | 8 | - name: Clear out old wheels 9 | file: 10 | path: "{{repo_path}}/dist" 11 | state: absent 12 | 13 | - name: Ensure we have a local dist directory 14 | file: 15 | path: "{{repo_path}}/dist" 16 | state: directory 17 | 18 | - name: Build a wheel for {{package}} 19 | shell: 20 | cmd: python3 setup.py sdist bdist_wheel 21 | chdir: "{{repo_path}}" 22 | 23 | - name: Find the most recent wheel for {{package}} 24 | shell: ls -rt {{repo_path}}/dist/{{package}}-*.whl | tail -1 | sed 's/.*\/dist\///' 25 | register: wheel_complex 26 | 27 | - name: Extract wheel filename 28 | set_fact: 29 | "{{package}}_wheel_path": "{{wheel_complex.stdout}}" 30 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/tasks/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Write an ansible inventory file to handle ad hoc commands 3 | template: 4 | src: templates/inventory.yaml 5 | dest: /etc/sf/inventory.yaml 6 | owner: root 7 | group: sudo 8 | mode: u=r,g=r,o= 9 | 10 | - name: Find hypervisor with lowest MTU 11 | set_fact: 12 | lowest_mtu_hypervisor: "{{ groups['hypervisors'] | sort('node_mtu' | int) | first }}" 13 | 14 | - name: Find lowest MTU 15 | set_fact: 16 | lowest_mtu: "{{ hostvars[lowest_mtu_hypervisor]['node_mtu'] }}" 17 | 18 | - name: Write syslog file 19 | template: 20 | src: templates/rsyslog-server-01-sf.conf 21 | dest: /etc/rsyslog.d/01-sf.conf 22 | owner: root 23 | group: sudo 24 | mode: u=r,g=r,o= 25 | 26 | - name: Restart syslog 27 | service: 28 | name: rsyslog 29 | enabled: yes 30 | state: restarted -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_uploads.py: -------------------------------------------------------------------------------- 1 | import string 2 | 3 | from shakenfist_ci import base 4 | 5 | 6 | class TestUploads(base.BaseNamespacedTestCase): 7 | def __init__(self, *args, **kwargs): 8 | kwargs['namespace_prefix'] = 'uploads' 9 | super().__init__(*args, **kwargs) 10 | 11 | def test_upload(self): 12 | upl = self.test_client.create_upload() 13 | for _ in range(100): 14 | self.test_client.send_upload(upl['uuid'], string.ascii_letters) 15 | 16 | self.test_client.truncate_upload(upl['uuid'], 100) 17 | 18 | for _ in range(50): 19 | self.test_client.send_upload(upl['uuid'], string.ascii_letters) 20 | 21 | a = self.test_client.upload_artifact('test', upl['uuid']) 22 | 23 | self.assertEqual( 24 | len(string.ascii_letters) * 50 + 100, a['blobs']['1']['size']) 25 | -------------------------------------------------------------------------------- /docs/developer_guide/release_process.md: -------------------------------------------------------------------------------- 1 | Shaken Fist's release process 2 | ============================= 3 | 4 | Shaken Fist is now split across a number of repositories to simplify development 5 | and usage. Unfortunately, that complicated the release process. This page 6 | documents the current release process although the reality is that only Michael 7 | can do a release right now because of the requirement to sign releases with his 8 | GPG key. 9 | 10 | ## Testing 11 | 12 | We only release things which have passed CI testing, and preferably have had a 13 | period running as the underlying cloud for the CI cluster as well. Sometimes in 14 | an emergency we will bend the rules for a hotfix, but we should try and avoid 15 | doing that. 16 | 17 | ## For reach repository to be released 18 | 19 | Checkout the repository and ensure you're in the right branch. Then just run 20 | `release.sh` and follow the bounching ball. -------------------------------------------------------------------------------- /protos/nodelock.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package shakenfist.protos; 4 | 5 | message LockRequest { 6 | string requester = 1; 7 | string key = 2; 8 | } 9 | 10 | message UnlockRequest { 11 | string requester = 1; 12 | string key = 2; 13 | } 14 | 15 | message NodeLockRequest { 16 | oneof request { 17 | LockRequest lock_request = 1; 18 | UnlockRequest unlock_request = 2; 19 | } 20 | } 21 | 22 | message LockReply { 23 | enum Outcome { 24 | OK = 0; 25 | ALREADY_HELD = 1; 26 | DENIED = 2; 27 | } 28 | 29 | Outcome outcome = 1; 30 | } 31 | 32 | message UnlockReply { 33 | enum Outcome { 34 | OK = 0; 35 | NOT_HELD = 2; 36 | } 37 | 38 | Outcome outcome = 1; 39 | } 40 | 41 | message NodeLockReply { 42 | oneof reply { 43 | LockReply lock_reply = 1; 44 | UnlockReply unlock_reply = 2; 45 | } 46 | } -------------------------------------------------------------------------------- /docs/developer_guide/standards.md: -------------------------------------------------------------------------------- 1 | Concepts and Standards 2 | === 3 | # Ensuring a Common Language within the code base 4 | 5 | This document records the standards and common language used within the Shaken Fist software system. 6 | 7 | It should also record why the choice was made. 8 | 9 | (This is actually just notes to save our future selves from tripping over the same problems.) 10 | 11 | ## etcd keys 12 | 13 | Key names in `etcd` should be in the singular, for example `/sf/namespace/` not `/sf/namespaces/` note that this is different than the REST API. 14 | 15 | ## Memory 16 | 17 | Memory is measured in MiB in Shaken Fist. All references to memory size are stored and transmitted in MiB: Gigabytes can be too big if you want a lot of small machines. Kilobytes is just too many numbers to type. The ```libvirt``` API measures memory in KiB. Therefore, interactions with the library need to be careful to convert from MiB to KiB. 18 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_nodes.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | from shakenfist_client import apiclient 3 | 4 | 5 | class TestNodes(base.BaseNamespacedTestCase): 6 | def __init__(self, *args, **kwargs): 7 | kwargs['namespace_prefix'] = 'nodes' 8 | super().__init__(*args, **kwargs) 9 | 10 | def test_get_node(self): 11 | # I know this is a bit weird and is just testing if both calls return 12 | # the same name, but what its _really_ doing is ensuring the get_nodes() 13 | # call returns at all. 14 | nodes = self.system_client.get_nodes() 15 | n = self.system_client.get_node(nodes[0]['name']) 16 | self.assertEqual(nodes[0]['name'], n['name']) 17 | 18 | def test_get_missing_node(self): 19 | self.assertRaises( 20 | apiclient.ResourceNotFoundException, self.system_client.get_node, 21 | 'banana') 22 | -------------------------------------------------------------------------------- /docs/operator_guide/locks.md: -------------------------------------------------------------------------------- 1 | # Locks 2 | 3 | Shaken Fist uses etcd for distributed locking. All locks are written into etcd 4 | with the `/sflocks` key prefix. Locks are effectively leases on a key within 5 | etcd, where the key contains metadata about the lock being held. This means its 6 | easy to determine who else is holding a lock if you see contention issues within 7 | your cluster. 8 | 9 | The easiest way to do this is with the `sf-client admin lock list` command, 10 | which will list all locks currently held in the cluster. For example, here's a 11 | relatively idle cluster: 12 | 13 | ``` 14 | $ sf-client admin lock list 15 | +----------------------+-------+------+---------------------+ 16 | | lock | pid | node | operation | 17 | +----------------------+-------+------+---------------------+ 18 | | /sflocks/sf/cluster/ | 26407 | sf-7 | Cluster maintenance | 19 | +----------------------+-------+------+---------------------+ 20 | ``` 21 | -------------------------------------------------------------------------------- /shakenfist/deploy/requirements.yml: -------------------------------------------------------------------------------- 1 | # Ansible galaxy roles 2 | 3 | # Yes, there really are two names for unarchive-deps depending on the version 4 | # of the other galaxy roles you're using. 5 | 6 | # - src: andrewrothstein.unarchive-deps 7 | - src: https://github.com/mikalstill/ansible-unarchive-deps 8 | version: master 9 | name: andrewrothstein.unarchive-deps 10 | 11 | # - src: andrewrothstein.unarchivedeps 12 | - src: https://github.com/mikalstill/ansible-unarchivedeps 13 | version: master 14 | name: andrewrothstein.unarchivedeps 15 | 16 | # - src: andrewrothstein.etcd 17 | # Works with everything except Ubuntu 20.04: 2.1.43 18 | - src: https://github.com/mikalstill/ansible-etcd 19 | version: 2.3.10 20 | name: andrewrothstein.etcd 21 | 22 | # - src: andrewrothstein.etcd-cluster 23 | # Works with everything except Ubuntu 20.04: 3.0.1.1 24 | - src: https://github.com/mikalstill/ansible-etcd-cluster 25 | version: 3.0.7 26 | name: andrewrothstein.etcd-cluster -------------------------------------------------------------------------------- /shakenfist/tests/test_images.py: -------------------------------------------------------------------------------- 1 | from shakenfist import images 2 | from shakenfist.tests import base 3 | 4 | 5 | class ImageResolverTestCase(base.ShakenFistTestCase): 6 | def test_resolve_ubuntu_1604(self): 7 | self.assertEqual( 8 | 'https://images.shakenfist.com/ubuntu:16.04/latest.qcow2', 9 | images._resolve_image('ubuntu:16.04')) 10 | 11 | def test_resolve_ubuntu_16804(self): 12 | self.assertEqual( 13 | 'https://images.shakenfist.com/ubuntu:18.04/latest.qcow2', 14 | images._resolve_image('ubuntu:18.04')) 15 | 16 | def test_resolve_ubuntu_2004(self): 17 | self.assertEqual( 18 | 'https://images.shakenfist.com/ubuntu:20.04/latest.qcow2', 19 | images._resolve_image('ubuntu:20.04')) 20 | 21 | def test_resolve_ubuntu_2204(self): 22 | self.assertEqual( 23 | 'https://images.shakenfist.com/ubuntu:22.04/latest.qcow2', 24 | images._resolve_image('ubuntu:22.04')) 25 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "search.exclude": { 3 | "**/.venv": true, 4 | "**/.tox": true, 5 | "**/.mypy_cache": true, 6 | "**/__pycache__": true, 7 | "**/*.egg-info": true, 8 | "**/build": true, 9 | "**/dist": true, 10 | "**/.pytest_cache": true, 11 | "**/.coverage": true, 12 | "**/htmlcov": true, 13 | "**/.eggs": true, 14 | "**/site": true 15 | }, 16 | "files.exclude": { 17 | "**/.venv": true, 18 | "**/.tox": true, 19 | "**/.mypy_cache": true, 20 | "**/__pycache__": true, 21 | "**/*.egg-info": true, 22 | "**/.pytest_cache": true, 23 | "**/.coverage": true 24 | }, 25 | "files.watcherExclude": { 26 | "**/.venv/**": true, 27 | "**/.tox/**": true, 28 | "**/.mypy_cache/**": true, 29 | "**/__pycache__/**": true, 30 | "**/*.egg-info/**": true, 31 | "**/.pytest_cache/**": true 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /shakenfist/protos/agent_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | import warnings 5 | 6 | 7 | GRPC_GENERATED_VERSION = '1.70.0' 8 | GRPC_VERSION = grpc.__version__ 9 | _version_not_supported = False 10 | 11 | try: 12 | from grpc._utilities import first_version_is_lower 13 | _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) 14 | except ImportError: 15 | _version_not_supported = True 16 | 17 | if _version_not_supported: 18 | raise RuntimeError( 19 | f'The grpc package installed is at version {GRPC_VERSION},' 20 | + f' but the generated code in agent_pb2_grpc.py depends on' 21 | + f' grpcio>={GRPC_GENERATED_VERSION}.' 22 | + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' 23 | + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' 24 | ) 25 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/smoke_ci_tests/test_disk_specs.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | 3 | 4 | class TestDiskSpecifications(base.BaseNamespacedTestCase): 5 | def __init__(self, *args, **kwargs): 6 | kwargs['namespace_prefix'] = 'diskspecs' 7 | super().__init__(*args, **kwargs) 8 | 9 | def test_default(self): 10 | inst = self.test_client.create_instance( 11 | 'test-default-disk', 1, 1024, None, 12 | [ 13 | { 14 | 'size': 8, 15 | 'base': base.CLUSTER_CI_IMAGE, 16 | 'type': 'disk' 17 | } 18 | ], None, None) 19 | 20 | self.assertIsNotNone(inst['uuid']) 21 | self._await_instance_ready(inst['uuid']) 22 | 23 | results = self._await_command(inst['uuid'], 'df -h') 24 | self.assertEqual(0, results['return-code']) 25 | self.assertEqual('', results['stderr']) 26 | self.assertTrue('vda' in results['stdout']) 27 | -------------------------------------------------------------------------------- /shakenfist/protos/common_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | import warnings 5 | 6 | 7 | GRPC_GENERATED_VERSION = '1.70.0' 8 | GRPC_VERSION = grpc.__version__ 9 | _version_not_supported = False 10 | 11 | try: 12 | from grpc._utilities import first_version_is_lower 13 | _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) 14 | except ImportError: 15 | _version_not_supported = True 16 | 17 | if _version_not_supported: 18 | raise RuntimeError( 19 | f'The grpc package installed is at version {GRPC_VERSION},' 20 | + f' but the generated code in common_pb2_grpc.py depends on' 21 | + f' grpcio>={GRPC_GENERATED_VERSION}.' 22 | + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' 23 | + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' 24 | ) 25 | -------------------------------------------------------------------------------- /shakenfist/protos/nodelock_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | import warnings 5 | 6 | 7 | GRPC_GENERATED_VERSION = '1.70.0' 8 | GRPC_VERSION = grpc.__version__ 9 | _version_not_supported = False 10 | 11 | try: 12 | from grpc._utilities import first_version_is_lower 13 | _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) 14 | except ImportError: 15 | _version_not_supported = True 16 | 17 | if _version_not_supported: 18 | raise RuntimeError( 19 | f'The grpc package installed is at version {GRPC_VERSION},' 20 | + f' but the generated code in nodelock_pb2_grpc.py depends on' 21 | + f' grpcio>={GRPC_GENERATED_VERSION}.' 22 | + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' 23 | + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' 24 | ) 25 | -------------------------------------------------------------------------------- /shakenfist/protos/privexec_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | import warnings 5 | 6 | 7 | GRPC_GENERATED_VERSION = '1.70.0' 8 | GRPC_VERSION = grpc.__version__ 9 | _version_not_supported = False 10 | 11 | try: 12 | from grpc._utilities import first_version_is_lower 13 | _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) 14 | except ImportError: 15 | _version_not_supported = True 16 | 17 | if _version_not_supported: 18 | raise RuntimeError( 19 | f'The grpc package installed is at version {GRPC_VERSION},' 20 | + f' but the generated code in privexec_pb2_grpc.py depends on' 21 | + f' grpcio>={GRPC_GENERATED_VERSION}.' 22 | + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' 23 | + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' 24 | ) 25 | -------------------------------------------------------------------------------- /docs/operator_guide/power_states.md: -------------------------------------------------------------------------------- 1 | Instance power states 2 | ===================== 3 | 4 | Shaken Fist version 0.2.1 introduced power states for instances. Before this, you could power on or off an instance, or pause it, but you couldn't tell what power state the instance was actually in. That was pretty confusing and was therefore treated as a bug. 5 | 6 | The following power states are implemented: 7 | 8 | * **on**: the instance is running 9 | * **off**: the instance is not running 10 | * **paused**: the instance is paused 11 | * **crashed**: the instance is crashed according to the hypervisor. Instances in this power state will also be in an instance state of "error". 12 | 13 | There are additionally a set of "transition states" which are used to indicate that you have requested a change of state that might not yet have completed. These are: 14 | 15 | * transition-to-on 16 | * transition-to-off 17 | * transition-to-paused 18 | 19 | We're hoping to not have to implement a transition-to-crashed state, but you never know. 20 | -------------------------------------------------------------------------------- /.github/workflows/publish-website.yml: -------------------------------------------------------------------------------- 1 | # With thanks to https://bluegenes.github.io/mkdocs-github-actions/ 2 | name: Build and publish shakenfist.com 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - develop 8 | 9 | jobs: 10 | deploy: 11 | runs-on: [self-hosted, static] 12 | steps: 13 | - uses: actions/checkout@v4 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Setup Python 18 | uses: actions/setup-python@v5 19 | with: 20 | python-version: "3.13" 21 | architecture: "x64" 22 | 23 | - name: Install dependencies 24 | run: | 25 | python3 -m pip install --upgrade pip 26 | python3 -m pip install mkdocs 27 | python3 -m pip install mkdocs-material 28 | 29 | - name: Build site 30 | run: mkdocs build 31 | 32 | - name: Deploy 33 | uses: peaceiris/actions-gh-pages@v4 34 | with: 35 | github_token: ${{ secrets.GITHUB_TOKEN }} 36 | publish_dir: ./site 37 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/tasks/distro-check.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Warn if unsupported distro 3 | pause: 4 | prompt: "\nWARNING: Only Ubuntu and Debian are supported at present ({{ ansible_distribution }} detected), this will probably fail" 5 | seconds: 30 6 | when: ansible_distribution.split(' ', 1)[0] | lower not in ["ubuntu", "debian"] 7 | 8 | - name: Warn if unsupported version of Debian 9 | pause: 10 | prompt: "\nWARNING: Only Debian 12 is supported at present ({{ ansible_distribution_major_version }} detected), this will probably fail" 11 | seconds: 30 12 | when: ansible_distribution.split(' ', 1)[0] | lower == "debian" and ansible_distribution_major_version != "12" 13 | 14 | - name: Warn if unsupported version of Ubuntu 15 | pause: 16 | prompt: "\nWARNING: Only Ubuntu 24.04 is supported at present ({{ ansible_distribution_major_version }} detected), this will probably fail" 17 | seconds: 30 18 | when: ansible_distribution.split(' ', 1)[0] | lower == "ubuntu" and ansible_distribution_major_version != "24.04" 19 | -------------------------------------------------------------------------------- /docs/developer_guide/api_reference/admin.md: -------------------------------------------------------------------------------- 1 | # Admin (/admin/) 2 | 3 | ## Locks 4 | 5 | As discussed in the [operator guide](/operator_guide/locks/), you can query 6 | what locks exist in a Shaken Fist cluster, as well as who is currently holding 7 | those locks (machine and process id). 8 | 9 | ???+ tip "REST API calls" 10 | 11 | * [GET /admin/locks](https://openapi.shakenfist.com/#/admin/get_admin_locks): List locks currently held in the cluster. 12 | 13 | ??? example "Python API client: list cluster locks" 14 | 15 | ```python 16 | from shakenfist_client import apiclient 17 | 18 | sf_client = apiclient.Client() 19 | locks = sf_client.get_existing_locks() 20 | 21 | print('lock,pid,node,operation') 22 | for ref, meta in locks.items(): 23 | print('%s,%s,%s,%s' % (ref, meta['pid'], meta['node'], meta.get('operation'))) 24 | ``` 25 | 26 | ```bash 27 | $ python3 example.py 28 | lock,pid,node,operation 29 | /sflocks/sf/network/d2950d74-50c7-4790-a985-c43d9eb9bad9,2834066,sf-3,Network ensure mesh 30 | ``` -------------------------------------------------------------------------------- /protos/_make_stubs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run this from directory containing the generated gRPC code 4 | 5 | python3 -m grpc_tools.protoc -I../../protos --python_out=. --pyi_out=. \ 6 | --grpc_python_out=. $(find ../../protos -name '*.proto') 7 | 8 | # This is terrible, but gRPC lacks a python_package option, so we have to 9 | # tweak the imports in the _grpc.py files. 10 | 11 | # Detect OS for sed in-place syntax (macOS uses -i '', Linux uses -i) 12 | if [[ "$OSTYPE" == "darwin"* ]]; then 13 | SED_INPLACE="sed -i ''" 14 | else 15 | SED_INPLACE="sed -i" 16 | fi 17 | 18 | for item in *.py; do 19 | importname="common_pb2" 20 | echo "Correcting ${importname} import in ${item}..." 21 | $SED_INPLACE "s/import ${importname}/from shakenfist.protos import ${importname}/g" ${item} 22 | done 23 | 24 | for item in *_grpc.py; do 25 | importname=$(echo ${item} | sed 's/_grpc.py//') 26 | echo "Correcting ${importname} import in ${item}..." 27 | $SED_INPLACE "s/import ${importname}/from shakenfist.protos import ${importname}/g" ${item} 28 | done -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/database/tasks/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # The database daemon provides a gRPC interface to etcd, centralizing all 3 | # database access. This is only used when DATABASE_USE_DIRECT_ETCD is False. 4 | # When enabled, the database daemon must start before other daemons that 5 | # depend on database access. 6 | 7 | # Note: We must use direct etcd access for registering the database daemon 8 | # itself, since the database gRPC service isn't running yet. This is a 9 | # bootstrap chicken-and-egg problem. 10 | - name: Register the database daemon 11 | shell: | 12 | /srv/shakenfist/venv/bin/sf-ctl register-daemon database 13 | environment: 14 | SHAKENFIST_ETCD_HOST: "{{hostvars[groups['etcd_master'][0]]['node_mesh_ip']}}" 15 | SHAKENFIST_NODE_NAME: "{{node_name}}" 16 | SHAKENFIST_NODE_MESH_IP: "{{node_mesh_ip}}" 17 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 18 | 19 | - name: Start the database daemon 20 | service: 21 | name: sf-database 22 | enabled: yes 23 | state: restarted 24 | daemon_reload: no 25 | -------------------------------------------------------------------------------- /protos/event.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package shakenfist.protos; 4 | 5 | service EventService { 6 | rpc RecordEvent (EventRequest) returns (EventReply) {} 7 | rpc RecordMultiEvent (EventMultiRequest) returns (EventReply) {} 8 | } 9 | 10 | // An event against a single object, should be removed one day 11 | message EventRequest { 12 | string object_type = 1; 13 | string object_uuid = 2; 14 | string event_type = 3; 15 | optional float obsolete_timestamp = 4; 16 | string fqdn = 5; 17 | float duration = 6; 18 | string message = 7; 19 | string extra = 8; 20 | double timestamp = 9; 21 | } 22 | 23 | // An event against multiple objects 24 | message EventObject { 25 | string object_type = 1; 26 | string object_uuid = 2; 27 | } 28 | 29 | message EventMultiRequest { 30 | repeated EventObject objects = 1; 31 | string event_type = 2; 32 | string fqdn = 3; 33 | float duration = 4; 34 | string message = 5; 35 | string extra = 6; 36 | double timestamp = 7; 37 | } 38 | 39 | // An ack from the eventlog daemon 40 | message EventReply { 41 | bool ack = 1; 42 | } -------------------------------------------------------------------------------- /docs/operator_guide/python_versions.md: -------------------------------------------------------------------------------- 1 | # Supported python versions 2 | 3 | The versions of python we support are driven by the versions packaged in our 4 | supported operating systems. For Shaken Fist itself, we support: 5 | 6 | [//]: # (Note that if you change the list of supported operating systems you must also update installation.md in this directory) 7 | 8 | * Ubuntu 24.04: 3.12 9 | * Debian 11: 3.9 10 | * Debian 12: 3.11 11 | 12 | [//]: # (Note that if you change this version, you must also update the minimum python version in setup.cfg) 13 | 14 | We therefore support Python 3.9 and above for server side software. 15 | 16 | For client side software we are significantly more liberal. At the moment we 17 | build guest images for the following Linux distributions: 18 | 19 | * CentOS 9-stream: 3.11 20 | * Debian 11: 3.9 21 | * Debian 12: 3.11 22 | * Fedora 40: 3.12 23 | * Fedora 41: 3.13 24 | * Rocky Linux 8: 3.6 25 | * Rocky Linux 9: 3.9 26 | * Ubuntu 20.04: 3.8 27 | * Ubuntu 22.04: 3.10 28 | * Ubuntu 24.04: 3.12 29 | 30 | We therefore support Python 3.8 and above in client code such as the Shaken Fist 31 | client and in-guest agent. -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/files/apache-site-primary.conf: -------------------------------------------------------------------------------- 1 | 2 | ServerName sf-primary 3 | 4 | ServerAdmin webmaster@localhost 5 | DocumentRoot /var/www/html 6 | 7 | ErrorLog ${APACHE_LOG_DIR}/error.log 8 | CustomLog ${APACHE_LOG_DIR}/access.log combined 9 | 10 | 11 | {% for svr in groups.hypervisors %} 12 | BalancerMember "http://{{hostvars[svr]['node_mesh_ip']}}:13000" 13 | {% endfor %} 14 | 15 | 16 | ProxyPass "/api" "balancer://sfapi" 17 | ProxyPassReverse "/api" "balancer://sfapi" 18 | 19 | # Required for OpenAPI to be accessible 20 | ProxyPass "/apidocs" "balancer://sfapi/apidocs" 21 | ProxyPassReverse "/apidocs" "balancer://sfapi/apidocs" 22 | ProxyPass "/flasgger_static" "balancer://sfapi/flasgger_static" 23 | ProxyPassReverse "/flasgger_static" "balancer://sfapi/flasgger_static" 24 | ProxyPass "/apispec_1.json" "balancer://sfapi/apispec_1.json" 25 | ProxyPassReverse "/apispec_1.json" "balancer://sfapi/apispec_1.json" 26 | 27 | 28 | # vim: syntax=apache ts=4 sw=4 sts=4 sr noet -------------------------------------------------------------------------------- /docs/user_guide/metadata.md: -------------------------------------------------------------------------------- 1 | # Object metadata 2 | 3 | All objects in Shaken Fist support a simple metadata system. This system is 4 | presented as a series of key, value pairs which are stored against the 5 | relevant object. A worked example can be seen in the [description of instance 6 | affinity](/user_guide/affinity/) which requires specific keys and formats for 7 | their values, but you're not limited to that -- other keys can have data of any 8 | format that you can express in an API call. 9 | 10 | ???+ note 11 | 12 | It is not intended that you store large amounts of data in a metadata key. 13 | If you want to store more than a couple of kilobytes in a value, then instead 14 | store a reference to an external system or a blob which contains the data. 15 | 16 | You can set a metadata key's value on the command line like this: 17 | 18 | `sf-client instance set-metadata ...uuid... key-name key-value` 19 | 20 | Metadata values are show in the `show` output for the various object types, and 21 | there is no separate command to look them up. You can also delete a metadata key 22 | like this: 23 | 24 | `sf-client instance delete-metadata ...uuid... key-name` -------------------------------------------------------------------------------- /.github/workflows/docs-tests.yml: -------------------------------------------------------------------------------- 1 | name: Documentation tests 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - develop 7 | - v*-releases 8 | paths: 9 | - 'docs/**' 10 | - mkdocs.yml 11 | 12 | jobs: 13 | build-docs: 14 | runs-on: [self-hosted, vm] 15 | concurrency: 16 | group: ${{ github.workflow }}-${{ github.ref }}-lint 17 | cancel-in-progress: true 18 | 19 | steps: 20 | - name: Checkout code with two commits 21 | uses: actions/checkout@v4 22 | with: 23 | fetch-depth: 2 24 | 25 | - name: Ensure the docs build 26 | run: | 27 | cd ${GITHUB_WORKSPACE}/shakenfist 28 | 29 | mkdir -p /srv/github/artifacts/site 30 | /usr/bin/tox -edocs -- --site-dir /srv/github/artifacts/site | tee /srv/github/artifacts/site/build.log 31 | 32 | cd /srv/github/artifacts/ 33 | zip -r /srv/github/artifacts/site.zip site 34 | 35 | - uses: actions/upload-artifact@v4 36 | if: always() 37 | with: 38 | name: site 39 | retention-days: 90 40 | if-no-files-found: error 41 | path: /srv/github/artifacts/site.zip -------------------------------------------------------------------------------- /shakenfist/tests/schema/operations/test_baseclusteroperation.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | 3 | from pydantic import ValidationError 4 | 5 | from shakenfist.schema.operations.baseclusteroperation \ 6 | import CLUSTER_OPERATIONS 7 | from shakenfist.schema.operations.baseclusteroperation \ 8 | import dependency 9 | from shakenfist.tests import base 10 | 11 | 12 | class BaseClusterOperationTestCase(base.ShakenFistTestCase): 13 | def test_dependency_serialization_uuid4(self): 14 | dependency( 15 | op_type=CLUSTER_OPERATIONS.artifact_fetch_op, 16 | op_uuid=uuid4()) 17 | 18 | def test_dependency_serialization_str_uuid(self): 19 | dependency( 20 | op_type=CLUSTER_OPERATIONS.artifact_fetch_op, 21 | op_uuid=str(uuid4())) 22 | 23 | def test_dependency_serialization_bad_op_type(self): 24 | self.assertRaises( 25 | ValidationError, dependency, op_type='banana', op_uuid=uuid4()) 26 | 27 | def test_dependency_serialization_bad_uuid(self): 28 | self.assertRaises( 29 | ValidationError, dependency, 30 | op_type=CLUSTER_OPERATIONS.artifact_fetch_op, op_uuid='banana') 31 | -------------------------------------------------------------------------------- /docs/community.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Community 3 | --- 4 | # Welcome to the Shaken Fist community 5 | 6 | First off, Shaken Fist is an open source project covered by the Apache2 license. The community is relatively small at this point and we're still working out what works for us. For now, we communicate through the following mechanisms: 7 | 8 | * GitHub issues and pull requests in the various repositories. 9 | * A slack workspace at https://shakenfist.slack.com -- unfortunately you need to be invited to that workspace, but we're happy to do that. If you're interested, please email mikal@stillhq.com. 10 | 11 | # Supporters 12 | 13 | We try to track supporters here. Our apologies if we've missed someone, let us know and we'll fix it. 14 | 15 | * [Michael Still](https://www.madebymikal.com) and Andrew McCallum have contributed significant personal time to the project. 16 | * [Aptira](https://www.aptira.com) has donated developer time, as well as provided invaluable feedback from the largest real world deployments that we've had so far. 17 | * [FifthDomain](https://www.fifthdomain.com.au) has also donated developer time. 18 | * [Shaken Fist was a recipient of a 2020 Icculus micro grant](https://icculus.org/microgrant2020/). -------------------------------------------------------------------------------- /tools/flake8wrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # A simple wrapper around flake8 which makes it possible 4 | # to ask it to only verify files changed in the current 5 | # git HEAD patch. 6 | # 7 | # Intended to be invoked via tox: 8 | # 9 | # tox -eflake8 -- -HEAD 10 | # 11 | # Originally from the OpenStack project. 12 | 13 | FLAKE_COMMAND="flake8 --max-line-length=120" 14 | 15 | if test "x$1" = "x-HEAD" ; then 16 | shift 17 | files=$(git diff --name-only HEAD~1 | grep -v _pb2 | egrep ".py$") 18 | if [ -z "${files}" ]; then 19 | echo "No python files in change." 20 | exit 0 21 | fi 22 | 23 | filtered_files="" 24 | for file in $files; do 25 | if [ -e "$file" ]; then 26 | filtered_files="${filtered_files} ${file}" 27 | else 28 | echo "$file does not exist in the end state, skipping." 29 | fi 30 | done 31 | if [ -z "${filtered_files}" ]; then 32 | echo "No python files in change post filtration." 33 | exit 0 34 | fi 35 | 36 | echo "Running flake8 on ${filtered_files}" 37 | diff -u --from-file /dev/null ${filtered_files} | $FLAKE_COMMAND ${filtered_files} 38 | else 39 | echo "Running flake8 on all files" 40 | exec $FLAKE_COMMAND "$@" 41 | fi 42 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "gitAuthor": "shakenfist-bot ", 4 | "dependencyDashboard": "true", 5 | "assignees": [ 6 | "mikalstill" 7 | ], 8 | "schedule": [ 9 | "after 9pm" 10 | ], 11 | "packageRules": [ 12 | { 13 | "description": "Automatically merge minor and patch-level updates", 14 | "matchUpdateTypes": [ 15 | "minor", 16 | "patch", 17 | "digest" 18 | ], 19 | "automerge": false 20 | }, 21 | { 22 | "description": "Group pydantic updates together", 23 | "matchPackagePatterns": [ 24 | "pydantic" 25 | ], 26 | "groupName": "pydantic" 27 | }, 28 | { 29 | "description": "Group grpcio updates together", 30 | "matchPackagePatterns": [ 31 | "grpcio", 32 | "protobuf" 33 | ], 34 | "groupName": "grpcio" 35 | }, 36 | { 37 | "description": "Group things entangled with wraps", 38 | "matchPackagePatterns": [ 39 | "wraps", 40 | "iters", 41 | "async-extensions" 42 | ], 43 | "groupName": "entangled-wraps" 44 | } 45 | ], 46 | "minimumReleaseAge": "3 days", 47 | "rollbackPrs": "true" 48 | } 49 | -------------------------------------------------------------------------------- /BRANCHES.rst: -------------------------------------------------------------------------------- 1 | This document describes the Shaken Fist branching model, which only really matters for developers. 2 | 3 | All active development is done in the `develop` branch. This is where pull requests should target. 4 | Once a pull request has been merged to develop, it might be backported to a release branch, but that 5 | is optional and would depend on the particular situation. 6 | 7 | For each major release there is a branch with `-releases` at the end of its name. At the time of 8 | writing these are: 9 | 10 | * `v0.5-releases` 11 | * `v0.6-releases` 12 | 13 | These release branches have the component versions for Shaken Fist packages pinned to their 14 | corresponding release number. This is different from the `develop` branch, which has no component 15 | pinning. 16 | 17 | Thus, to the release process looks like this: 18 | 19 | ## Minor release 20 | 21 | Run `./release.sh` from the relevant release branch. 22 | 23 | ## Major release 24 | 25 | Branch off develop into a new release branch. Pin the component versions in `requirements.txt` and 26 | `getsf`. Addittionally, use the clingwrap hashin output to lock all our dependancies to specific 27 | versions. It would be good if this was done with hashes, but that is currently not possible. Then 28 | run `./release.sh`. 29 | -------------------------------------------------------------------------------- /docs/user_guide/objects.md: -------------------------------------------------------------------------------- 1 | # Objects 2 | 3 | Everything that you interact with in Shaken Fist is an object. Objects are 4 | almost always referred to by a UUID (specifically a version 4 UUID) as a 5 | string. The exceptions are: `node`s; `namespace`s; and `key`s within a namespace. 6 | 7 | In general an object is referred to in the API or on the command line "by 8 | reference", which means you can either pass the object's name or its UUID to the 9 | command. So for example if we had an instance with the UUID 10 | 0a38d51e-2f72-4848-80fb-03031978633b named "mikal", then we could run either of 11 | the commands below to the same effect: 12 | 13 | ``` 14 | sf-client instance show 0a38d51e-2f72-4848-80fb-03031978633b 15 | sf-client instance show mikal 16 | ``` 17 | 18 | In the case where you refer to an object by name, a lookup occurs of all 19 | objects visible to you (those in your namespace, and namespaces that trust 20 | your namespace). Additionally, shared artifacts are included if you're using 21 | an artifact command. 22 | 23 | It is possible that the name you're using isn't unique. For example there might 24 | be two instances named "mikal" with different UUIDs. In that case, you will get 25 | an error indicating that there was more than one object which matched, and you'll 26 | need to use a UUID to refer to the object. -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/mariadb/tasks/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Install and configure MariaDB server on etcd_master nodes 3 | 4 | - name: Install MariaDB server and client 5 | apt: 6 | name: 7 | - mariadb-server 8 | - mariadb-client 9 | - python3-mysqldb 10 | state: latest 11 | register: apt_action 12 | retries: 100 13 | until: apt_action is success or ('Failed to lock apt for exclusive operation' not in apt_action.msg and '/var/lib/dpkg/lock' not in apt_action.msg) 14 | 15 | - name: Enable and start MariaDB service 16 | service: 17 | name: mariadb 18 | enabled: yes 19 | state: started 20 | 21 | - name: Create shakenfist database 22 | community.mysql.mysql_db: 23 | name: shakenfist 24 | state: present 25 | login_unix_socket: /run/mysqld/mysqld.sock 26 | 27 | - name: Create shakenfist database user 28 | community.mysql.mysql_user: 29 | name: shakenfist 30 | password: "{{ mariadb_password }}" 31 | priv: 'shakenfist.*:ALL' 32 | host: '%' 33 | state: present 34 | login_unix_socket: /run/mysqld/mysqld.sock 35 | 36 | - name: Configure MariaDB to listen on mesh network only 37 | lineinfile: 38 | path: /etc/mysql/mariadb.conf.d/50-server.cnf 39 | regexp: '^bind-address' 40 | line: 'bind-address = {{ node_mesh_ip }}' 41 | notify: Restart MariaDB 42 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/pki_internal_ca/tasks/distribute_certificates.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: 4 | - "Distributing host certificate for {{ hostname }}." 5 | - "CA cert path: {{ ca_cert_path }}" 6 | - "CA key path: {{ ca_key_path }}" 7 | - "CA template path: {{ ca_template_path }}" 8 | - "Host cert path: {{ host_cert_path }}" 9 | - "Host key path: {{ host_key_path }}" 10 | - "Host template path: {{ host_template_path }}" 11 | 12 | - name: Make /etc/pki/CA 13 | file: 14 | path: /etc/pki/CA 15 | state: directory 16 | mode: '0755' 17 | 18 | - name: Make /etc/pki/libvirt-spice 19 | file: 20 | path: /etc/pki/libvirt-spice 21 | state: directory 22 | mode: '0755' 23 | 24 | - name: Copy CA certificate to host 25 | copy: 26 | src: "{{ ca_cert_path }}" 27 | dest: /etc/pki/libvirt-spice/ca-cert.pem 28 | owner: root 29 | group: root 30 | mode: '0444' 31 | 32 | - name: Copy host key to host 33 | copy: 34 | src: "{{ host_key_path }}" 35 | dest: /etc/pki/libvirt-spice/server-key.pem 36 | owner: root 37 | group: root 38 | mode: '0444' 39 | 40 | - name: Copy host certificate to host 41 | copy: 42 | src: "{{ host_cert_path }}" 43 | dest: /etc/pki/libvirt-spice/server-cert.pem 44 | owner: root 45 | group: root 46 | mode: '0444' -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/templates/inventory.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | all: 3 | hosts: 4 | {% for svr in groups.allsf %} 5 | {{hostvars[svr]['ansible_fqdn']}}: 6 | ansible_fqdn: {{hostvars[svr]['ansible_fqdn']}} 7 | ansible_ssh_host: {{hostvars[svr]['ansible_ssh_host']}} 8 | ansible_ssh_user: {{hostvars[svr]['ansible_ssh_user']}} 9 | ansible_ssh_private_key_file: {{hostvars[svr]['ansible_ssh_private_key_file']}} 10 | {% endfor %} 11 | 12 | children: 13 | primary_node: 14 | hosts: 15 | {% for svr in groups.primary_node %} 16 | {{hostvars[svr]['ansible_fqdn']}}: 17 | {% endfor %} 18 | 19 | hypervisors: 20 | hosts: 21 | {% for svr in groups.hypervisors %} 22 | {{hostvars[svr]['ansible_fqdn']}}: 23 | {% endfor %} 24 | 25 | etcd: 26 | hosts: 27 | {% for svr in groups.etcd_master %} 28 | {{hostvars[svr]['ansible_fqdn']}}: 29 | {% endfor %} 30 | 31 | network: 32 | hosts: 33 | {% for svr in groups.network_node %} 34 | {{hostvars[svr]['ansible_fqdn']}}: 35 | {% endfor %} 36 | 37 | eventlog: 38 | hosts: 39 | {% for svr in groups.eventlog_node %} 40 | {{hostvars[svr]['ansible_fqdn']}}: 41 | {% endfor %} 42 | 43 | storage: 44 | hosts: 45 | {% for svr in groups.storage %} 46 | {{hostvars[svr]['ansible_fqdn']}}: 47 | {% endfor %} -------------------------------------------------------------------------------- /shakenfist/util/access_tokens.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from flask_jwt_extended import create_access_token 4 | from flask_jwt_extended import get_jwt_identity 5 | 6 | from shakenfist.config import config 7 | from shakenfist.constants import EVENT_TYPE_AUDIT 8 | from shakenfist.exceptions import CannotParseJWTIdentity 9 | 10 | 11 | def create_token(ns, keyname, nonce, duration=config.API_TOKEN_DURATION): 12 | # NOTE(mikal): the "identity" here must be a string, which was not always 13 | # true for tokens we issued. 14 | token = create_access_token( 15 | identity=f'{ns.uuid}:{keyname}', 16 | additional_claims={ 17 | 'iss': config.ZONE, 18 | 'nonce': nonce 19 | }, 20 | expires_delta=datetime.timedelta(minutes=duration)) 21 | ns.add_event( 22 | EVENT_TYPE_AUDIT, 'token created from key', 23 | extra={ 24 | 'keyname': keyname, 25 | 'nonce': nonce, 26 | 'token': token 27 | }) 28 | return { 29 | 'access_token': token, 30 | 'token_type': 'Bearer', 31 | 'expires_in': duration * 60 32 | } 33 | 34 | 35 | def parse_jwt_identity(): 36 | ident_string = get_jwt_identity() 37 | ident = ident_string.split(':') 38 | if len(ident) != 2: 39 | raise CannotParseJWTIdentity(f'Cannot parse identity "{ident_string}"') 40 | return ident 41 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_auth.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | from shakenfist_client import apiclient 3 | 4 | 5 | def _namespace_names(namespaces): 6 | namespace_names = [] 7 | for ns in namespaces: 8 | namespace_names.append(ns['name']) 9 | return namespace_names 10 | 11 | 12 | class TestAuth(base.BaseTestCase): 13 | def test_namespaces(self): 14 | name = 'ci-auth-%s' % self._uniquifier() 15 | key = self._uniquifier() 16 | 17 | self.assertNotIn(name, self.system_client.get_namespaces()) 18 | self.system_client.create_namespace(name) 19 | self.system_client.add_namespace_key(name, 'test', key) 20 | self.assertIn( 21 | name, _namespace_names(self.system_client.get_namespaces())) 22 | 23 | self.assertRaises(apiclient.ResourceNotFoundException, 24 | self.system_client.delete_namespace_key, name, 'banana') 25 | self.assertIn( 26 | name, _namespace_names(self.system_client.get_namespaces())) 27 | 28 | self.system_client.delete_namespace_key(name, 'test') 29 | self.assertIn( 30 | name, _namespace_names(self.system_client.get_namespaces())) 31 | 32 | self.system_client.delete_namespace(name) 33 | self.assertNotIn( 34 | name, _namespace_names(self.system_client.get_namespaces())) 35 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/smoke_ci_tests/test_auth.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | from shakenfist_client import apiclient 3 | 4 | 5 | def _namespace_names(namespaces): 6 | namespace_names = [] 7 | for ns in namespaces: 8 | namespace_names.append(ns['name']) 9 | return namespace_names 10 | 11 | 12 | class TestAuth(base.BaseTestCase): 13 | def test_namespaces(self): 14 | name = 'ci-auth-%s' % self._uniquifier() 15 | key = self._uniquifier() 16 | 17 | self.assertNotIn(name, self.system_client.get_namespaces()) 18 | self.system_client.create_namespace(name) 19 | self.system_client.add_namespace_key(name, 'test', key) 20 | self.assertIn( 21 | name, _namespace_names(self.system_client.get_namespaces())) 22 | 23 | self.assertRaises(apiclient.ResourceNotFoundException, 24 | self.system_client.delete_namespace_key, name, 'banana') 25 | self.assertIn( 26 | name, _namespace_names(self.system_client.get_namespaces())) 27 | 28 | self.system_client.delete_namespace_key(name, 'test') 29 | self.assertIn( 30 | name, _namespace_names(self.system_client.get_namespaces())) 31 | 32 | self.system_client.delete_namespace(name) 33 | self.assertNotIn( 34 | name, _namespace_names(self.system_client.get_namespaces())) 35 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible_module_ci/001.yml: -------------------------------------------------------------------------------- 1 | # Tests: sf_network 2 | 3 | - hosts: localhost 4 | gather_facts: no 5 | 6 | tasks: 7 | - name: Ensure network doesn't exist at the start 8 | sf_network: 9 | name: "ansibleci-001" 10 | state: "absent" 11 | 12 | - name: Create a test network 13 | sf_network: 14 | netblock: "10.0.0.0/24" 15 | name: "ansibleci-001" 16 | state: present 17 | register: ci_network 18 | 19 | - name: Assert the network is new 20 | assert: 21 | that: ci_network['changed'] 22 | fail_msg: "changed should be true in {{ ci_network }}" 23 | 24 | - name: Assert no errors 25 | assert: 26 | that: not ci_network['failed'] 27 | fail_msg: "failed should be false in {{ ci_network }}" 28 | 29 | - name: Log network UUID 30 | debug: 31 | msg: "CI network has UUID {{ ci_network['meta']['uuid'] }}" 32 | 33 | - name: Delete the network 34 | sf_network: 35 | uuid: "{{ ci_network['meta']['uuid'] }}" 36 | state: absent 37 | register: ci_network 38 | 39 | - name: Assert the network was changed 40 | assert: 41 | that: ci_network['changed'] 42 | fail_msg: "changed should be true in {{ ci_network }}" 43 | 44 | - name: Assert no errors 45 | assert: 46 | that: not ci_network['failed'] 47 | fail_msg: "failed should be false in {{ ci_network }}" -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Release a specified version of Shaken Fist 4 | 5 | echo "--- Determine verison number ---" 6 | PREVIOUS=`git tag | egrep "^v" | sort -n | tail -1 | sed 's/^v//'` 7 | 8 | echo 9 | echo -n "What is the version number (previous was $PREVIOUS)? " 10 | read VERSION 11 | 12 | echo 13 | echo "--- Preparing deployer ---" 14 | set -x 15 | rm -rf deploy/shakenfist_ci.egg-info deploy/gitrepos deploy/.tox 16 | find deploy/ansible/terraform -type f -name "*tfstate*" -exec rm {} \; || true 17 | find deploy/ansible/terraform -type d -name ".terraform" -exec rm -rf {} \; || true 18 | 19 | rm -f deploy.tgz 20 | tar cvzf deploy.tgz deploy 21 | 22 | rm -f docs.tgz 23 | tar cvzf docs.tgz docs 24 | set +x 25 | 26 | echo 27 | echo "--- Setup ---" 28 | set -x 29 | pip install --upgrade readme-renderer 30 | pip install --upgrade twine 31 | rm -rf build dist *.egg-info 32 | git pull || true 33 | set +x 34 | 35 | echo 36 | echo "--- Setup ---" 37 | echo "Do you want to apply a git tag for this release (yes to tag)?" 38 | read TAG 39 | set -x 40 | 41 | if [ "%$TAG%" == "%yes%" ] 42 | then 43 | git tag -s "v$VERSION" -m "Release v$VERSION" 44 | git push origin "v$VERSION" 45 | fi 46 | 47 | python3 setup.py sdist bdist_wheel 48 | twine check dist/* 49 | set +x 50 | 51 | echo 52 | echo "--- Uploading ---" 53 | echo "This is the point where we push files to pypi. Hit ctrl-c to abort." 54 | read DUMMY 55 | 56 | set -x 57 | twine upload dist/* 58 | set +x -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/guest_ci_tests/test_ubuntu.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | 3 | 4 | class TestUbuntu(base.BaseNamespacedTestCase): 5 | def __init__(self, *args, **kwargs): 6 | kwargs['namespace_prefix'] = 'ubuntu' 7 | super().__init__(*args, **kwargs) 8 | 9 | def setUp(self): 10 | super().setUp() 11 | self.net = self.test_client.allocate_network( 12 | '192.168.242.0/24', True, True, '%s-net' % self.namespace) 13 | self._await_networks_ready([self.net['uuid']]) 14 | 15 | def test_ubuntu_pings(self): 16 | inst = self.test_client.create_instance( 17 | 'ubuntu', 1, 1024, 18 | [ 19 | { 20 | 'network_uuid': self.net['uuid'] 21 | } 22 | ], 23 | [ 24 | { 25 | 'size': 8, 26 | 'base': 'sf://upload/system/ubuntu-2004', 27 | 'type': 'disk' 28 | } 29 | ], None, None) 30 | 31 | self._await_instance_ready(inst['uuid']) 32 | 33 | # NOTE(mikal): Ubuntu 18.04 has a bug where DHCP doesn't always work in the 34 | # cloud image. This is ok though, because we should be using the config drive 35 | # style interface information anyway. 36 | ip = self.test_client.get_instance_interfaces(inst['uuid'])[0]['ipv4'] 37 | self._test_ping(inst['uuid'], self.net['uuid'], ip, True) 38 | -------------------------------------------------------------------------------- /shakenfist/tests/files/dhcp.tmpl: -------------------------------------------------------------------------------- 1 | # dnsmasq configuration is documented at 2 | # https://thekelleys.org.uk/dnsmasq/docs/setup.html 3 | # 4 | # Enabled features: 5 | # provide_dhcp: {{provide_dhcp}} 6 | # provide_nat: {{provide_nat}} 7 | # provide_dns: {{provide_dns}} 8 | 9 | domain-needed # Do not forward DNS lookups for unqualified names 10 | bogus-priv # Do not forward DNS lookups for RFC1918 blocks 11 | no-hosts # Do not use /etc/hosts 12 | no-resolv # Do not use /etc/resolv.conf 13 | filterwin2k # Filter weird windows 2000 queries 14 | 15 | {%- if provide_dns %} 16 | port=53 # Enable DNS 17 | server={{dns_server}} 18 | addn-hosts={{config_dir}}/dnshosts 19 | expand-hosts 20 | {%- else %} 21 | port=0 # Disable DNS 22 | {%- endif %} 23 | 24 | pid-file={{config_dir}}/pid 25 | 26 | interface={{interface}} 27 | listen-address={{router}} 28 | 29 | domain={{namespace}}.{{zone}} 30 | local=/{{namespace}}.{{zone}}/ 31 | 32 | {%- if provide_dhcp %} 33 | dhcp-leasefile={{config_dir}}/leases 34 | dhcp-range={{dhcp_start}},static,{{netmask}},{{broadcast}},1h 35 | 36 | # DHCP options are documented at 37 | # https://blog.abysm.org/2020/06/human-readable-dhcp-options-for-dnsmasq/ 38 | {%- if provide_dns %} 39 | dhcp-option=6,{{router}} 40 | {%- elif provide_nat %} 41 | dhcp-option=6,{{dns_server}} 42 | {%- endif %} 43 | dhcp-option=1,{{netmask}} 44 | dhcp-option=15,{{namespace}}.{{zone}} 45 | dhcp-option=26,{{mtu}} 46 | dhcp-hostsfile={{config_dir}}/hosts 47 | {%- endif %} -------------------------------------------------------------------------------- /shakenfist/deploy/templates/dhcp.tmpl: -------------------------------------------------------------------------------- 1 | # dnsmasq configuration is documented at 2 | # https://thekelleys.org.uk/dnsmasq/docs/setup.html 3 | # 4 | # Enabled features: 5 | # provide_dhcp: {{provide_dhcp}} 6 | # provide_nat: {{provide_nat}} 7 | # provide_dns: {{provide_dns}} 8 | 9 | domain-needed # Do not forward DNS lookups for unqualified names 10 | bogus-priv # Do not forward DNS lookups for RFC1918 blocks 11 | no-hosts # Do not use /etc/hosts 12 | no-resolv # Do not use /etc/resolv.conf 13 | filterwin2k # Filter weird windows 2000 queries 14 | 15 | {%- if provide_dns %} 16 | port=53 # Enable DNS 17 | server={{dns_server}} 18 | addn-hosts={{config_dir}}/dnshosts 19 | expand-hosts 20 | {%- else %} 21 | port=0 # Disable DNS 22 | {%- endif %} 23 | 24 | pid-file={{config_dir}}/pid 25 | 26 | interface={{interface}} 27 | listen-address={{router}} 28 | 29 | domain={{namespace}}.{{zone}} 30 | local=/{{namespace}}.{{zone}}/ 31 | 32 | {%- if provide_dhcp %} 33 | dhcp-leasefile={{config_dir}}/leases 34 | dhcp-range={{dhcp_start}},static,{{netmask}},{{broadcast}},1h 35 | 36 | # DHCP options are documented at 37 | # https://blog.abysm.org/2020/06/human-readable-dhcp-options-for-dnsmasq/ 38 | {%- if provide_dns %} 39 | dhcp-option=6,{{router}} 40 | {%- elif provide_nat %} 41 | dhcp-option=6,{{dns_server}} 42 | {%- endif %} 43 | dhcp-option=1,{{netmask}} 44 | dhcp-option=15,{{namespace}}.{{zone}} 45 | dhcp-option=26,{{mtu}} 46 | dhcp-hostsfile={{config_dir}}/hosts 47 | {%- endif %} -------------------------------------------------------------------------------- /shakenfist/tests/test_config.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from shakenfist.config import SFConfig 4 | from shakenfist.tests import base 5 | 6 | 7 | class ConfigTestCase(base.ShakenFistTestCase): 8 | @mock.patch('socket.getfqdn', return_value='a.b.com') 9 | def test_hostname(self, mock_fqdn): 10 | conf = SFConfig() 11 | mock_fqdn.assert_called() 12 | self.assertEqual('a.b.com', str(conf.NODE_NAME)) 13 | 14 | @mock.patch.dict('os.environ', {'SHAKENFIST_STORAGE_PATH': 'foo'}) 15 | def test_string_override(self): 16 | conf = SFConfig() 17 | self.assertTrue(isinstance(conf.STORAGE_PATH, str)) 18 | self.assertEqual('foo', conf.STORAGE_PATH) 19 | 20 | @mock.patch.dict('os.environ', {'SHAKENFIST_CPU_OVERCOMMIT_RATIO': '1'}) 21 | def test_int_override(self): 22 | conf = SFConfig() 23 | self.assertTrue(isinstance(conf.CPU_OVERCOMMIT_RATIO, float)) 24 | self.assertEqual(1, conf.CPU_OVERCOMMIT_RATIO) 25 | 26 | @mock.patch.dict('os.environ', 27 | {'SHAKENFIST_RAM_SYSTEM_RESERVATION': '4.0'}) 28 | def test_float_override(self): 29 | conf = SFConfig() 30 | self.assertTrue(isinstance(conf.RAM_SYSTEM_RESERVATION, float)) 31 | self.assertEqual(4.0, conf.RAM_SYSTEM_RESERVATION) 32 | 33 | @mock.patch.dict('os.environ', 34 | {'SHAKENFIST_RAM_SYSTEM_RESERVATION': 'banana'}) 35 | def test_bogus_override(self): 36 | self.assertRaises(ValueError, SFConfig) 37 | -------------------------------------------------------------------------------- /docs/developer_guide/api_reference/clusteroperations.md: -------------------------------------------------------------------------------- 1 | # Cluster operations 2 | 3 | Much like [agent operations](/developer_guide/api_reference/agentoperations), 4 | since v0.8 Shaken Fist has exposed its internal work queuing system via the REST 5 | API to external viewers. Internal work is queued with a series of objects called 6 | cluster operations. While you cannot directly create a cluster operation, 7 | being able to see the internal processes Shaken Fist is using to complete 8 | requests is quite useful, especially if you're trying to determine what the 9 | cluster is currently doing. 10 | 11 | In general, when a Shaken Fist component wants to request another Shaken Fist 12 | component perform an action, a cluster operation is created and queued in the 13 | database. That other component is regularly polling for work to complete, and 14 | will execute an operation as soon as it has an opportunity and the dependencies 15 | for that work have been met. An important edge case is that the Shaken Fist 16 | component can also queue work for itself, if that work is going to take longer 17 | than the component is willing to wait while performing its primary request. So 18 | for example if you request an instance start, a series of cluster operations 19 | will be created to do things like fetch the required images, plug into the 20 | virtual network, and create the actual instance. 21 | 22 | Cluster operations cannot be directly looked up. They are accessible from the 23 | objects they act upon, and are currently exposed for artifacts, instances, and networks. -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_disk_specs.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | from shakenfist_client import apiclient 3 | 4 | 5 | class TestDiskSpecifications(base.BaseNamespacedTestCase): 6 | def __init__(self, *args, **kwargs): 7 | kwargs['namespace_prefix'] = 'diskspecs' 8 | super().__init__(*args, **kwargs) 9 | 10 | def test_default(self): 11 | inst = self.test_client.create_instance( 12 | 'test-default-disk', 1, 1024, None, 13 | [ 14 | { 15 | 'size': 8, 16 | 'base': base.CLUSTER_CI_IMAGE, 17 | 'type': 'disk' 18 | } 19 | ], None, None) 20 | 21 | self.assertIsNotNone(inst['uuid']) 22 | self._await_instance_ready(inst['uuid']) 23 | 24 | results = self._await_command(inst['uuid'], 'df -h') 25 | self.assertEqual(0, results['return-code']) 26 | self.assertEqual('', results['stderr']) 27 | self.assertTrue('vda' in results['stdout']) 28 | 29 | def test_bad_bus(self): 30 | self.assertRaises( 31 | apiclient.RequestMalformedException, 32 | self.test_client.create_instance, 33 | 'test-bad-bus-disk', 1, 1024, None, 34 | [ 35 | { 36 | 'size': 8, 37 | 'base': base.CLUSTER_CI_IMAGE, 38 | 'type': 'disk', 39 | 'bus': 'banana' 40 | } 41 | ], None, None) 42 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/guest_ci_tests/test_disks.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | 3 | 4 | class TestDisks(base.BaseNamespacedTestCase): 5 | """Make sure instances boot under various configurations.""" 6 | 7 | def __init__(self, *args, **kwargs): 8 | kwargs['namespace_prefix'] = 'disks' 9 | super().__init__(*args, **kwargs) 10 | 11 | def setUp(self): 12 | super().setUp() 13 | self.net = self.test_client.allocate_network( 14 | '192.168.242.0/24', True, True, self.namespace) 15 | self._await_networks_ready([self.net['uuid']]) 16 | 17 | def test_boot_nvme(self): 18 | self.skipTest('This test is flakey in CI for reasons I do not understand.') 19 | inst = self.test_client.create_instance( 20 | 'test-cirros-boot-nvme', 1, 1024, 21 | [ 22 | { 23 | 'network_uuid': self.net['uuid'] 24 | } 25 | ], 26 | [ 27 | { 28 | 'size': 8, 29 | 'base': 'sf://upload/system/ubuntu-2004', 30 | 'type': 'disk', 31 | 'bus': 'nvme' 32 | } 33 | ], None, None) 34 | 35 | self._await_instance_ready(inst['uuid']) 36 | inst = self.test_client.get_instance(inst['uuid']) 37 | self.assertNotIn(inst['agent_system_boot_time'], [None, 0]) 38 | 39 | self.test_client.delete_instance(inst['uuid']) 40 | self._await_instance_deleted(inst['uuid']) 41 | -------------------------------------------------------------------------------- /shakenfist/operations/clusteroperationmapping.py: -------------------------------------------------------------------------------- 1 | from shakenfist.operations.artifact_fetch_op import ArtifactFetchOp 2 | from shakenfist.operations.imgcache_op import ImageCacheOp 3 | from shakenfist.operations.net_iface_ip_op import NetIfaceIPOp 4 | from shakenfist.operations.net_iface_op import NetIfaceOp 5 | from shakenfist.operations.net_ip_op import NetIPOp 6 | from shakenfist.operations.net_macaddr_ip_op import NetMacaddrIPOp 7 | from shakenfist.operations.net_op import NetOp 8 | from shakenfist.operations.node_aop_op import NodeAgentopOp 9 | from shakenfist.operations.node_blob_op import NodeBlobOp 10 | from shakenfist.operations.node_inst_net_iface_op import NodeInstNetIfaceOp 11 | from shakenfist.operations.node_inst_op import NodeInstOp 12 | from shakenfist.operations.node_inst_netdesc_op import NodeInstNetdescOp 13 | from shakenfist.operations.node_inst_snap_op import NodeInstSnapOp 14 | from shakenfist.operations.node_net_op import NodeNetOp 15 | 16 | 17 | OPERATION_NAMES_TO_CLASSES = { 18 | 'artifact_fetch_op': ArtifactFetchOp, 19 | 'imgcache_op': ImageCacheOp, 20 | 'net_iface_ip_op': NetIfaceIPOp, 21 | 'net_iface_op': NetIfaceOp, 22 | 'net_ip_op': NetIPOp, 23 | 'net_macaddr_ip_op': NetMacaddrIPOp, 24 | 'net_op': NetOp, 25 | 'node_aop_op': NodeAgentopOp, 26 | 'node_blob_op': NodeBlobOp, 27 | 'node_inst_net_iface_op': NodeInstNetIfaceOp, 28 | 'node_inst_op': NodeInstOp, 29 | 'node_inst_netdesc_op': NodeInstNetdescOp, 30 | 'node_inst_snap_op': NodeInstSnapOp, 31 | 'node_net_op': NodeNetOp 32 | } 33 | -------------------------------------------------------------------------------- /shakenfist/daemons/network/mtus.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import time 3 | 4 | from shakenfist_utilities import logs # noreorder 5 | 6 | from shakenfist.daemons import daemon 7 | from shakenfist.util import concurrency as util_concurrency 8 | from shakenfist.util import network as util_network 9 | 10 | 11 | LOG, _ = logs.setup(__name__) 12 | 13 | 14 | class Job(util_concurrency.Job): 15 | def __init__(self, name): 16 | super().__init__() 17 | self.name = name 18 | 19 | self.abort_path = f'/run/sf/net-{name}.abort' 20 | daemon.clear_abort_path(self.abort_path) 21 | 22 | def execute(self): 23 | LOG.info('Starting MTU watchdog') 24 | last_loop = 0 25 | 26 | while daemon.check_abort_path(self.abort_path): 27 | if time.time() - last_loop < 30: 28 | time.sleep(1) 29 | continue 30 | 31 | last_loop = time.time() 32 | LOG.info('Validating network interface MTUs') 33 | 34 | by_mtu = defaultdict(list) 35 | for iface, mtu in util_network.get_interface_mtus(): 36 | by_mtu[mtu].append(iface) 37 | 38 | for mtu in sorted(by_mtu): 39 | log = LOG.with_fields({ 40 | 'mtu': mtu, 41 | 'interfaces': by_mtu[mtu] 42 | }) 43 | if mtu < 1501: 44 | log.warning('Interface MTU is 1500 bytes or less') 45 | else: 46 | log.debug('Interface MTU is normal') 47 | -------------------------------------------------------------------------------- /docs/user_guide/authentication.md: -------------------------------------------------------------------------------- 1 | # Authentication 2 | 3 | While there is a detailed discussion of the Shaken Fist authentication system 4 | in the [developer guide](/developer_guide/authentication/), that is likely more 5 | detail than a day to day user of Shaken Fist is interested in. This page 6 | therefore provides the details in a simpler and more direct form. 7 | 8 | As a user of Shaken Fist, the administrator of the cluster you are using will 9 | have created a _namespace_ to contain the resources you create in Shaken Fist. 10 | This namespace can have several authentication keys associated with it, which 11 | are simply strings you pass to Shaken Fist to prove your identity, much like API 12 | keys for GitHub or other web services. Normally your administrator will create 13 | a key per user, but its also possible to create a key per system -- there are 14 | not real rules imposed by Shaken Fist on when you should create a key not reuse 15 | an existing one. 16 | 17 | For most users, this key will be provided in the form of a file you should place 18 | at `.shakenfist` in your home direct. An example file might be: 19 | 20 | ``` 21 | { 22 | "namespace": "mynamespace", 23 | "key": "oisoSe7T", 24 | "apiurl": "https://shakenfist/api" 25 | } 26 | ``` 27 | 28 | This file specifies your namespace, the key you will use to authenticate, and 29 | the location of the API server for that Shaken Fist cluster. 30 | 31 | Once you have that file in the correct location, the the Shaken Fist command line 32 | client and API client will function correctly with no further configuration 33 | required. -------------------------------------------------------------------------------- /shakenfist/tests/files/cirros-MD5SUMS-0.3.4: -------------------------------------------------------------------------------- 1 | 4c3f5015257cc3b0af28b424fc73d3f2 cirros-0.3.4-arm-initramfs 2 | ad7a723018f2168e7517839306063792 cirros-0.3.4-arm-kernel 3 | 5fd856d158dc6787264a1b4236126aab cirros-0.3.4-arm-lxc.tar.gz 4 | 1f025d5f56e544e77b06a57e489af2dd cirros-0.3.4-arm-rootfs.img.gz 5 | 1a2aea370f9f2d95d837f6b84bef658d cirros-0.3.4-arm-uec.tar.gz 6 | 79b4436412283bb63c2cba4ac796bcd9 cirros-0.3.4-i386-disk.img 7 | ffb77b84aeda89bf9f82c34f427715b8 cirros-0.3.4-i386-initramfs 8 | ce2066231eef2494a4f5bbefcc570464 cirros-0.3.4-i386-kernel 9 | ffe02ef2b49cb34881f96e2b4c69383c cirros-0.3.4-i386-lxc.tar.gz 10 | 9018a79b097adbf2717f8835f412d96d cirros-0.3.4-i386-rootfs.img.gz 11 | c1630b33986cc09df2e0cbca7f5c5346 cirros-0.3.4-i386-uec.tar.gz 12 | 453b21916c47c6ff8c615f8a5f7b76d2 cirros-0.3.4-powerpc-disk.img 13 | d2cfe82be1ac23aff253d4faf4d3366c cirros-0.3.4-powerpc-initramfs 14 | e549dc08be00f7307b810d33157a0c6c cirros-0.3.4-powerpc-kernel 15 | 7e3a147b5665a1e14fe6d0ffef9ca410 cirros-0.3.4-powerpc-lxc.tar.gz 16 | 2c2cc7595712a201171b181b6afb6b8f cirros-0.3.4-powerpc-rootfs.img.gz 17 | 2f8a48928a6b0b2c9afbcb7255ff2aec cirros-0.3.4-powerpc-uec.tar.gz 18 | 26c125889d13bb429437d25eac3022b0 cirros-0.3.4-source.tar.gz 19 | ee1eca47dc88f4879d8a229cc70a07c6 cirros-0.3.4-x86_64-disk.img 20 | be575a2b939972276ef675752936977f cirros-0.3.4-x86_64-initramfs 21 | 8a40c862b5735975d82605c1dd395796 cirros-0.3.4-x86_64-kernel 22 | 6554ba2d15401c8caa90212ace985593 cirros-0.3.4-x86_64-lxc.tar.gz 23 | 6852a1d1c64f0507bf1c2f41977f00d7 cirros-0.3.4-x86_64-rootfs.img.gz 24 | 067d511efcc4acd034f7d64b716273bf cirros-0.3.4-x86_64-uec.tar.gz 25 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_system_namespace.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | from shakenfist_client import apiclient 3 | 4 | 5 | class TestSystemNamespace(base.BaseTestCase): 6 | def test_system_namespace(self): 7 | self.assertEqual('system', self.system_client.namespace) 8 | 9 | net = self.system_client.allocate_network( 10 | '192.168.242.0/24', True, True, 11 | 'ci-system-net') 12 | nets = [] 13 | for n in self.system_client.get_networks(): 14 | nets.append(n['uuid']) 15 | self.assertIn(net['uuid'], nets) 16 | 17 | inst = self.system_client.create_instance( 18 | 'test-system-ns', 1, 1024, 19 | [ 20 | { 21 | 'network_uuid': net['uuid'] 22 | } 23 | ], 24 | [ 25 | { 26 | 'size': 8, 27 | 'base': base.CLUSTER_CI_IMAGE, 28 | 'type': 'disk' 29 | } 30 | ], None, None) 31 | 32 | self.assertIsNotNone(inst['uuid']) 33 | self.assertIsNotNone(inst['node']) 34 | 35 | insts = [] 36 | for i in self.system_client.get_instances(): 37 | insts.append(i['uuid']) 38 | self.assertIn(inst['uuid'], insts) 39 | 40 | self.system_client.delete_instance(inst['uuid']) 41 | self._await_instance_deleted(inst['uuid']) 42 | 43 | self.system_client.delete_network(net['uuid']) 44 | 45 | self.assertRaises( 46 | apiclient.UnauthorizedException, 47 | self.system_client.delete_namespace, None) 48 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/network/tasks/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: 4 | - "Template path: {{ template_path }}" 5 | 6 | - name: Install network node dependencies 7 | apt: 8 | name: 9 | - dnsmasq 10 | - dnsmasq-utils 11 | state: latest 12 | register: apt_action 13 | retries: 100 14 | until: apt_action is success or ('Failed to lock apt for exclusive operation' not in apt_action.msg and '/var/lib/dpkg/lock' not in apt_action.msg) 15 | 16 | - name: Disable dnsmasq 17 | service: 18 | name: dnsmasq 19 | enabled: no 20 | state: stopped 21 | 22 | - name: Remove dnsmasq unit 23 | file: 24 | path: /usr/lib/systemd/system/dnsmasq.service 25 | state: absent 26 | notify: Reload systemd 27 | 28 | - name: Copy dhcp config template 29 | copy: 30 | src: "{{ template_path }}/dhcp.tmpl" 31 | dest: /srv/shakenfist/dhcp.tmpl 32 | owner: root 33 | group: root 34 | mode: "0644" 35 | 36 | - name: Copy dhcp hosts template 37 | copy: 38 | src: "{{ template_path }}/dhcphosts.tmpl" 39 | dest: /srv/shakenfist/dhcphosts.tmpl 40 | owner: root 41 | group: root 42 | mode: "0644" 43 | 44 | - name: Copy DNS hosts template 45 | copy: 46 | src: "{{ template_path }}/dnshosts.tmpl" 47 | dest: /srv/shakenfist/dnshosts.tmpl 48 | owner: root 49 | group: root 50 | mode: "0644" 51 | 52 | - name: Configure ipforwarding to be enabled on boot 53 | copy: 54 | content: | 55 | net.ipv4.ip_forward = 1 56 | dest: /etc/sysctl.d/10-sf-ipforwarding.conf 57 | owner: root 58 | mode: ugo+r 59 | 60 | - name: Configure ipforwarding to be enabled now 61 | shell: | 62 | sysctl -w net.ipv4.ip_forward=1 63 | ignore_errors: True -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/pki_internal_ca/tasks/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: 4 | - "Bootstrapping CA for {{ deploy_name }}." 5 | - "CA cert path: {{ ca_cert_path }}" 6 | - "CA key path: {{ ca_key_path }}" 7 | - "CA template path: {{ ca_template_path }}" 8 | 9 | - name: Install CA setup tools 10 | apt: 11 | name: 12 | - gnutls-bin 13 | state: latest 14 | register: apt_action 15 | retries: 100 16 | until: apt_action is success or ('Failed to lock apt for exclusive operation' not in apt_action.msg and '/var/lib/dpkg/lock' not in apt_action.msg) 17 | 18 | - name: Make {{ ca_path }} 19 | file: 20 | path: "{{ ca_path }}" 21 | state: directory 22 | mode: u=rwx 23 | 24 | - name: Check if the CA is already setup 25 | stat: 26 | path: "{{ ca_cert_path }}" 27 | register: ca_cert 28 | 29 | - name: Setup CA template for {{deploy_name}} 30 | template: 31 | src: templates/ca_template.tmpl 32 | dest: "{{ ca_template_path }}" 33 | owner: root 34 | mode: u=r 35 | when: not ca_cert.stat.exists 36 | 37 | - name: Create CA authority key 38 | shell: 39 | umask 277 && certtool --generate-privkey > "{{ ca_key_path }}" 40 | when: not ca_cert.stat.exists 41 | 42 | - name: Set permissions on the CA authority key 43 | file: 44 | path: "{{ ca_key_path }}" 45 | mode: u=r 46 | 47 | - name: Create CA certificate 48 | shell: 49 | certtool --generate-self-signed \ 50 | --template "{{ ca_template_path }}" \ 51 | --load-privkey "{{ ca_key_path }}" \ 52 | --outfile "{{ ca_cert_path }}" 53 | when: not ca_cert.stat.exists 54 | 55 | - name: Set permissions on the CA certificate 56 | file: 57 | path: "{{ ca_cert_path }}" 58 | mode: ugo=r -------------------------------------------------------------------------------- /shakenfist/daemons/sentinel_last/main.py: -------------------------------------------------------------------------------- 1 | # NOTE(mikal): this daemon's role is to notice that you've exited the Shaken 2 | # Fist target run-level and therefore the node is stopping not going missing. 3 | # You should never manually stop this daemon! 4 | import setproctitle 5 | import signal 6 | import time 7 | 8 | from shakenfist_utilities import logs # noreorder 9 | 10 | from shakenfist.config import config 11 | from shakenfist.daemons import daemon 12 | from shakenfist.daemons.daemon import send_systemd_ready 13 | from shakenfist.daemons.daemon import send_systemd_stopping 14 | from shakenfist.node import Node 15 | 16 | 17 | LOG, _ = logs.setup(__name__) 18 | ABORT_PATH = '/run/sf/sentinel-last.abort' 19 | 20 | 21 | def exit_gracefully(sig, _frame): 22 | if sig == signal.SIGTERM: 23 | LOG.info('Received SIGTERM') 24 | daemon.set_abort_path(ABORT_PATH, 'from sentinel last exit_gracefully') 25 | 26 | 27 | signal.signal(signal.SIGTERM, exit_gracefully) 28 | 29 | 30 | def main(): 31 | daemon.clear_abort_path(ABORT_PATH) 32 | setproctitle.setproctitle('sf-sentinel-last') 33 | LOG.info('Started') 34 | 35 | n = Node.from_db(config.NODE_NAME) 36 | n.set_daemon_state('sentinel-last', Node.DAEMON_STATE_RUNNING) 37 | send_systemd_ready() 38 | 39 | while daemon.check_abort_path(ABORT_PATH): 40 | LOG.debug('Checking in') 41 | Node.observe_this_node() 42 | time.sleep(15) 43 | 44 | LOG.info('Stopping') 45 | send_systemd_stopping() 46 | n.set_daemon_state('sentinel-last', Node.DAEMON_STATE_STOPPED) 47 | n.state = Node.STATE_STOPPING 48 | LOG.info('Stopped') 49 | 50 | # This is here because sometimes the grpc bits don't shut down cleanly 51 | # by themselves. 52 | raise SystemExit(0) 53 | -------------------------------------------------------------------------------- /shakenfist/tests/test_networkinterface.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from shakenfist.network.interface import NetworkInterfaces 4 | from shakenfist.tests import base 5 | 6 | 7 | GET_ALL_INTERFACES = [ 8 | (None, { 9 | 'uuid': 'ifaceuuid', 10 | 'instance_uuid': 'instuuid', 11 | 'network_uuid': 'netuuid', 12 | 'macaddr': '1a:91:64:d2:15:39', 13 | 'ipv4': '127.0.0.5', 14 | 'order': 0, 15 | 'model': 'virtio', 16 | 'version': 4 17 | }), 18 | (None, { 19 | 'uuid': 'ifaceuuid2', 20 | 'instance_uuid': 'instuuid', 21 | 'network_uuid': 'netuuid', 22 | 'macaddr': '1a:91:64:d2:15:40', 23 | 'ipv4': '127.0.0.6', 24 | 'order': 1, 25 | 'model': 'virtio', 26 | 'version': 4 27 | }) 28 | ] 29 | 30 | JUST_INTERFACES = [ 31 | { 32 | 'uuid': 'ifaceuuid', 33 | 'instance_uuid': 'instuuid', 34 | 'network_uuid': 'netuuid', 35 | 'macaddr': '1a:91:64:d2:15:39', 36 | 'ipv4': '127.0.0.5', 37 | 'order': 0, 38 | 'model': 'virtio', 39 | 'version': 4 40 | }, 41 | { 42 | 'uuid': 'ifaceuuid2', 43 | 'instance_uuid': 'instuuid', 44 | 'network_uuid': 'netuuid', 45 | 'macaddr': '1a:91:64:d2:15:40', 46 | 'ipv4': '127.0.0.6', 47 | 'order': 1, 48 | 'model': 'virtio', 49 | 'version': 4 50 | } 51 | ] 52 | 53 | 54 | class NetworkInterfaceTestCase(base.ShakenFistTestCase): 55 | @mock.patch('shakenfist.etcd.get', side_effect=JUST_INTERFACES) 56 | @mock.patch('shakenfist.etcd.get_all', return_value=GET_ALL_INTERFACES) 57 | def test_ni_iterator_mocking(self, mock_get_all, mock_get): 58 | self.assertEqual(2, len(list(NetworkInterfaces([])))) 59 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_events.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | 3 | 4 | # NOTE(mikal): yes, I know not all objects are represented here yet. 5 | class TestEvents(base.BaseNamespacedTestCase): 6 | def __init__(self, *args, **kwargs): 7 | kwargs['namespace_prefix'] = 'events' 8 | super().__init__(*args, **kwargs) 9 | 10 | def setUp(self): 11 | super().setUp() 12 | self.net_one = self.test_client.allocate_network( 13 | '192.168.242.0/24', True, True, '%s-net-one' % self.namespace, 14 | provide_dns=True) 15 | self._await_networks_ready([self.net_one['uuid']]) 16 | 17 | # NOTE(mikal): needs ArtifactsUrlRefEndpoint implemented first 18 | # def test_artifact_events(self): 19 | # a = self.test_client.get_artifact(base.CLUSTER_CI_IMAGE) 20 | # self.assertNotEqual(0, len(self.test_client.get_artifact_events(a['uuid']))) 21 | 22 | def test_network_events(self): 23 | self.assertNotEqual( 24 | 0, len(self.test_client.get_network_events(self.net_one['uuid']))) 25 | 26 | def test_instance_events(self): 27 | inst1 = self.test_client.create_instance( 28 | 'test-instance-events', 1, 1024, 29 | [ 30 | { 31 | 'network_uuid': self.net_one['uuid'] 32 | } 33 | ], 34 | [ 35 | { 36 | 'size': 8, 37 | 'base': base.CLUSTER_CI_IMAGE, 38 | 'type': 'disk' 39 | } 40 | ], None, None) 41 | 42 | # Wait for the instance agent to report in 43 | self._await_instance_ready(inst1['uuid']) 44 | 45 | self.assertNotEqual( 46 | 0, len(self.test_client.get_instance_events(inst1['uuid']))) 47 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/smoke_ci_tests/test_events.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | 3 | 4 | # NOTE(mikal): yes, I know not all objects are represented here yet. 5 | class TestEvents(base.BaseNamespacedTestCase): 6 | def __init__(self, *args, **kwargs): 7 | kwargs['namespace_prefix'] = 'events' 8 | super().__init__(*args, **kwargs) 9 | 10 | def setUp(self): 11 | super().setUp() 12 | self.net_one = self.test_client.allocate_network( 13 | '192.168.242.0/24', True, True, '%s-net-one' % self.namespace, 14 | provide_dns=True) 15 | self._await_networks_ready([self.net_one['uuid']]) 16 | 17 | # NOTE(mikal): needs ArtifactsUrlRefEndpoint implemented first 18 | # def test_artifact_events(self): 19 | # a = self.test_client.get_artifact(base.CLUSTER_CI_IMAGE) 20 | # self.assertNotEqual(0, len(self.test_client.get_artifact_events(a['uuid']))) 21 | 22 | def test_network_events(self): 23 | self.assertNotEqual( 24 | 0, len(self.test_client.get_network_events(self.net_one['uuid']))) 25 | 26 | def test_instance_events(self): 27 | inst1 = self.test_client.create_instance( 28 | 'test-instance-events', 1, 1024, 29 | [ 30 | { 31 | 'network_uuid': self.net_one['uuid'] 32 | } 33 | ], 34 | [ 35 | { 36 | 'size': 8, 37 | 'base': base.CLUSTER_CI_IMAGE, 38 | 'type': 'disk' 39 | } 40 | ], None, None) 41 | 42 | # Wait for the instance agent to report in 43 | self._await_instance_ready(inst1['uuid']) 44 | 45 | self.assertNotEqual( 46 | 0, len(self.test_client.get_instance_events(inst1['uuid']))) 47 | -------------------------------------------------------------------------------- /shakenfist/external_api/clusteroperation.py: -------------------------------------------------------------------------------- 1 | # Documentation state: 2 | # - Has metadata calls: deliberately not implemented 3 | # - OpenAPI complete: yes 4 | # - Covered in user or operator docs: 5 | # - API reference docs exist: 6 | # - and link to OpenAPI docs: 7 | # - and include examples: 8 | # - Has complete CI coverage: 9 | from flasgger import swag_from 10 | from shakenfist_utilities import api as sf_api 11 | from shakenfist_utilities import logs # noreorder 12 | 13 | from shakenfist.constants import OPERATION_NAMES_TO_CLASSES 14 | from shakenfist.constants import get_object_class 15 | from shakenfist.daemons import daemon 16 | from shakenfist.external_api import base as api_base 17 | 18 | 19 | LOG, HANDLER = logs.setup(__name__) 20 | daemon.set_log_level(LOG, 'api') 21 | 22 | 23 | clusteroperation_get_example = """{ 24 | }""" 25 | 26 | 27 | class ClusterOperationEndpoint(sf_api.Resource): 28 | @swag_from(api_base.swagger_helper( 29 | 'clusteroperations', 'Get information for a cluster operation.', 30 | [ 31 | ('operation_type', 'query', 'uuid', 'The UUID of the operation.', True), 32 | ('operation_uuid', 'query', 'uuid', 'The UUID of the operation.', True) 33 | ], 34 | [(200, 'Information about a single cluster operation.', clusteroperation_get_example), 35 | (404, 'Operation not found.', None)])) 36 | @api_base.verify_token 37 | @api_base.log_token_use 38 | def get(self, operation_type=None, operation_uuid=None): 39 | if operation_type not in OPERATION_NAMES_TO_CLASSES: 40 | return sf_api.error(404, 'operation type not found') 41 | op = get_object_class(operation_type).from_db(operation_uuid) 42 | if not op: 43 | return sf_api.error(404, 'operation not found') 44 | return op.external_view() 45 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [master, ] 6 | pull_request: 7 | # The branches below must be a subset of the branches above 8 | branches: [master] 9 | schedule: 10 | - cron: '0 17 * * 2' 11 | 12 | jobs: 13 | analyze: 14 | name: Analyze 15 | runs-on: [self-hosted, static] 16 | 17 | steps: 18 | - name: Checkout repository 19 | uses: actions/checkout@v4 20 | with: 21 | # We must fetch at least the immediate parents so that if this is 22 | # a pull request then we can checkout the head. 23 | fetch-depth: 2 24 | 25 | # If this run was triggered by a pull request event, then checkout 26 | # the head of the pull request instead of the merge commit. 27 | - run: git checkout HEAD^2 28 | if: ${{ github.event_name == 'pull_request' }} 29 | 30 | # Initializes the CodeQL tools for scanning. 31 | - name: Initialize CodeQL 32 | uses: github/codeql-action/init@v3 33 | # Override language selection by uncommenting this and choosing your languages 34 | # with: 35 | # languages: go, javascript, csharp, python, cpp, java 36 | 37 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 38 | # If this step fails, then you should remove it and run the build manually (see below) 39 | - name: Autobuild 40 | uses: github/codeql-action/autobuild@v3 41 | 42 | # ℹ️ Command-line programs to run using the OS shell. 43 | # 📚 https://git.io/JvXDl 44 | 45 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 46 | # and modify them (or add more) to build your code if your project 47 | # uses a compiled language 48 | 49 | #- run: | 50 | # make bootstrap 51 | # make release 52 | 53 | - name: Perform CodeQL Analysis 54 | uses: github/codeql-action/analyze@v3 55 | -------------------------------------------------------------------------------- /shakenfist/daemons/sentinel_first/main.py: -------------------------------------------------------------------------------- 1 | # NOTE(mikal): this daemon's role is to notice that the node has been started 2 | # or shutdown. You should never manually stop this daemon! 3 | # 4 | # This daemon starts after the database service, so it uses the database 5 | # service for both etcd and MariaDB access like other daemons. 6 | import setproctitle 7 | import signal 8 | import time 9 | 10 | from shakenfist_utilities import logs # noreorder 11 | 12 | from shakenfist.config import config 13 | from shakenfist.daemons import daemon 14 | from shakenfist.daemons.daemon import send_systemd_ready 15 | from shakenfist.daemons.daemon import send_systemd_stopping 16 | from shakenfist.node import Node 17 | 18 | 19 | LOG, _ = logs.setup(__name__) 20 | ABORT_PATH = '/run/sf/sentinel-first.abort' 21 | 22 | 23 | def exit_gracefully(sig, _frame): 24 | if sig == signal.SIGTERM: 25 | LOG.info('Received SIGTERM') 26 | daemon.set_abort_path( 27 | ABORT_PATH, 'from sentinel first exit_gracefully') 28 | 29 | 30 | signal.signal(signal.SIGTERM, exit_gracefully) 31 | 32 | 33 | def main(): 34 | daemon.clear_abort_path(ABORT_PATH) 35 | setproctitle.setproctitle('sf-sentinel-first') 36 | LOG.info('Started') 37 | 38 | n = Node.from_db(config.NODE_NAME) 39 | n.set_daemon_state('sentinel-first', Node.DAEMON_STATE_RUNNING) 40 | n.state = Node.STATE_DEGRADED 41 | send_systemd_ready() 42 | 43 | while daemon.check_abort_path(ABORT_PATH): 44 | LOG.debug('Checking in') 45 | Node.observe_this_node() 46 | time.sleep(15) 47 | 48 | LOG.info('Stopping') 49 | send_systemd_stopping() 50 | 51 | n.set_daemon_state('sentinel-first', Node.DAEMON_STATE_STOPPED) 52 | n.state = Node.STATE_STOPPED 53 | LOG.info('Stopped') 54 | 55 | # This is here because sometimes the grpc bits don't shut down cleanly 56 | # by themselves. 57 | raise SystemExit(0) 58 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_object_names.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | 3 | 4 | class TestObjectNames(base.BaseNamespacedTestCase): 5 | """Make sure instances boot under various configurations.""" 6 | 7 | def __init__(self, *args, **kwargs): 8 | kwargs['namespace_prefix'] = 'namespace_test' 9 | super().__init__(*args, **kwargs) 10 | 11 | def test_object_names(self): 12 | """Check instances and networks names 13 | 14 | Testing API create_instance() using network name and instance/network 15 | retrieval by name. 16 | """ 17 | 18 | nets = {} 19 | for i in ['barry', 'dave', 'alice']: 20 | n = self.test_client.allocate_network( 21 | '192.168.242.0/24', True, True, i+'_net') 22 | nets[i+'_net'] = n['uuid'] 23 | 24 | for name, uuid in nets.items(): 25 | n = self.system_client.get_network(name) 26 | self.assertEqual(uuid, n['uuid']) 27 | 28 | self._await_networks_ready(['barry_net']) 29 | 30 | inst_uuids = {} 31 | for name in ['barry', 'dave', 'trouble-writing-tests']: 32 | new_inst = self.test_client.create_instance( 33 | name, 1, 1024, 34 | [ 35 | { 36 | 'network_uuid': 'barry_net' 37 | } 38 | ], 39 | [ 40 | { 41 | 'size': 8, 42 | 'base': base.CLUSTER_CI_IMAGE, 43 | 'type': 'disk' 44 | } 45 | ], None, None, namespace=self.namespace) 46 | inst_uuids[name] = new_inst['uuid'] 47 | 48 | # Get instance by name 49 | for name, uuid in inst_uuids.items(): 50 | inst = self.system_client.get_instance(name) 51 | self.assertEqual(uuid, inst['uuid']) 52 | -------------------------------------------------------------------------------- /docs/user_guide/events.md: -------------------------------------------------------------------------------- 1 | # Events 2 | 3 | Events are Shaken Fist's audit log mechanism. Many operations, ranging from 4 | creation and subsequent use of an authentication token, to any change in the data 5 | for an object, will result in an event being created in the event log for the 6 | relevant object. Importantly, regular "usage events" are also emitted, which we 7 | expect would form the basis for a consumption based billing system. 8 | 9 | Events may be requested using the `sf-client ...object... events` command, for 10 | example `sf-client artifact events ...uuid...` will return the events for the 11 | relevant artifact. 12 | 13 | The schema for events is still in flux, so be careful implementing automated 14 | systems which consume events. This will remain true until we are more confident 15 | that all relevant audit lock events are being collected. At that point we will 16 | standardize and stabilize the interface. 17 | 18 | As of v0.7, each event log entry has a type. The currently implemented types 19 | are: 20 | 21 | * audit: audit log entries such as object creation or deletion, and authentication. 22 | * mutate: object modifications which are not an audit log entry, such as minor updates. 23 | * status: status messages useful to a user, such as progress of fetching an image. 24 | * usage: billing information. 25 | * resources: cluster resource usage information of interest to an operator. 26 | * prune: messages relating to pruning of other message types. 27 | * historic: events from before the type system was introduced. 28 | 29 | For each of these types, an operator can configure a retention period. The 30 | default periods are: 31 | 32 | * audit (MAX_AUDIT_EVENT_AGE): 90 days. 33 | * mutate (MAX_MUTATE_EVENT_AGE): 90 days. 34 | * status (MAX_STATUS_EVENT_AGE): 7 days. 35 | * usage (MAX_USAGE_EVENT_AGE): 30 days. 36 | * resources (MAX_RESOURCES_EVENT_AGE): 7 days. 37 | * prune (MAX_PRUNE_EVENT_AGE): 30 days. 38 | * historic (MAX_HISTORIC_EVENT_AGE): 90 days. 39 | 40 | To permanently retain a type of event log entry, set the corresponding configuration 41 | value to -1. -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/templates/etc_prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | external_labels: 3 | monitor: 'shakenfist' 4 | origin_prometheus: {{ deploy_name }} 5 | 6 | scrape_configs: 7 | - job_name: 'node' 8 | static_configs: 9 | - targets: [ 10 | {% for svr in groups.allsf %} 11 | '{{hostvars[svr]['node_name']}}:9100', 12 | {% endfor %} 13 | ] 14 | - job_name: 'shakenfist' 15 | static_configs: 16 | - targets: [ 17 | {% for svr in groups.sf_prometheus_exporters %} 18 | '{{hostvars[svr]['node_name']}}:13001', 19 | {% endfor %} 20 | ] 21 | # metric_relabel_configs: 22 | # - source_labels: [__name__] 23 | # regex: '(python\w*|process_\w*)' 24 | # action: drop 25 | - job_name: 'shakenfist_eventlog' 26 | static_configs: 27 | - targets: [ 28 | {% for svr in groups.eventlog_node %} 29 | '{{hostvars[svr]['node_name']}}:13002', 30 | {% endfor %} 31 | ] 32 | # metric_relabel_configs: 33 | # - source_labels: [__name__] 34 | # regex: (?i)(etcd_mvcc_db_total_size_in_bytes|etcd_network_client_grpc_received_bytes_total|etcd_network_client_grpc_sent_bytes_total|etcd_disk_wal_fsync_duration_seconds) 35 | # action: keep 36 | - job_name: 'etcd' 37 | static_configs: 38 | - targets: [ 39 | {% for svr in groups.etcd_master %} 40 | '{{hostvars[svr]['node_name']}}:2379', 41 | {% endfor %} 42 | ] 43 | # metric_relabel_configs: 44 | # - source_labels: [__name__] 45 | # regex: (?i)(etcd_mvcc_db_total_size_in_bytes|etcd_network_client_grpc_received_bytes_total|etcd_network_client_grpc_sent_bytes_total|etcd_disk_wal_fsync_duration_seconds) 46 | # action: keep 47 | - job_name: 'shakenfist_database' 48 | static_configs: 49 | - targets: [ 50 | {% for svr in groups.etcd_master %} 51 | '{{hostvars[svr]['node_name']}}:13006', 52 | {% endfor %} 53 | ] -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py38,flake8,cover 3 | isolated_build = true 4 | 5 | [testenv] 6 | extras = 7 | test 8 | sitepackages = true 9 | commands = 10 | # NOTE: you can run any command line tool here - not just tests 11 | pip list 12 | stestr run {posargs} 13 | stestr slowest 14 | allowlist_externals = 15 | bash 16 | find 17 | rm 18 | env 19 | tar 20 | setenv = 21 | VIRTUAL_ENV={envdir} 22 | LANGUAGE=en_US 23 | LC_ALL=en_US.utf-8 24 | OS_STDOUT_CAPTURE=1 25 | OS_STDERR_CAPTURE=1 26 | OS_TEST_TIMEOUT=160 27 | PYTHONDONTWRITEBYTECODE=1 28 | SHAKENFIST_ETCD_HOST='127.0.0.1' 29 | SHAKENFIST_EVENTLOG_SUPPRESS_GRPC=1 30 | SHAKENFIST_LOG_TO_STDOUT=1 31 | 32 | [testenv:flake8] 33 | description = 34 | Run style checks on the changes made since HEAD~. 35 | envdir = {toxworkdir}/shared 36 | commands = 37 | bash tools/flake8wrap.sh -HEAD 38 | 39 | [testenv:docs] 40 | description = 41 | Ensure the docs build. 42 | envdir = {toxworkdir}/shared 43 | commands = 44 | mkdocs build {posargs} 45 | 46 | [testenv:py38] 47 | description = 48 | Run python3.8 unit tests 49 | 50 | [testenv:failing] 51 | description = 52 | List failed tests 53 | envdir = {toxworkdir}/shared 54 | commands = 55 | stestr failing 56 | 57 | [testenv:cover] 58 | description = 59 | Generate a coverage report in cover/. 60 | setenv = 61 | PYTHON=coverage run --source shakenfist --parallel-mode 62 | commands = 63 | coverage erase 64 | stestr run {posargs} 65 | coverage combine 66 | coverage html -d cover 67 | coverage report 68 | 69 | [testenv:mypy] 70 | description = 71 | Run type checks on the schema module (incremental mypy rollout). 72 | envdir = {toxworkdir}/shared 73 | commands = 74 | mypy --strict --ignore-missing-imports shakenfist/schema/sqlalchemy.py 75 | mypy --strict --ignore-missing-imports shakenfist/schema/object_state.py 76 | mypy --strict --ignore-missing-imports shakenfist/schema/object_types.py 77 | mypy --strict --ignore-missing-imports shakenfist/schema/external_view.py 78 | mypy --strict --ignore-missing-imports --follow-imports=silent shakenfist/mariadb.py 79 | -------------------------------------------------------------------------------- /shakenfist/external_api/util.py: -------------------------------------------------------------------------------- 1 | from shakenfist_utilities import api as sf_api # noreorder 2 | from shakenfist_utilities import logs # noreorder 3 | 4 | from shakenfist import ipam 5 | from shakenfist.network import network 6 | from shakenfist.daemons import daemon 7 | from shakenfist.instance import Instance 8 | from shakenfist.network.interface import NetworkInterface 9 | from shakenfist.util.access_tokens import parse_jwt_identity 10 | 11 | 12 | LOG, HANDLER = logs.setup(__name__) 13 | daemon.set_log_level(LOG, 'api') 14 | 15 | 16 | def assign_floating_ip(ni): 17 | # Address is allocated and added to the record here, so the job has it later. 18 | fn = network.floating_network() 19 | ni.floating = fn.ipam.reserve_random_free_address( 20 | ni.unique_label(), ipam.RESERVATION_TYPE_FLOATING, '') 21 | 22 | 23 | def assign_routed_ip(n): 24 | # Address is allocated and then returned, as there is no network interface 25 | # to associate it with. 26 | fn = network.floating_network() 27 | return fn.ipam.reserve_random_free_address( 28 | n.unique_label(), ipam.RESERVATION_TYPE_ROUTED, '') 29 | 30 | 31 | def safe_get_network_interface(interface_uuid): 32 | ni = NetworkInterface.from_db(interface_uuid) 33 | if not ni: 34 | return None, None, sf_api.error(404, 'interface not found') 35 | 36 | log = LOG.with_fields({ 37 | 'network': ni.network_uuid, 38 | 'networkinterface': ni.uuid 39 | }) 40 | 41 | n = network.Network.from_db(ni.network_uuid) 42 | if not n: 43 | log.info('Network not found or deleted') 44 | return None, None, sf_api.error(404, 'interface network not found') 45 | 46 | if parse_jwt_identity()[0] not in [n.namespace, 'system']: 47 | log.info('Interface not found, failed ownership test') 48 | return None, None, sf_api.error(404, 'interface not found') 49 | 50 | i = Instance.from_db(ni.instance_uuid) 51 | if parse_jwt_identity()[0] not in [i.namespace, 'system']: 52 | log.with_fields({'instance': i}).info( 53 | 'Instance not found, failed ownership test') 54 | return None, None, sf_api.error(404, 'interface not found') 55 | 56 | return ni, n, None 57 | -------------------------------------------------------------------------------- /shakenfist/schema/operations/baseclusteroperation.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | from pydantic import BaseModel 4 | from pydantic import field_serializer 5 | from pydantic import UUID4 6 | 7 | 8 | class PRIORITY(Enum): 9 | user_waiting = 10 10 | user_facing = 20 11 | user_facing_high_io = 25 12 | background = 30 13 | background_high_io = 40 14 | 15 | 16 | class CLUSTER_OPERATIONS(Enum): 17 | node_blob_op = 1 18 | # ... verify_size_and_checksum 19 | # ... ensure_local 20 | 21 | node_inst_op = 2 22 | # ... collect_billing_statistics 23 | # ... health_check_kvm_process 24 | # ... instance_delete 25 | 26 | node_inst_netdesc_op = 3 27 | # ... instance_preflight 28 | # ... instance_start 29 | 30 | artifact_fetch_op = 4 31 | # ... image_fetch 32 | 33 | node_inst_net_iface_op = 5 34 | # ... hot_plug_instance_interface 35 | 36 | node_inst_snap_op = 6 37 | # ... instance_snapshot 38 | 39 | imgcache_op = 7 40 | # ... archive_transcode 41 | 42 | node_aop_op = 8 43 | # ... preflight 44 | 45 | node_net_op = 9 46 | # ... network_destroy 47 | 48 | net_op = 10 49 | # ... network_deploy 50 | # ... network_destroy 51 | # ... network_update_dnsmasq 52 | # ... network_remove_dnsmasq 53 | # ... network_remove_nat 54 | 55 | net_macaddr_ip_op = 11 56 | # ... remove_dhcp_lease 57 | 58 | net_ip_op = 12 59 | # ... route_address 60 | # ... unroute_address 61 | 62 | net_iface_op = 13 63 | # ... interface_float 64 | 65 | net_iface_ip_op = 14 66 | # ... interface_defloat 67 | 68 | 69 | class dependency(BaseModel): 70 | op_type: CLUSTER_OPERATIONS 71 | op_uuid: UUID4 72 | 73 | @field_serializer('op_type') 74 | def serialize_op_type(self, op_type: CLUSTER_OPERATIONS, _info): 75 | return op_type.name 76 | 77 | 78 | def _convert_deps(deps): 79 | if deps and len(deps) > 0: 80 | converted = [] 81 | for op in deps: 82 | if op: 83 | converted.append( 84 | dependency( 85 | op_type=CLUSTER_OPERATIONS[op['op_type']], 86 | op_uuid=op['op_uuid'] 87 | ) 88 | ) 89 | return converted 90 | return None 91 | -------------------------------------------------------------------------------- /shakenfist/schema/external_view.py: -------------------------------------------------------------------------------- 1 | # Pydantic schemas for REST API external views. 2 | # 3 | # These schemas define the structure of API responses and handle field 4 | # transformations (e.g., State object -> state value string). The models 5 | # are used incrementally during migration from dict-based external_view() 6 | # methods to fully typed Pydantic responses. 7 | # 8 | # Migration strategy: 9 | # 1. Add fields to these models as they're migrated 10 | # 2. _external_view() builds the model, dumps to dict, then merges remaining 11 | # fields that haven't been migrated yet 12 | # 3. Once all fields are in the model, _external_view() just returns 13 | # model.model_dump() 14 | 15 | from typing import Any, Dict, Optional 16 | 17 | from pydantic import BaseModel, field_serializer 18 | 19 | from shakenfist.schema.object_state import State 20 | 21 | 22 | class BaseExternalView(BaseModel): 23 | """Base external view with fields common to all object types. 24 | 25 | This model handles the core fields that appear in every object's 26 | external API representation. The state field is transformed from 27 | a State object to just its value string. 28 | 29 | Fields handled here: 30 | - uuid: Object identifier 31 | - state: Transformed from State object to value string 32 | - version: Schema version 33 | - metadata: User-defined key-value pairs 34 | """ 35 | 36 | uuid: str 37 | state: State 38 | version: int 39 | metadata: Dict[str, Any] 40 | 41 | @field_serializer('state') 42 | def serialize_state(self, state: State) -> Optional[str]: 43 | """Transform State object to just its value string for API output.""" 44 | return state.value 45 | 46 | 47 | class BlobExternalView(BaseExternalView): 48 | """External view for Blob objects. 49 | 50 | Extends BaseExternalView with blob-specific fields. As more fields 51 | are migrated from the dict-based external_view(), add them here. 52 | 53 | Currently handled by Pydantic: 54 | - All fields from BaseExternalView (uuid, state, version, metadata) 55 | 56 | Still handled by dict merge in Blob.external_view(): 57 | - size, modified, fetched_at, depends_on, transcodes, reference_count, 58 | sha512, last_used, checksums, locations, info fields (mime-type, etc.) 59 | """ 60 | pass 61 | -------------------------------------------------------------------------------- /docs/operator_guide/authentication.md: -------------------------------------------------------------------------------- 1 | # Authentication 2 | 3 | ???+ info 4 | 5 | For a detailed discussion of how Shaken Fist authentication works, please see 6 | the discussion in the [developer guide](/developer_guide/authentication/). 7 | 8 | ## Trusts 9 | 10 | ???+ info 11 | 12 | Trusts are a newer way of sharing between namespaces with granular control. 13 | If you instead are interested in making artifacts available to all users of 14 | a Shaken Fist cluster, then you should also consider artifact sharing, which 15 | is discussed in the [artifacts section of the operators guide](artifacts.md). 16 | 17 | The system namespace is special in a Shaken Fist cluster in that it can see 18 | objects in all other namespaces. That is, if you are authenticated as the system 19 | namespace and list instances, you get not only the instances in the system 20 | namespace, but also all those in other namespaces. The same is true for other 21 | namespaced objects such as networks and artifacts. 22 | 23 | In older versions of Shaken Fist this behavior was hard coded, but as of 24 | Shaken Fist v0.7 this is now implemented more flexibly. The system namespace 25 | must still be able to see every other namespace, but you can also create a 26 | "trust" relationship between two arbitrary namespaces to achieve the same result 27 | on a smaller scale. In fact, the system namespace is now simply a default trust 28 | that all other namespaces have a relationship with. 29 | 30 | The Shaken Fist CI system uses these trusts for base images for CI runs. Each 31 | night we rebuild a series of base test images -- Debian 10, Debian 11, Ubuntu 32 | 20.04 and so on. Each Shaken Fist CI job is run in its own namespace, so we needed 33 | a place to store these base images, as well as a mechanism for other CI jobs to 34 | be able to see them. 35 | 36 | What we implemented was: 37 | 38 | * a namespace to store the base images (we called it `ci-images`). 39 | * when our CI conductor creates a new CI runner and associated namespace, it 40 | creates a trust between that ephemeral namespace and the `ci-images` namespace. 41 | * jobs to create new images build them in their local namespace, and then "gift" 42 | them to the `ci-images` namespace via a label. 43 | * jobs which need to boot a test image can now see the images from the `ci-images` 44 | namespace by virtue of this trust relationship. -------------------------------------------------------------------------------- /shakenfist/deploy/ansible_module_ci/002.yml: -------------------------------------------------------------------------------- 1 | # Tests: sf_network 2 | 3 | - hosts: localhost 4 | gather_facts: no 5 | 6 | tasks: 7 | - name: Ensure network doesn't exist at the start 8 | sf_network: 9 | name: "ansibleci-002" 10 | state: "absent" 11 | 12 | - name: Create a test network 13 | sf_network: 14 | netblock: "10.0.0.0/24" 15 | name: "ansibleci-002" 16 | state: present 17 | register: ci_network 18 | 19 | - name: Assert the network is new 20 | assert: 21 | that: ci_network['changed'] 22 | fail_msg: "changed should be true in {{ ci_network }}" 23 | 24 | - name: Assert no errors 25 | assert: 26 | that: not ci_network['failed'] 27 | fail_msg: "failed should be false in {{ ci_network }}" 28 | 29 | - name: Noop create the test network 30 | sf_network: 31 | netblock: "10.0.0.0/24" 32 | name: "ansibleci-002" 33 | state: present 34 | register: ci_network 35 | 36 | - name: Assert the network is unchanged 37 | assert: 38 | that: not ci_network['changed'] 39 | fail_msg: "changed should be false in {{ ci_network }}" 40 | 41 | - name: Assert no errors 42 | assert: 43 | that: not ci_network['failed'] 44 | fail_msg: "failed should be false in {{ ci_network }}" 45 | 46 | - name: Try to change a field 47 | sf_network: 48 | netblock: "10.0.0.0/24" 49 | name: "ansibleci-002" 50 | state: present 51 | dhcp: false 52 | register: ci_network 53 | 54 | - name: Assert the network is new 55 | assert: 56 | that: ci_network['changed'] 57 | fail_msg: "changed should be true in {{ ci_network }}" 58 | 59 | - name: Assert no errors 60 | assert: 61 | that: not ci_network['failed'] 62 | fail_msg: "failed should be false in {{ ci_network }}" 63 | 64 | - name: Delete the network 65 | sf_network: 66 | uuid: "{{ ci_network['meta']['uuid'] }}" 67 | state: absent 68 | register: ci_network 69 | 70 | - name: Assert the network was changed 71 | assert: 72 | that: ci_network['changed'] 73 | fail_msg: "changed should be true in {{ ci_network }}" 74 | 75 | - name: Assert no errors 76 | assert: 77 | that: not ci_network['failed'] 78 | fail_msg: "failed should be false in {{ ci_network }}" -------------------------------------------------------------------------------- /shakenfist/tests/files/ubuntu-MD5SUMS-groovy: -------------------------------------------------------------------------------- 1 | 70e2b444f5ccf687591d55d2dadaf278 *groovy-server-cloudimg-amd64-azure.vhd.zip 2 | ddfb79bcf42f325c4cbb0e716c9cc583 *groovy-server-cloudimg-amd64-disk-kvm.img 3 | 85a1acff6c2827e5c187c944ef522b52 *groovy-server-cloudimg-amd64-lxd.tar.xz 4 | 98781564ea4a1bea6dbc13fb53c46d07 *groovy-server-cloudimg-amd64-root.tar.xz 5 | c81ea958d37ac3ba8fc5878b1676dd07 *groovy-server-cloudimg-amd64-vagrant.box 6 | b3c659970056320f1d9b25dde9faed86 *groovy-server-cloudimg-amd64-wsl.rootfs.tar.gz 7 | 1c19b08060b9feb1cd0e7ee28fd463fb *groovy-server-cloudimg-amd64.img 8 | f9ad068a802e8714af182f2d81aa8f72 *groovy-server-cloudimg-amd64.ova 9 | cd9a3b26318b4e7e5f6a54d73c89a248 *groovy-server-cloudimg-amd64.squashfs 10 | 63c35aa48e4ae768b20ec1e111607279 *groovy-server-cloudimg-amd64.tar.gz 11 | 8be49be11f111431c60ca4241b43c364 *groovy-server-cloudimg-amd64.vmdk 12 | a1518419d4ac91420501eca0bedc6bbf *groovy-server-cloudimg-arm64-lxd.tar.xz 13 | ca161a8df2bcaaa01c318e2a7b13b93c *groovy-server-cloudimg-arm64-root.tar.xz 14 | f4718239a29b4791f3d41ee2cfa2a980 *groovy-server-cloudimg-arm64-wsl.rootfs.tar.gz 15 | a89bdce1a02eda7a707fe29476769022 *groovy-server-cloudimg-arm64.img 16 | 64181e8b45c7f33cd80d9923629edf03 *groovy-server-cloudimg-arm64.squashfs 17 | addabd8c23bfb58baf91cc9984ee21c1 *groovy-server-cloudimg-arm64.tar.gz 18 | fb32507a792c9746f6ac932b97c5be0e *groovy-server-cloudimg-armhf-lxd.tar.xz 19 | a141677ef0645dce145f30b48cfc34a9 *groovy-server-cloudimg-armhf-root.tar.xz 20 | 321a681a86fa3015dc6b690fc4dff497 *groovy-server-cloudimg-armhf.img 21 | 3aeb34573d28755591688d8b253c990b *groovy-server-cloudimg-armhf.squashfs 22 | 22d290686d2c64d5295e0268e458fe2f *groovy-server-cloudimg-armhf.tar.gz 23 | 193f388393296eb1c874558dbaf6ee55 *groovy-server-cloudimg-ppc64el-lxd.tar.xz 24 | 35971ce7142192bdf8f8b4579529ea63 *groovy-server-cloudimg-ppc64el-root.tar.xz 25 | 32485c25341e9180e275b445afde07c0 *groovy-server-cloudimg-ppc64el.img 26 | fc44ec7a2b5748501c7358edb198abf6 *groovy-server-cloudimg-ppc64el.squashfs 27 | d9bf5e5b2bfda25afab9bcc44427f91b *groovy-server-cloudimg-ppc64el.tar.gz 28 | ba7477e9b822eae410da635eda4969a2 *groovy-server-cloudimg-s390x-lxd.tar.xz 29 | c7011e1f2fe455baa1f4a93b59e69ea4 *groovy-server-cloudimg-s390x-root.tar.xz 30 | 55fe26bbe4926ff325aa1bc33d54302f *groovy-server-cloudimg-s390x.img 31 | 386ee39671355e3c8d2f5d0d306262cb *groovy-server-cloudimg-s390x.squashfs 32 | aa0ecad47e24eca6205430cd4708b438 *groovy-server-cloudimg-s390x.tar.gz 33 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/guest_ci_tests/test_multiple_nics.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | from shakenfist_ci import base 4 | 5 | 6 | class TestMultipleNics(base.BaseNamespacedTestCase): 7 | def __init__(self, *args, **kwargs): 8 | kwargs['namespace_prefix'] = 'multinic' 9 | super().__init__(*args, **kwargs) 10 | 11 | def setUp(self): 12 | super().setUp() 13 | self.net_one = self.test_client.allocate_network( 14 | '192.168.242.0/24', True, True, '%s-net-one' % self.namespace) 15 | self.net_two = self.test_client.allocate_network( 16 | '192.168.243.0/24', True, True, '%s-net-two' % self.namespace) 17 | self._await_networks_ready([self.net_one['uuid'], 18 | self.net_two['uuid']]) 19 | 20 | def test_simple(self): 21 | self.skipTest('systemctl says degraded via sf-agent, requires debugging') 22 | 23 | ud = """#!/bin/sh 24 | sudo echo '' > /etc/network/interfaces 25 | sudo echo 'auto eth0' >> /etc/network/interfaces 26 | sudo echo 'iface eth0 inet dhcp' >> /etc/network/interfaces 27 | sudo echo 'auto eth1' >> /etc/network/interfaces 28 | sudo echo 'iface eth1 inet dhcp' >> /etc/network/interfaces 29 | sudo /etc/init.d/S40network restart""" 30 | 31 | inst = self.test_client.create_instance( 32 | 'test-multiple-nics', 1, 1024, 33 | [ 34 | { 35 | 'network_uuid': self.net_one['uuid'] 36 | }, 37 | { 38 | 'network_uuid': self.net_two['uuid'] 39 | } 40 | ], 41 | [ 42 | { 43 | 'size': 8, 44 | 'base': 'sf://upload/system/debian-11', 45 | 'type': 'disk' 46 | } 47 | ], None, str(base64.b64encode(ud.encode('utf-8')), 'utf-8')) 48 | 49 | self.assertIsNotNone(inst['uuid']) 50 | self._await_instance_ready(inst['uuid']) 51 | 52 | ifaces = self.test_client.get_instance_interfaces(inst['uuid']) 53 | self.assertEqual(2, len(ifaces)) 54 | for iface in ifaces: 55 | self.assertEqual('created', iface['state'], 56 | 'Interface %s is not in correct state' % iface['uuid']) 57 | 58 | for iface in ifaces: 59 | self._test_ping( 60 | inst['uuid'], iface['network_uuid'], iface['ipv4'], True) 61 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/tasks/etcd_config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set auth secret in etcd 3 | shell: | 4 | /srv/shakenfist/venv/bin/sf-ctl set-etcd-config \ 5 | AUTH_SECRET_SEED "{{ auth_secret }}" 6 | environment: 7 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 8 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 9 | 10 | - name: Set ram reservation in etcd 11 | shell: | 12 | /srv/shakenfist/venv/bin/sf-ctl set-etcd-config \ 13 | RAM_SYSTEM_RESERVATION "{{ ram_system_reservation }}" 14 | environment: 15 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 16 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 17 | 18 | - name: Set lowest_mtu in etcd 19 | shell: | 20 | /srv/shakenfist/venv/bin/sf-ctl set-etcd-config \ 21 | MAX_HYPERVISOR_MTU "{{ lowest_mtu }}" 22 | environment: 23 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 24 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 25 | 26 | - name: Set DNS server in etcd 27 | shell: | 28 | /srv/shakenfist/venv/bin/sf-ctl set-etcd-config \ 29 | DNS_SERVER "{{ dns_server }}" 30 | environment: 31 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 32 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 33 | 34 | - name: Set HTTP proxy in etcd 35 | shell: | 36 | /srv/shakenfist/venv/bin/sf-ctl set-etcd-config \ 37 | HTTP_PROXY "{{ http_proxy }}" 38 | environment: 39 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 40 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 41 | when: hostvars['localhost']['http_proxy'] | length != 0 42 | 43 | - name: Log additional config 44 | debug: 45 | msg: "{{ extra_config | from_json }}" 46 | 47 | - name: Apply additional config 48 | shell: | 49 | /srv/shakenfist/venv/bin/sf-ctl set-etcd-config \ 50 | "{{item['name']}}" "{{item['value']}}" 51 | environment: 52 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 53 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 54 | loop: "{{ extra_config | from_json }}" 55 | 56 | - name: Verify configuration 57 | shell: | 58 | /srv/shakenfist/venv/bin/sf-ctl verify-config 59 | environment: 60 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 61 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 62 | 63 | - name: Parse back etcd configuration 64 | shell: | 65 | /srv/shakenfist/venv/bin/sf-ctl show-etcd-config 66 | environment: 67 | SHAKENFIST_ETCD_HOST: "{{ etcd_host }}" 68 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD: "True" 69 | register: etcd_config 70 | 71 | - name: Log etcd config 72 | debug: 73 | msg: "{{ etcd_config.stdout }}" -------------------------------------------------------------------------------- /shakenfist/daemons/network/stray_nics.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from shakenfist_utilities import logs # noreorder 4 | 5 | from shakenfist.baseobject import DatabaseBackedObject as dbo 6 | from shakenfist.daemons import daemon 7 | from shakenfist import exceptions 8 | from shakenfist import instance 9 | from shakenfist.network import network 10 | from shakenfist.network.interface import NetworkInterface 11 | from shakenfist.util import concurrency as util_concurrency 12 | 13 | 14 | LOG, _ = logs.setup(__name__) 15 | 16 | 17 | class Job(util_concurrency.Job): 18 | def __init__(self, name): 19 | super().__init__() 20 | self.name = name 21 | 22 | self.abort_path = f'/run/sf/net-{name}.abort' 23 | daemon.clear_abort_path(self.abort_path) 24 | 25 | def execute(self): 26 | LOG.info('Starting NIC IP reaper') 27 | last_loop = 0 28 | 29 | while daemon.check_abort_path(self.abort_path): 30 | if time.time() - last_loop < 30: 31 | time.sleep(1) 32 | continue 33 | 34 | last_loop = time.time() 35 | LOG.info('Scanning for stray network interfaces') 36 | for n in network.Networks([], prefilter='active'): 37 | try: 38 | t = time.time() 39 | for ni_uuid in n.networkinterfaces: 40 | ni = NetworkInterface.from_db(ni_uuid) 41 | if not ni: 42 | continue 43 | 44 | inst = instance.Instance.from_db(ni.instance_uuid) 45 | if not inst: 46 | ni.delete() 47 | LOG.with_fields({ 48 | 'interface': ni, 49 | 'instance': ni.instance_uuid}).info( 50 | 'Deleted stray network interface for missing instance') 51 | else: 52 | s = inst.state 53 | if (s.update_time + 30 < t and 54 | s.value in [dbo.STATE_DELETED, dbo.STATE_ERROR, 'unknown']): 55 | ni.delete() 56 | LOG.with_fields({ 57 | 'interface': ni, 58 | 'instance': ni.instance_uuid}).info( 59 | 'Deleted stray network interface') 60 | 61 | except exceptions.LockException: 62 | pass 63 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/files/grafana/ldap.toml: -------------------------------------------------------------------------------- 1 | # To troubleshoot and get more log info enable ldap debug logging in grafana.ini 2 | # [log] 3 | # filters = ldap:debug 4 | 5 | [[servers]] 6 | # Ldap server host (specify multiple hosts space separated) 7 | host = "127.0.0.1" 8 | # Default port is 389 or 636 if use_ssl = true 9 | port = 389 10 | # Set to true if ldap server supports TLS 11 | use_ssl = false 12 | # Set to true if connect ldap server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS) 13 | start_tls = false 14 | # set to true if you want to skip ssl cert validation 15 | ssl_skip_verify = false 16 | # set to the path to your root CA certificate or leave unset to use system defaults 17 | # root_ca_cert = "/path/to/certificate.crt" 18 | # Authentication against LDAP servers requiring client certificates 19 | # client_cert = "/path/to/client.crt" 20 | # client_key = "/path/to/client.key" 21 | 22 | # Search user bind dn 23 | bind_dn = "cn=admin,dc=grafana,dc=org" 24 | # Search user bind password 25 | # If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" 26 | bind_password = 'grafana' 27 | 28 | # User search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)" 29 | search_filter = "(cn=%s)" 30 | 31 | # An array of base dns to search through 32 | search_base_dns = ["dc=grafana,dc=org"] 33 | 34 | ## For Posix or LDAP setups that does not support member_of attribute you can define the below settings 35 | ## Please check grafana LDAP docs for examples 36 | # group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" 37 | # group_search_base_dns = ["ou=groups,dc=grafana,dc=org"] 38 | # group_search_filter_user_attribute = "uid" 39 | 40 | # Specify names of the ldap attributes your ldap uses 41 | [servers.attributes] 42 | name = "givenName" 43 | surname = "sn" 44 | username = "cn" 45 | member_of = "memberOf" 46 | email = "email" 47 | 48 | # Map ldap groups to grafana org roles 49 | [[servers.group_mappings]] 50 | group_dn = "cn=admins,dc=grafana,dc=org" 51 | org_role = "Admin" 52 | # To make user an instance admin (Grafana Admin) uncomment line below 53 | # grafana_admin = true 54 | # The Grafana organization database id, optional, if left out the default org (id 1) will be used 55 | # org_id = 1 56 | 57 | [[servers.group_mappings]] 58 | group_dn = "cn=users,dc=grafana,dc=org" 59 | org_role = "Editor" 60 | 61 | [[servers.group_mappings]] 62 | # If you want to match all (or no ldap groups) then you can use wildcard 63 | group_dn = "*" 64 | org_role = "Viewer" 65 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/base/templates/config: -------------------------------------------------------------------------------- 1 | SHAKENFIST_ETCD_HOST="{{hostvars[groups['etcd_master'][0]]['node_mesh_ip']}}" 2 | 3 | # It is recommended that you set the following options: 4 | # API_ADVERTISED_HOST="shakenfist.mycompany.com" 5 | # API_ADVERTISED_BASE_PATH="/api" 6 | # API_ADVERTISED_HTTP_SCHEMES="https" 7 | 8 | SHAKENFIST_NODE_EGRESS_IP="{{node_egress_ip}}" 9 | SHAKENFIST_NODE_EGRESS_NIC="{{node_egress_nic}}" 10 | SHAKENFIST_NODE_MESH_IP="{{node_mesh_ip}}" 11 | SHAKENFIST_NODE_MESH_NIC="{{node_mesh_nic}}" 12 | SHAKENFIST_NODE_NAME="{{node_name}}" 13 | SHAKENFIST_ZONE="{{deploy_name}}" 14 | 15 | {% if inventory_hostname in groups['etcd_master'] %} 16 | SHAKENFIST_NODE_IS_ETCD_MASTER=True 17 | {% else %} 18 | SHAKENFIST_NODE_IS_ETCD_MASTER=False 19 | {% endif %} 20 | 21 | {% if inventory_hostname in groups['hypervisors'] %} 22 | SHAKENFIST_NODE_IS_HYPERVISOR=True 23 | {% else %} 24 | SHAKENFIST_NODE_IS_HYPERVISOR=False 25 | {% endif %} 26 | 27 | {% if inventory_hostname in groups['network_node'] %} 28 | SHAKENFIST_NODE_IS_NETWORK_NODE=True 29 | {% else %} 30 | SHAKENFIST_NODE_IS_NETWORK_NODE=False 31 | {% endif %} 32 | 33 | {% if inventory_hostname in groups['eventlog_node'] %} 34 | SHAKENFIST_NODE_IS_EVENTLOG_NODE=True 35 | {% else %} 36 | SHAKENFIST_NODE_IS_EVENTLOG_NODE=False 37 | {% endif %} 38 | 39 | SHAKENFIST_FLOATING_NETWORK="{{floating_network_ipblock}}" 40 | SHAKENFIST_NETWORK_NODE_IP="{{hostvars[groups['network_node'][0]]['node_mesh_ip']}}" 41 | SHAKENFIST_EVENTLOG_NODE_IP="{{hostvars[groups['eventlog_node'][0]]['node_mesh_ip']}}" 42 | 43 | # Database service configuration 44 | {% if inventory_hostname in groups['etcd_master'] %} 45 | SHAKENFIST_NODE_IS_DATABASE_NODE=True 46 | {% else %} 47 | SHAKENFIST_NODE_IS_DATABASE_NODE=False 48 | {% endif %} 49 | SHAKENFIST_DATABASE_NODE_IP="{{hostvars[groups['etcd_master'][0]]['node_mesh_ip']}}" 50 | SHAKENFIST_DATABASE_API_PORT=13005 51 | SHAKENFIST_DATABASE_METRICS_PORT=13006 52 | SHAKENFIST_DATABASE_USE_DIRECT_ETCD={{database_use_direct_etcd | default('True')}} 53 | 54 | # MariaDB configuration - only rendered on database nodes (etcd_master) for security. 55 | # Only the database service daemon needs direct MariaDB access; all other daemons 56 | # access MariaDB through the database service's gRPC interface. 57 | {% if inventory_hostname in groups['etcd_master'] %} 58 | SHAKENFIST_MARIADB_HOST="{{hostvars[groups['etcd_master'][0]]['node_mesh_ip']}}" 59 | SHAKENFIST_MARIADB_PORT=3306 60 | SHAKENFIST_MARIADB_USER="shakenfist" 61 | SHAKENFIST_MARIADB_PASSWORD="{{mariadb_password}}" 62 | SHAKENFIST_MARIADB_DATABASE="shakenfist" 63 | {% endif %} 64 | -------------------------------------------------------------------------------- /shakenfist/protos/common_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # NO CHECKED-IN PROTOBUF GENCODE 4 | # source: common.proto 5 | # Protobuf Python Version: 5.29.0 6 | """Generated protocol buffer code.""" 7 | from google.protobuf import descriptor as _descriptor 8 | from google.protobuf import descriptor_pool as _descriptor_pool 9 | from google.protobuf import runtime_version as _runtime_version 10 | from google.protobuf import symbol_database as _symbol_database 11 | from google.protobuf.internal import builder as _builder 12 | _runtime_version.ValidateProtobufRuntimeVersion( 13 | _runtime_version.Domain.PUBLIC, 14 | 5, 15 | 29, 16 | 0, 17 | '', 18 | 'common.proto' 19 | ) 20 | # @@protoc_insertion_point(imports) 21 | 22 | _sym_db = _symbol_database.Default() 23 | 24 | 25 | 26 | 27 | DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x63ommon.proto\x12\x11shakenfist.protos\"2\n\x13\x45nvironmentVariable\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\xb8\x02\n\x0e\x45xecuteRequest\x12\x0f\n\x07\x63ommand\x18\x01 \x01(\t\x12\x45\n\x15\x65nvironment_variables\x18\x02 \x03(\x0b\x32&.shakenfist.protos.EnvironmentVariable\x12\x19\n\x11network_namespace\x18\x03 \x01(\t\x12\x41\n\x0bio_priority\x18\x04 \x01(\x0e\x32,.shakenfist.protos.ExecuteRequest.IOPriority\x12\x19\n\x11working_directory\x18\x05 \x01(\t\x12\x12\n\nrequest_id\x18\x06 \x01(\t\x12\x14\n\x0c\x65xecution_id\x18\x07 \x01(\t\"+\n\nIOPriority\x12\n\n\x06NORMAL\x10\x00\x12\x07\n\x03LOW\x10\x01\x12\x08\n\x04HIGH\x10\x02\"\x86\x01\n\x0c\x45xecuteReply\x12\x0e\n\x06stdout\x18\x01 \x01(\t\x12\x0e\n\x06stderr\x18\x02 \x01(\t\x12\x11\n\texit_code\x18\x03 \x01(\x05\x12\x12\n\nrequest_id\x18\x04 \x01(\t\x12\x14\n\x0c\x65xecution_id\x18\x05 \x01(\t\x12\x19\n\x11\x65xecution_seconds\x18\x06 \x01(\x01\x62\x06proto3') 28 | 29 | _globals = globals() 30 | _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) 31 | _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'common_pb2', _globals) 32 | if not _descriptor._USE_C_DESCRIPTORS: 33 | DESCRIPTOR._loaded_options = None 34 | _globals['_ENVIRONMENTVARIABLE']._serialized_start=35 35 | _globals['_ENVIRONMENTVARIABLE']._serialized_end=85 36 | _globals['_EXECUTEREQUEST']._serialized_start=88 37 | _globals['_EXECUTEREQUEST']._serialized_end=400 38 | _globals['_EXECUTEREQUEST_IOPRIORITY']._serialized_start=357 39 | _globals['_EXECUTEREQUEST_IOPRIORITY']._serialized_end=400 40 | _globals['_EXECUTEREPLY']._serialized_start=403 41 | _globals['_EXECUTEREPLY']._serialized_end=537 42 | # @@protoc_insertion_point(module_scope) 43 | -------------------------------------------------------------------------------- /shakenfist/upload.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Michael Still 2 | import time 3 | 4 | from shakenfist_utilities import logs # noreorder 5 | 6 | from shakenfist.baseobject import DatabaseBackedObject as dbo 7 | from shakenfist.baseobject import DatabaseBackedObjectIterator as dbo_iter 8 | from shakenfist.schema.object_types import ObjectType 9 | 10 | 11 | LOG, _ = logs.setup(__name__) 12 | 13 | 14 | class Upload(dbo): 15 | object_type = ObjectType.UPLOAD 16 | initial_version = 2 17 | current_version = 4 18 | 19 | # docs/developer_guide/state_machine.md has a description of these states. 20 | state_targets = { 21 | None: (dbo.STATE_CREATED), 22 | dbo.STATE_CREATED: (dbo.STATE_DELETED), 23 | dbo.STATE_DELETED: (), 24 | } 25 | 26 | ACTIVE_STATES = {dbo.STATE_CREATED} 27 | 28 | def __init__(self, static_values): 29 | self.upgrade(static_values) 30 | 31 | super().__init__(static_values.get('uuid'), static_values.get('version')) 32 | self.__node = static_values['node'] 33 | self.__created_at = static_values['created_at'] 34 | 35 | @classmethod 36 | def _upgrade_step_2_to_3(cls, static_values): 37 | cls._upgrade_metadata_to_attribute(static_values['uuid']) 38 | 39 | @classmethod 40 | def _upgrade_step_3_to_4(cls, static_values): 41 | # State migration to MariaDB is now handled by sf-ctl migrate-state-to-mariadb 42 | ... 43 | 44 | @classmethod 45 | def new(cls, upload_uuid, node): 46 | static_values = { 47 | 'uuid': upload_uuid, 48 | 'node': node, 49 | 'created_at': time.time(), 50 | 51 | 'version': cls.current_version 52 | } 53 | Upload._db_create(upload_uuid, static_values) 54 | u = Upload(static_values) 55 | u.state = Upload.STATE_CREATED 56 | return u 57 | 58 | # Static values 59 | @property 60 | def node(self): 61 | return self.__node 62 | 63 | @property 64 | def created_at(self): 65 | return self.__created_at 66 | 67 | def external_view(self): 68 | retval = self._external_view() 69 | retval.update({ 70 | 'node': self.node, 71 | 'created_at': self.created_at 72 | }) 73 | return retval 74 | 75 | 76 | class Uploads(dbo_iter): 77 | base_object = Upload 78 | 79 | def __iter__(self): 80 | for _, u in self.get_iterator(): 81 | u = Upload(u) 82 | if not u: 83 | continue 84 | 85 | out = self.apply_filters(u) 86 | if out: 87 | yield out 88 | -------------------------------------------------------------------------------- /.github/workflows/code-formatting.yml: -------------------------------------------------------------------------------- 1 | name: Run automated code formatters 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '13 00 * * *' # utc 7 | pull_request: 8 | branches: 9 | - develop 10 | paths: 11 | - '.github/workflows/code-formatting.yml' 12 | 13 | jobs: 14 | code-formatters: 15 | runs-on: [self-hosted, vm] 16 | timeout-minutes: 60 17 | concurrency: 18 | group: ${{ github.workflow }}-${{ github.ref }} 19 | cancel-in-progress: true 20 | 21 | steps: 22 | - name: Set environment variables 23 | run: | 24 | echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV 25 | 26 | - name: Checkout the shakenfist repository 27 | uses: actions/checkout@v4 28 | with: 29 | path: shakenfist 30 | fetch-depth: 0 31 | 32 | - name: Checkout the actions repository 33 | uses: actions/checkout@v4 34 | with: 35 | repository: shakenfist/actions 36 | path: actions 37 | fetch-depth: 0 38 | 39 | - name: Install the github command line 40 | run: | 41 | sudo apt update 42 | sudo apt install -y curl 43 | 44 | curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg 45 | sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg 46 | echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null 47 | 48 | sudo apt update 49 | sudo apt install -y gh 50 | 51 | - name: Install formatting tools in a venv 52 | run: | 53 | sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=-1 -o Dpkg::Options::="--force-confold" -y \ 54 | install git python3-cffi python3-dev python3-grpcio python3-pip python3-venv python3-wheel 55 | 56 | python3 -m venv --system-site-packages /srv/ci/venv 57 | . /srv/ci/venv/bin/activate 58 | 59 | cd ${GITHUB_WORKSPACE}/shakenfist 60 | 61 | pip3 install uv 62 | uv pip install -r pyproject.toml 63 | uv pip install pyupgrade reorder-python-imports 64 | 65 | - name: Scan for obsolete syntax 66 | env: 67 | GITHUB_TOKEN: ${{ secrets.DEPENDENCIES_TOKEN }} 68 | run: | 69 | cd ${GITHUB_WORKSPACE}/shakenfist 70 | . /srv/ci/venv/bin/activate 71 | ${GITHUB_WORKSPACE}/actions/tools/ci_code_formatting.sh 39 72 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/guest_ci_tests/test_boot.py: -------------------------------------------------------------------------------- 1 | from shakenfist_ci import base 2 | 3 | import testscenarios 4 | 5 | 6 | class TestBoot(testscenarios.WithScenarios, base.BaseNamespacedTestCase): 7 | """Make sure instances boot under various configurations.""" 8 | 9 | scenarios = [ 10 | ( 11 | 'debian-11', 12 | { 13 | 'base': 'debian-11' 14 | } 15 | ), 16 | ( 17 | 'debian-12', 18 | { 19 | 'base': 'debian-12' 20 | } 21 | ), 22 | ] 23 | 24 | def __init__(self, *args, **kwargs): 25 | kwargs['namespace_prefix'] = 'boot' 26 | super().__init__(*args, **kwargs) 27 | 28 | def setUp(self): 29 | super().setUp() 30 | self.net = self.test_client.allocate_network( 31 | '192.168.242.0/24', True, True, '%s-net' % self.namespace) 32 | self._await_networks_ready([self.net['uuid']]) 33 | 34 | def _boot_no_network(self): 35 | """Check that instances without a network still boot. 36 | 37 | Once we had a bug that only stopped instance creation when no network 38 | was specified. 39 | """ 40 | inst = self.test_client.create_instance( 41 | f'test-boot-no-network-{self.base}', 1, 1024, None, 42 | [ 43 | { 44 | 'size': 8, 45 | 'base': f'sf://upload/system/{self.base}', 46 | 'type': 'disk' 47 | } 48 | ], None, None) 49 | 50 | self._await_instance_ready(inst['uuid']) 51 | 52 | def _boot_network(self): 53 | inst = self.test_client.create_instance( 54 | f'test-boot-network-{self.base}', 1, 1024, 55 | [ 56 | { 57 | 'network_uuid': self.net['uuid'] 58 | } 59 | ], 60 | [ 61 | { 62 | 'size': 8, 63 | 'base': f'sf://upload/system/{self.base}', 64 | 'type': 'disk' 65 | } 66 | ], None, None) 67 | 68 | self._await_instance_ready(inst['uuid']) 69 | 70 | def _boot_large_disk(self): 71 | inst = self.test_client.create_instance( 72 | f'test-boot-large-disk-{self.base}', 1, 1024, None, 73 | [ 74 | { 75 | 'size': 30, 76 | 'base': f'sf://upload/system/{self.base}', 77 | 'type': 'disk' 78 | } 79 | ], None, None) 80 | 81 | self._await_instance_ready(inst['uuid']) 82 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_console_log.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from shakenfist_ci import base 4 | from shakenfist_client import apiclient 5 | 6 | 7 | class TestConsoleLog(base.BaseNamespacedTestCase): 8 | def __init__(self, *args, **kwargs): 9 | kwargs['namespace_prefix'] = 'console' 10 | super().__init__(*args, **kwargs) 11 | 12 | def setUp(self): 13 | super().setUp() 14 | self.net = self.test_client.allocate_network( 15 | '192.168.242.0/24', True, True, '%s-net-one' % self.namespace) 16 | self._await_networks_ready([self.net['uuid']]) 17 | 18 | def test_console_log(self): 19 | # Start our test instance 20 | inst = self.test_client.create_instance( 21 | 'test-console', 1, 1024, 22 | [ 23 | { 24 | 'network_uuid': self.net['uuid'] 25 | }, 26 | ], 27 | [ 28 | { 29 | 'size': 8, 30 | 'base': base.CLUSTER_CI_IMAGE, 31 | 'type': 'disk' 32 | } 33 | ], None, base.load_userdata('cluster_ci_tests', 'console_scribbler')) 34 | 35 | # Wait for our test instance to boot 36 | self.assertIsNotNone(inst['uuid']) 37 | self._await_instance_ready(inst['uuid']) 38 | time.sleep(30) 39 | 40 | # Get 1000 bytes of console log 41 | c = self.test_client.get_console_data(inst['uuid'], 1000, decode=None) 42 | if len(c) < 1000: 43 | self.fail( 44 | 'Console response was not 1000 characters (%d instead):\n\n%s' 45 | % (len(c), c)) 46 | 47 | # Get 2000 bytes of console log 48 | c = self.test_client.get_console_data(inst['uuid'], 2000, decode=None) 49 | if len(c) < 2000: 50 | self.fail( 51 | 'Console response was not 2000 characters (%d instead):\n\n%s' 52 | % (len(c), c)) 53 | 54 | # Get the default amount of the console log 55 | c = self.test_client.get_console_data(inst['uuid'], decode=None) 56 | if len(c) < 10240: 57 | self.fail( 58 | 'Console response was not 10240 characters (%d instead):\n\n%s' 59 | % (len(c), c)) 60 | 61 | # Get all of the console log 62 | c = self.test_client.get_console_data(inst['uuid'], -1, decode=None) 63 | self.assertGreaterEqual(len(c), 11000) 64 | 65 | # Check we handle non-numbers reasonably 66 | self.assertRaises( 67 | apiclient.RequestMalformedException, 68 | self.test_client.get_console_data, inst['uuid'], 'banana') 69 | -------------------------------------------------------------------------------- /shakenfist/schema/object_types.py: -------------------------------------------------------------------------------- 1 | # Pydantic schema for object type validation. 2 | # 3 | # This module defines an enum of all valid object types in Shaken Fist. This 4 | # provides type safety and validation for fields that reference object types, 5 | # such as user_type in IPAM reservations. 6 | # 7 | # This is the single source of truth for object type names. 8 | 9 | from enum import Enum 10 | 11 | 12 | class ObjectType(str, Enum): 13 | """Enum of all valid object types in Shaken Fist. 14 | 15 | This enum inherits from str so that the value can be used directly as a 16 | string in SQL queries and JSON serialization. For example: 17 | ObjectType.INSTANCE.value == 'instance' 18 | ObjectType.INSTANCE == 'instance' # Also works due to str inheritance 19 | str(ObjectType.INSTANCE) == 'instance' # Works due to __str__ override 20 | f'{ObjectType.INSTANCE}' == 'instance' # Works in f-strings too 21 | 22 | The enum values match the object_type class attribute on each 23 | DatabaseBackedObject subclass. 24 | """ 25 | 26 | def __str__(self) -> str: 27 | """Return the enum value as a string. 28 | 29 | This override is needed because the default str(Enum) returns 30 | 'EnumName.MEMBER_NAME' rather than the value. Since we want to use 31 | these values in etcd paths, error messages, and other string contexts, 32 | we override __str__ to return the value directly. 33 | """ 34 | return self.value 35 | 36 | # Core objects 37 | AGENTOPERATION = 'agentoperation' 38 | ARTIFACT = 'artifact' 39 | BLOB = 'blob' 40 | DHCP = 'dhcp' 41 | INSTANCE = 'instance' 42 | INTERFACE = 'interface' 43 | IPAM = 'ipam' 44 | NAMESPACE = 'namespace' 45 | NETWORK = 'network' 46 | NODE = 'node' 47 | UPLOAD = 'upload' 48 | 49 | # Operation objects 50 | ARTIFACT_FETCH_OP = 'artifact_fetch_op' 51 | IMGCACHE_OP = 'imgcache_op' 52 | NET_IFACE_OP = 'net_iface_op' 53 | NET_IFACE_IP_OP = 'net_iface_ip_op' 54 | NET_IP_OP = 'net_ip_op' 55 | NET_MACADDR_IP_OP = 'net_macaddr_ip_op' 56 | NET_OP = 'net_op' 57 | NODE_AOP_OP = 'node_aop_op' 58 | NODE_BLOB_OP = 'node_blob_op' 59 | NODE_INST_NET_IFACE_OP = 'node_inst_net_iface_op' 60 | NODE_INST_NETDESC_OP = 'node_inst_netdesc_op' 61 | NODE_INST_OP = 'node_inst_op' 62 | NODE_INST_SNAP_OP = 'node_inst_snap_op' 63 | NODE_NET_OP = 'node_net_op' 64 | 65 | # Meta object for API request tracing 66 | API_REQUESTS = 'api-requests' 67 | 68 | # Base/placeholder types (used by base classes before subclass override) 69 | UNKNOWN = 'unknown' 70 | UNKNOWN_MANAGED_EXECUTABLE = 'unknown_managed_executable' 71 | -------------------------------------------------------------------------------- /shakenfist/external_api/snapshot.py: -------------------------------------------------------------------------------- 1 | # Documentation state: 2 | # - Has metadata calls: 3 | # - OpenAPI complete: 4 | # - Covered in user or operator docs: 5 | # - API reference docs exist: 6 | # - and link to OpenAPI docs: 7 | # - and include examples: 8 | # - Has complete CI coverage: 9 | from functools import partial 10 | 11 | from shakenfist_utilities import api as sf_api # noreorder 12 | from shakenfist_utilities import logs # noreorder 13 | 14 | from shakenfist import artifact 15 | from shakenfist import blob 16 | from shakenfist.artifact import Artifacts 17 | from shakenfist.config import config 18 | from shakenfist.constants import EVENT_TYPE_AUDIT 19 | from shakenfist.daemons import daemon 20 | from shakenfist.external_api import base as api_base 21 | from shakenfist.instance import instance_usage_for_blob_uuid 22 | 23 | 24 | LOG, HANDLER = logs.setup(__name__) 25 | daemon.set_log_level(LOG, 'api') 26 | 27 | 28 | class InstanceSnapshotEndpoint(sf_api.Resource): 29 | @api_base.verify_token 30 | @api_base.arg_is_instance_ref 31 | @api_base.requires_instance_ownership 32 | @api_base.redirect_instance_request 33 | @api_base.requires_instance_active 34 | @api_base.log_token_use 35 | def post(self, instance_ref=None, instance_from_db=None, all=None, 36 | device=None, max_versions=0, thin=None): 37 | if not thin: 38 | thin = config.SNAPSHOTS_DEFAULT_TO_THIN 39 | 40 | instance_from_db.add_event( 41 | EVENT_TYPE_AUDIT, 'snapshot request from REST API') 42 | return instance_from_db.snapshot( 43 | all=all, device=device, max_versions=max_versions, thin=thin) 44 | 45 | @api_base.verify_token 46 | @api_base.arg_is_instance_ref 47 | @api_base.requires_instance_ownership 48 | @api_base.log_token_use 49 | def get(self, instance_ref=None, instance_from_db=None): 50 | out = [] 51 | for snap in Artifacts([ 52 | partial(artifact.instance_snapshot_filter, instance_from_db.uuid)]): 53 | ev = snap.external_view_without_index() 54 | for idx in snap.get_all_indexes(): 55 | # Give the blob uuid a better name 56 | b = blob.Blob.from_db(idx['blob_uuid']) 57 | if not b: 58 | continue 59 | 60 | bout = b.external_view() 61 | bout['blob_uuid'] = bout['uuid'] 62 | bout['instances'] = instance_usage_for_blob_uuid(b.uuid) 63 | del bout['uuid'] 64 | 65 | # Merge it with the parent artifact 66 | a = ev.copy() 67 | a.update(bout) 68 | out.append(a) 69 | return out 70 | -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/pki_internal_ca/tasks/host_certificate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: 4 | - "Managing host certificate for {{ hostname }}." 5 | - "CA cert path: {{ ca_cert_path }}" 6 | - "CA key path: {{ ca_key_path }}" 7 | - "CA template path: {{ ca_template_path }}" 8 | - "Host cert path: {{ host_cert_path }}" 9 | - "Host key path: {{ host_key_path }}" 10 | - "Host template path: {{ host_template_path }}" 11 | 12 | - name: Check if the host certificate is already created 13 | stat: 14 | path: /etc/pki/libvirt-spice/server_cert.pem 15 | register: host_cert 16 | 17 | - name: Check certificate lifetime 18 | shell: | 19 | openssl x509 -checkend $(( 3600 * 24 * 30 )) -noout \ 20 | -in /etc/pki/libvirt-spice/server_cert.pem || true 21 | register: host_cert_expires 22 | when: host_cert.stat.exists 23 | 24 | - name: Log certificate expiry result 25 | debug: 26 | msg: "{{ host_cert_expires }}" 27 | when: host_cert.stat.exists 28 | 29 | - name: Move aside old certificate 30 | shell: 31 | mv "{{ host_cert_path }}" "{{ host_key_path }}".$(date +%Y%m%d)".pem" 32 | delegate_to: localhost 33 | delegate_facts: true 34 | when: host_cert.stat.exists and not host_cert_expires.meta.stdout == "Certificate will expire" 35 | 36 | - name: Check if the host certificate still exists 37 | stat: 38 | path: "{{ host_cert_path }}" 39 | delegate_to: localhost 40 | delegate_facts: true 41 | register: host_cert 42 | 43 | - name: Check if the host key exists 44 | stat: 45 | path: "{{ host_key_path }}" 46 | delegate_to: localhost 47 | delegate_facts: true 48 | register: host_key 49 | 50 | - name: Write host certificate template 51 | copy: 52 | content: | 53 | organization = Shaken Fist CA for {{deploy_name}} 54 | cn = {{ hostname }} 55 | tls_www_server 56 | encryption_key 57 | signing_key 58 | dest: "{{ host_template_path }}" 59 | owner: root 60 | mode: u=r 61 | delegate_to: localhost 62 | delegate_facts: true 63 | when: not host_cert.stat.exists 64 | 65 | - name: Create host key 66 | shell: 67 | umask 277 && certtool --generate-privkey > "{{ host_key_path }}" 68 | delegate_to: localhost 69 | delegate_facts: true 70 | when: not host_key.stat.exists 71 | 72 | - name: Create host certificate 73 | shell: 74 | certtool --generate-certificate \ 75 | --template "{{ host_template_path }}" \ 76 | --load-privkey "{{ host_key_path }}" \ 77 | --load-ca-certificate "{{ ca_cert_path }}" \ 78 | --load-ca-privkey "{{ ca_key_path }}" \ 79 | --outfile "{{ host_cert_path }}" 80 | delegate_to: localhost 81 | delegate_facts: true 82 | when: not host_cert.stat.exists -------------------------------------------------------------------------------- /shakenfist/client/backup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Michael Still 2 | import importlib 3 | import io 4 | import json 5 | import logging 6 | import os 7 | import tarfile 8 | 9 | import click 10 | from shakenfist_utilities import logs # noreorder 11 | 12 | 13 | LOG = logs.setup_console(__name__) 14 | 15 | 16 | # Utilities not started by systemd need to load /etc/sf/config to ensure 17 | # that they are correctly configured. Environment variables set before 18 | # running the utility take precedence over values in the config file. 19 | if os.path.exists('/etc/sf/config'): 20 | with open('/etc/sf/config') as f: 21 | for line in f.readlines(): 22 | line = line.rstrip() 23 | 24 | if line.startswith('#'): 25 | continue 26 | if line == '': 27 | continue 28 | 29 | key, value = line.split('=') 30 | value = value.strip('\'"') 31 | 32 | if key not in os.environ: 33 | os.environ[key] = value 34 | 35 | sf_config = importlib.import_module('shakenfist.config') 36 | config = sf_config.config 37 | 38 | # These imports _must_ occur after the extra config setup has run. 39 | from shakenfist import etcd # noqa 40 | 41 | 42 | @click.group() 43 | @click.option('--verbose/--no-verbose', default=False) 44 | @click.pass_context 45 | def cli(ctx, verbose=None): 46 | if verbose: 47 | LOG.setLevel(logging.DEBUG) 48 | 49 | 50 | @click.command() 51 | @click.argument('output', type=click.Path(exists=False)) 52 | @click.option('-a', '--anonymise', is_flag=True, 53 | help='Remove authentication details from backup') 54 | @click.pass_context 55 | def backup(ctx, output, anonymise=False): 56 | with tarfile.open(output, 'w:gz') as tar: 57 | for data, metadata in etcd.get_etcd_client().get_prefix('/'): 58 | if metadata['key'].startswith(b'/sf/namespace'): 59 | d = json.loads(data) 60 | for k in d['keys']: 61 | d['keys'][k] = '...' 62 | data = json.dumps(d, indent=4, sort_keys=True).encode('utf-8') 63 | 64 | info = tarfile.TarInfo(metadata['key'].decode('utf-8').rstrip('/')) 65 | info.size = len(data) 66 | tar.addfile(info, io.BytesIO(data)) 67 | 68 | 69 | cli.add_command(backup) 70 | 71 | 72 | @click.command() 73 | @click.argument('input', type=click.Path(exists=True)) 74 | @click.pass_context 75 | def restore(ctx, input): 76 | with tarfile.open(input, 'r:gz') as tar: 77 | for tarinfo in tar: 78 | key = tarinfo.name 79 | data = tar.extractfile(key).read() 80 | 81 | etcd.get_etcd_client().put(key, data) 82 | 83 | 84 | cli.add_command(restore) 85 | -------------------------------------------------------------------------------- /docs/manifesto.md: -------------------------------------------------------------------------------- 1 | # The Shaken Fist Manifesto 2 | 3 | This document attempts to list Shaken Fist's defining features, give guidance on 4 | what type of features should be added to the project, how they should be implemented 5 | and how we work together. 6 | 7 | ## Shaken Fist Defining Characteristics 8 | 9 | * Shaken Fist is smaller, simpler cloud. 10 | * It is designed for relatively small environments with minimum management overhead. 11 | * Its features are highly opinionated. This means that the maintainers have chosen 12 | the best (in their opinion) features to support. 13 | * Opinionated features do not handle every single possible use case. This reduces 14 | the code base size thus increasing long-term maintainability. 15 | * The code base is understandable in its entirety by a single developer. 16 | * A Shaken Fist cluster does not need a team of engineers to install or operate. 17 | * A Shaken Fist cluster should be simple to set up. We define 'simple' as "a person 18 | with no knowledge of the project can build a reasonable cluster in an evening". 19 | 20 | 21 | ## Project Goals 22 | 23 | * Allow simple management of virtual machine instances without complexity. 24 | * Support networking between those machines and also facilitate access to external 25 | networks. 26 | * Avoid re-inventing the wheel (utilise other open source projects when appropriate). 27 | 28 | 29 | ## Feature Guidelines 30 | 31 | * Features should be deliberately limited in the options available. 32 | * The goal of limiting options is to reduce code complexity. If the option does 33 | not add significant code complexity then it should added. 34 | * The supported features and the options of those features should aim to cover 35 | the majority of use cases. 36 | * When a feature limits the available options, it should do so in a way that does 37 | not overly restrict a project fork from adding that option. 38 | * New code should conform to the conventions of the existing code base and written 39 | to be easily understood. 40 | * New code should have new tests (please). 41 | 42 | 43 | ## Significant Opinionated Design Decisions 44 | 45 | * The only supported hypervisor is KVM managed by libvirt. 46 | * Virtual networking is only implemented via VXLAN meshes. 47 | * Single machine clusters should always be possible. 48 | * Only the current Ubuntu LTS version and Debian supported by the main project 49 | (pull requests to support other operating systems are encouraged). 50 | 51 | 52 | ## Project Interaction Guidelines 53 | 54 | * Always polite. 55 | * Always generous. 56 | * Being opinionated is encouraged (but gently). 57 | * Updating the documentation is just as important as the code change itself. 58 | * Developers who write tests are the most highly prized of all the developers. 59 | -------------------------------------------------------------------------------- /shakenfist/external_api/admin.py: -------------------------------------------------------------------------------- 1 | # Documentation state: 2 | # - OpenAPI complete: yes 3 | # - Covered in user or operator docs: operator 4 | # - API reference docs exist: yes 5 | # - and link to OpenAPI docs: yes 6 | # - and include examples: yes 7 | # - Has complete CI coverage: 8 | import os 9 | 10 | import flask 11 | from flasgger import swag_from 12 | from shakenfist_utilities import api as sf_api # noreorder 13 | 14 | from shakenfist import etcd 15 | from shakenfist import scheduler 16 | from shakenfist.external_api import base as api_base 17 | 18 | 19 | admin_locks_get_example = """{ 20 | "/sflocks/sf/cluster/": { 21 | "node": "sf-1", 22 | "operation": "Cluster maintenance", 23 | "pid": 3326781 24 | } 25 | }""" 26 | 27 | 28 | class AdminLocksEndpoint(sf_api.Resource): 29 | @swag_from(api_base.swagger_helper( 30 | 'admin', 'List locks currently held in the cluster.', [], 31 | [(200, 'All locks currently held in the cluster.', 32 | admin_locks_get_example)], 33 | requires_admin=True)) 34 | @api_base.verify_token 35 | @api_base.caller_is_admin 36 | @api_base.log_token_use 37 | def get(self): 38 | return etcd.get_existing_locks() 39 | 40 | 41 | admin_cacert_get_example = """-----BEGIN CERTIFICATE----- 42 | MIIEFzCCAn+gAwIBAgIUCs+LmF8yISmu02Jht+LeM/9SF+owDQYJKoZIhvcNAQEL 43 | ... 44 | LFPuUi9WNH611ybJLriyFIN4a8v67CX0VJ8G9yIyYGrDlY6jBWu16br/Fw== 45 | -----END CERTIFICATE-----""" 46 | 47 | 48 | class AdminClusterCaCertificateEndpoint(sf_api.Resource): 49 | @swag_from(api_base.swagger_helper( 50 | 'admin', 'Retrieve the CA certificate used for TLS in this cluster.', [], 51 | [(200, 'A PEM encoded CA certificate.', 52 | admin_cacert_get_example)])) 53 | @api_base.verify_token 54 | @api_base.log_token_use 55 | def get(self): 56 | cacert = '' 57 | if os.path.exists('/etc/pki/libvirt-spice/ca-cert.pem'): 58 | with open('/etc/pki/libvirt-spice/ca-cert.pem') as f: 59 | cacert = f.read() 60 | 61 | resp = flask.Response(cacert, mimetype='text/plain') 62 | resp.status_code = 200 63 | return resp 64 | 65 | 66 | admin_resources_get_example = """{ 67 | ... 68 | }""" 69 | 70 | 71 | class AdminREsourcesEndpoint(sf_api.Resource): 72 | @swag_from(api_base.swagger_helper( 73 | 'admin', 'List resources currently available in the cluster.', [], 74 | [(200, 'All summary of resource usage and availability in the cluster.', 75 | admin_resources_get_example)], 76 | requires_admin=True)) 77 | @api_base.verify_token 78 | @api_base.caller_is_admin 79 | @api_base.log_token_use 80 | def get(self): 81 | s = scheduler.Scheduler() 82 | return s.summarize_resources() 83 | -------------------------------------------------------------------------------- /shakenfist/tests/files/ubuntu-MD5SUMS-bionic: -------------------------------------------------------------------------------- 1 | 0acb684326c348c217beb360fb571964 *bionic-server-cloudimg-amd64-azure.vhd.tar.gz 2 | d0524af6dca584b63e03b6869c776f00 *bionic-server-cloudimg-amd64-azure.vhd.zip 3 | e1d73a2595ca8dbffc76fd4189890df9 *bionic-server-cloudimg-amd64-lxd.tar.xz 4 | 159202a56ede93ce64e79dad3ef51f84 *bionic-server-cloudimg-amd64-root.tar.xz 5 | 3f9d28a0682697406cca5eadf3b931b2 *bionic-server-cloudimg-amd64-vagrant.box 6 | f30aff3227eef5ffe348bd7e42cf572e *bionic-server-cloudimg-amd64-wsl.rootfs.tar.gz 7 | ed44b9745b8d62bcbbc180b5f36c24bb *bionic-server-cloudimg-amd64.img 8 | b904362d1b604adb2e21288fed60aa78 *bionic-server-cloudimg-amd64.ova 9 | f986d327cdb5b9d2ae0e0dd08fc30f52 *bionic-server-cloudimg-amd64.squashfs 10 | 0efc982c99471080eca9db56036bbc1b *bionic-server-cloudimg-amd64.tar.gz 11 | 86e8455f170871973597cca8e082896b *bionic-server-cloudimg-amd64.vmdk 12 | e99dc6efd5e104eac6d4ee7ef58453f9 *bionic-server-cloudimg-arm64-lxd.tar.xz 13 | b81821d9e947beaaefba3ec01bdd1a77 *bionic-server-cloudimg-arm64-root.tar.xz 14 | 6fed8fbccaf78585483502ef9a527c8d *bionic-server-cloudimg-arm64-wsl.rootfs.tar.gz 15 | 76effaad27b15d373d541b0f9e704aef *bionic-server-cloudimg-arm64.img 16 | 09e16ceeaf7c5b503e50f1746d04cb5e *bionic-server-cloudimg-arm64.squashfs 17 | 2858dc2df7e33095e3848ea2641833fb *bionic-server-cloudimg-arm64.tar.gz 18 | 31734297f86ca47304656ea2e1b41ae6 *bionic-server-cloudimg-armhf-lxd.tar.xz 19 | 8149dc1ebfb59d151ae9a66d41063546 *bionic-server-cloudimg-armhf-root.tar.xz 20 | 50be58568da35191c108df120efd42f1 *bionic-server-cloudimg-armhf.img 21 | 253278d2035818546b5577ac471817dc *bionic-server-cloudimg-armhf.squashfs 22 | 85643b7864549086e6d372bcfd8f18f5 *bionic-server-cloudimg-armhf.tar.gz 23 | 1d2b05d50333dad4a685bcc9e36cf937 *bionic-server-cloudimg-i386-lxd.tar.xz 24 | 401957b0f0ccc6e7afbaa788594540a2 *bionic-server-cloudimg-i386-root.tar.xz 25 | 4f54cf916ee90bf16261f9718fae7aaf *bionic-server-cloudimg-i386.img 26 | 203e89fdb73d61f229af2b1d99255266 *bionic-server-cloudimg-i386.squashfs 27 | 42504c4ff4966aeb75543193163f7380 *bionic-server-cloudimg-i386.tar.gz 28 | 7227dc178cc8ee1b038a831e40f0745f *bionic-server-cloudimg-ppc64el-lxd.tar.xz 29 | 77cdb1b9cd98fa5c6c110946ce4e0e65 *bionic-server-cloudimg-ppc64el-root.tar.xz 30 | 52f4f22a6539eb3eb2831c0ef397dfc0 *bionic-server-cloudimg-ppc64el.img 31 | dbdb41699bdad42d9261f8e30cdb783b *bionic-server-cloudimg-ppc64el.squashfs 32 | b4a21bda5632afa40a7fa308128c1711 *bionic-server-cloudimg-ppc64el.tar.gz 33 | cacdf11956da128155d74aad4640b3bb *bionic-server-cloudimg-s390x-lxd.tar.xz 34 | 83517bf0af6de844c864bafad5e27ac6 *bionic-server-cloudimg-s390x-root.tar.xz 35 | 44027828beadc3b22224a087917bd06f *bionic-server-cloudimg-s390x.img 36 | d62e6a4243292c597ea6bb1c1ae79b21 *bionic-server-cloudimg-s390x.squashfs 37 | 40b81f5ec26d6775fa25037ec0684fae *bionic-server-cloudimg-s390x.tar.gz 38 | -------------------------------------------------------------------------------- /shakenfist/tests/test_baseobject.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import testtools 4 | from shakenfist import exceptions 5 | from shakenfist.baseobject import DatabaseBackedObject 6 | from shakenfist.baseobject import State 7 | from shakenfist.tests import base 8 | 9 | 10 | class DatabaseBackedObjectTestCase(base.ShakenFistTestCase): 11 | @mock.patch('shakenfist.mariadb.get_state', 12 | side_effect=[ 13 | State(value=None, update_time=2), 14 | State(value=DatabaseBackedObject.STATE_INITIAL, update_time=4), 15 | State(value=DatabaseBackedObject.STATE_CREATED, update_time=10), 16 | ]) 17 | def test_state(self, mock_mariadb_get_state): 18 | d = DatabaseBackedObject('uuid') 19 | self.assertEqual(d.state, State(value=None, update_time=2)) 20 | self.assertEqual(d.state, 21 | State(value=DatabaseBackedObject.STATE_INITIAL, 22 | update_time=4)) 23 | self.assertEqual(d.state, 24 | State(value=DatabaseBackedObject.STATE_CREATED, 25 | update_time=10)) 26 | 27 | def test_property_state_object_full(self): 28 | s = State(value='state1', update_time=3.0) 29 | 30 | self.assertEqual(s.value, 'state1') 31 | self.assertEqual(s.update_time, 3.0) 32 | 33 | self.assertEqual(s.obj_dict(), { 34 | 'value': 'state1', 35 | 'update_time': 3.0, 36 | }) 37 | 38 | self.assertEqual(s, State(value='state1', update_time=3.0)) 39 | self.assertEqual(str(s), 40 | "State({'value': 'state1', 'update_time': 3.0})") 41 | 42 | @mock.patch('shakenfist.eventlog.add_event') 43 | @mock.patch('shakenfist.baseobject.DatabaseBackedObject._db_set_attribute') 44 | @mock.patch('shakenfist.mariadb.get_state', 45 | side_effect=[ 46 | State(value=DatabaseBackedObject.STATE_INITIAL, update_time=4), 47 | State(value=DatabaseBackedObject.STATE_ERROR, update_time=4), 48 | ]) 49 | @mock.patch('shakenfist.baseobject.DatabaseBackedObject._db_get_attribute', 50 | side_effect=[ 51 | {}, 52 | {'message': 'bad error'}, 53 | {'message': 'real bad'}, 54 | ]) 55 | def test_property_error_msg(self, mock_get_attribute, mock_mariadb_get_state, 56 | mock_set_attribute, mock_add_event): 57 | d = DatabaseBackedObject('uuid') 58 | self.assertEqual(d.error, None) 59 | self.assertEqual(d.error, 'bad error') 60 | 61 | with testtools.ExpectedException(exceptions.InvalidStateException): 62 | d.error = 'real bad' 63 | 64 | d.error = 'real bad' 65 | -------------------------------------------------------------------------------- /.github/workflows/ci-dependencies.yml: -------------------------------------------------------------------------------- 1 | name: Rebuild CI dependencies 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '00 00 * * *' # utc 7 | 8 | jobs: 9 | ci-dependencies: 10 | runs-on: [self-hosted, vm] 11 | timeout-minutes: 120 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | steps: 17 | - name: Set environment variables 18 | run: | 19 | echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV 20 | 21 | - name: Checkout the actions repository 22 | uses: actions/checkout@v4 23 | with: 24 | repository: shakenfist/actions 25 | path: actions 26 | fetch-depth: 0 27 | 28 | - name: Install the github command line 29 | run: | 30 | sudo apt update 31 | sudo apt install -y curl 32 | 33 | curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg 34 | sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg 35 | echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null 36 | 37 | sudo apt update 38 | sudo apt install -y gh 39 | 40 | - name: Lookup latest version of the GitHub actions runner 41 | env: 42 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 43 | run: | 44 | actions_url=$(gh release view --repo actions/runner --json assets | \ 45 | jq -r '.assets[].url | select (contains("linux-x64-2")) | select (test("[0-9].tar.gz$"))') 46 | echo "GITHUB_ACTIONS_URL=$actions_url" >> $GITHUB_ENV 47 | 48 | - name: Build dependencies image 49 | run: | 50 | cd ${GITHUB_WORKSPACE}/actions 51 | ansible-playbook -i /home/debian/ansible-hosts \ 52 | --extra-vars "identifier=${SHAKENFIST_NAMESPACE} \ 53 | base_image=sf://label/ci-images/debian-11 base_image_user=debian \ 54 | label=ci-images/dependencies actions_url=$GITHUB_ACTIONS_URL" \ 55 | ansible/ci-dependencies.yml 56 | 57 | - uses: JasonEtco/create-an-issue@v2 58 | if: failure() 59 | id: create-issue 60 | env: 61 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 62 | SF_CI_NAME: ci-dependencies 63 | SF_ACTION_RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} 64 | with: 65 | filename: shakenfist/.github/workflows/ci-images-failure.md 66 | update_existing: true 67 | search_existing: open 68 | 69 | - if: failure() 70 | run: 'echo Created ${{ steps.create-issue.outputs.url }}' -------------------------------------------------------------------------------- /shakenfist/deploy/ansible/roles/primary/tasks/grafana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install prometheus 3 | apt: 4 | name: prometheus 5 | state: latest 6 | register: apt_action 7 | retries: 100 8 | until: apt_action is success or ('Failed to lock apt for exclusive operation' not in apt_action.msg and '/var/lib/dpkg/lock' not in apt_action.msg) 9 | 10 | - name: Write prometheus configuration file 11 | template: 12 | src: templates/etc_prometheus.yml 13 | dest: /etc/prometheus/prometheus.yml 14 | owner: root 15 | mode: u=rw,g=r,o=r 16 | 17 | - name: Restart prometheus 18 | service: 19 | name: prometheus 20 | enabled: yes 21 | state: restarted 22 | 23 | - name: Install Grafana prerequisites 24 | apt: 25 | name: 26 | - apt-transport-https 27 | - software-properties-common 28 | update_cache: yes 29 | state: latest 30 | register: apt_action 31 | retries: 100 32 | until: apt_action is success or ('Failed to lock apt for exclusive operation' not in apt_action.msg and '/var/lib/dpkg/lock' not in apt_action.msg) 33 | 34 | - name: Check if grafana packages are already setup 35 | stat: 36 | path: /etc/apt/sources.list.d/packages_grafana_com_oss_deb.list 37 | register: stat_result 38 | 39 | - name: Add Grafana GPG key 40 | apt_key: url=https://packages.grafana.com/gpg.key 41 | when: not stat_result.stat.exists 42 | 43 | - name: Add Grafana APT repository 44 | apt_repository: 45 | repo: deb [arch=amd64] http://packages.grafana.com/oss/deb stable main 46 | when: not stat_result.stat.exists 47 | 48 | - name: Install Grafana 49 | apt: 50 | name: grafana 51 | update_cache: yes 52 | state: latest 53 | register: apt_action 54 | retries: 100 55 | until: apt_action is success or ('Failed to lock apt for exclusive operation' not in apt_action.msg and '/var/lib/dpkg/lock' not in apt_action.msg) 56 | 57 | - name: Write grafana config 58 | template: 59 | src: templates/grafana.ini 60 | dest: /etc/grafana/grafana.ini 61 | owner: root 62 | mode: u=rw,g=r,o=r 63 | 64 | - name: Write grafana dashboard 65 | copy: 66 | src: files/grafana/provisioning/dashboards/shakenfist.json 67 | dest: /etc/grafana/provisioning/dashboards/shakenfist.json 68 | owner: root 69 | mode: u=rw,g=r,o=r 70 | 71 | - name: Write grafana dashboard config 72 | copy: 73 | src: files/grafana/provisioning/dashboards/dashboards.yaml 74 | dest: /etc/grafana/provisioning/dashboards/dashboards.yaml 75 | owner: root 76 | mode: u=rw,g=r,o=r 77 | 78 | - name: Write prometheus grafana configuration file 79 | template: 80 | src: templates/ds_prometheus.yml 81 | dest: /etc/grafana/provisioning/datasources/prometheus.yml 82 | owner: root 83 | mode: u=rwx,g=r,o=r 84 | 85 | - name: Restart grafana 86 | service: 87 | name: grafana-server 88 | enabled: yes 89 | state: restarted 90 | -------------------------------------------------------------------------------- /.github/workflows/pr-re-review.yml: -------------------------------------------------------------------------------- 1 | name: PR Re-review 2 | 3 | # Triggers a re-review of a PR when an authorized user comments 4 | # "@shakenfist-bot please re-review" 5 | 6 | on: 7 | issue_comment: 8 | types: [created] 9 | 10 | jobs: 11 | check_and_review: 12 | # Only run on PR comments (not issue comments) 13 | if: | 14 | github.event.issue.pull_request && 15 | contains(github.event.comment.body, '@shakenfist-bot please re-review') 16 | runs-on: [self-hosted, claude-code] 17 | name: "Re-review PR" 18 | 19 | steps: 20 | - name: Check commenter permissions 21 | id: check_permission 22 | env: 23 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | run: | 25 | # Get the commenter's permission level for this repository 26 | permission=$(gh api \ 27 | repos/${{ github.repository }}/collaborators/${{ github.event.comment.user.login }}/permission \ 28 | --jq '.permission' 2>/dev/null || echo "none") 29 | 30 | echo "User ${{ github.event.comment.user.login }} has permission: ${permission}" 31 | 32 | # Check if user has write or admin permission (can merge PRs) 33 | if [[ "${permission}" == "admin" || "${permission}" == "write" ]]; then 34 | echo "authorized=true" >> $GITHUB_OUTPUT 35 | echo "User is authorized to request re-review" 36 | else 37 | echo "authorized=false" >> $GITHUB_OUTPUT 38 | echo "User is not authorized to request re-review" 39 | fi 40 | 41 | - name: React to comment 42 | if: steps.check_permission.outputs.authorized == 'true' 43 | env: 44 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 45 | run: | 46 | # Add a reaction to acknowledge the request 47 | gh api \ 48 | repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \ 49 | -f content='+1' \ 50 | --silent || true 51 | 52 | - name: Post unauthorized message 53 | if: steps.check_permission.outputs.authorized == 'false' 54 | env: 55 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 56 | run: | 57 | gh pr comment ${{ github.event.issue.number }} \ 58 | --body "Sorry @${{ github.event.comment.user.login }}, only repository collaborators with write access can request a re-review." 59 | 60 | - name: Checkout code 61 | if: steps.check_permission.outputs.authorized == 'true' 62 | uses: actions/checkout@v4 63 | with: 64 | fetch-depth: 0 65 | 66 | - name: Run automated reviewer 67 | if: steps.check_permission.outputs.authorized == 'true' 68 | env: 69 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 70 | run: | 71 | tools/review-pr-with-claude.sh --ci --force --pr ${{ github.event.issue.number }} 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | .stestr/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | db.sqlite3 64 | db.sqlite3-journal 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 97 | __pypackages__/ 98 | 99 | # Celery stuff 100 | celerybeat-schedule 101 | celerybeat.pid 102 | 103 | # SageMath parsed files 104 | *.sage.py 105 | 106 | # Environments 107 | .env 108 | .venv 109 | env/ 110 | venv/ 111 | ENV/ 112 | env.bak/ 113 | venv.bak/ 114 | 115 | # Spyder project settings 116 | .spyderproject 117 | .spyproject 118 | 119 | # Rope project settings 120 | .ropeproject 121 | 122 | # mkdocs documentation 123 | /site 124 | 125 | # mypy 126 | .mypy_cache/ 127 | .dmypy.json 128 | dmypy.json 129 | 130 | # Pyre type checker 131 | .pyre/ 132 | 133 | # Terraform 134 | .terraform/ 135 | *.tfstate* 136 | 137 | # Deployer git repos 138 | deploy/gitrepos/ 139 | 140 | # VS Code - ignore user-specific files but keep workspace settings 141 | .vscode/* 142 | !.vscode/settings.json 143 | !.vscode/extensions.json 144 | 145 | # Generated version file 146 | shakenfist/_version.py 147 | 148 | # Claude planning files 149 | PLAN-*.md 150 | -------------------------------------------------------------------------------- /shakenfist/deploy/shakenfist_ci/cluster_ci_tests/test_upgrades.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from shakenfist_ci import base 4 | 5 | 6 | class TestUpgrades(base.BaseTestCase): 7 | def test_upgraded_data_exists(self): 8 | # There is an upgraded namespace called 'upgrade' 9 | if 'upgrade' not in self.system_client.get_namespaces(): 10 | self.skipTest('There is no upgrade namespace') 11 | 12 | # Collect networks and check 13 | networks_by_name = {} 14 | networks_by_uuid = {} 15 | for net in self.system_client.get_networks(): 16 | networks_by_name['{}/{}'.format(net['namespace'], net['name'])] = net 17 | networks_by_uuid[net['uuid']] = net 18 | 19 | self.assertIn('upgrade/upgrade-fe', networks_by_name) 20 | self.assertIn('upgrade/upgrade-be', networks_by_name) 21 | 22 | sys.stderr.write( 23 | 'Discovered networks post upgrade: %s\n' % networks_by_name) 24 | 25 | # Collect instances and check 26 | instances = {} 27 | for inst in self.system_client.get_instances(): 28 | instances['{}/{}'.format(inst['namespace'], inst['name'])] = inst 29 | 30 | sys.stderr.write( 31 | 'Discovered instances post upgrade: %s\n' % instances) 32 | 33 | # Determine interface information 34 | addresses = {} 35 | for name in ['upgrade/fe', 'upgrade/be-1', 'upgrade/be-2']: 36 | sys.stderr.write('Looking up interfaces for %s\n' % name) 37 | self.assertIn(name, instances) 38 | for iface in self.system_client.get_instance_interfaces(instances[name]['uuid']): 39 | sys.stderr.write(f'{name} has interface {iface}\n') 40 | net_name = networks_by_uuid.get( 41 | iface['network_uuid'], {'name': 'unknown'})['name'] 42 | addresses[f'{name}/{net_name}'] = iface['ipv4'] 43 | 44 | sys.stderr.write( 45 | 'Discovered addresses post upgrade: %s\n' % addresses) 46 | 47 | # Ensure we can ping all instances 48 | self._test_ping( 49 | instances['upgrade/fe']['uuid'], 50 | networks_by_name['upgrade/upgrade-fe']['uuid'], 51 | addresses['upgrade/fe/upgrade-fe'], 52 | True) 53 | self._test_ping( 54 | instances['upgrade/fe']['uuid'], 55 | networks_by_name['upgrade/upgrade-be']['uuid'], 56 | addresses['upgrade/fe/upgrade-be'], 57 | True) 58 | 59 | self._test_ping( 60 | instances['upgrade/be-1']['uuid'], 61 | networks_by_name['upgrade/upgrade-be']['uuid'], 62 | addresses['upgrade/be-1/upgrade-be'], 63 | True) 64 | self._test_ping( 65 | instances['upgrade/be-2']['uuid'], 66 | networks_by_name['upgrade/upgrade-be']['uuid'], 67 | addresses['upgrade/be-2/upgrade-be'], 68 | True) 69 | -------------------------------------------------------------------------------- /docs/operator_guide/artifacts.md: -------------------------------------------------------------------------------- 1 | # Artifacts 2 | 3 | ## Checksums 4 | 5 | As of Shaken Fist v0.7, blob replicas are regularly checksummed to verify that 6 | data loss has not occurred. The following events imply a checksum operation: 7 | 8 | * snapshotting an NVRAM template. 9 | * creation of a new blob replica by transfer of a blob from another machine in 10 | the cluster (the destination is checksummed to verify the transfer). 11 | * transcode of a blob into a new format (the new format is stored as a 12 | separate blob). 13 | * conversion of an upload to an artifact. 14 | 15 | The following events _should_ imply an artifact checksum, but we found that 16 | performance suffered too much for very large blobs: 17 | 18 | * download of a new blob from an external source (artifact fetch for example). 19 | * snapshotting a disk. 20 | 21 | Additionally, all blob replicas are regularly checksummed and compared with what 22 | the record in etcd believes the correct value should be. These comparisons are 23 | rate limited, but should happen with a maximum frequency of 24 | CHECKSUM_VERIFICATION_FREQUENCY seconds, which defaults to every 24 hours. It is 25 | possible if you have a large number of blob replicas on a given node that the node 26 | will be unable to keep up with checksum operations. 27 | 28 | If a blob replica fails the checksum verification, CHECKSUM_ENFORCEMENT is set 29 | to True _and is not in use on that node_, then the replica is deleted and the 30 | cluster will re-replicate the blob as required. If the blob replica is in use, 31 | there isn't much Shaken Fist can do without disturbing running instances, so the 32 | error is logged and then ignored for now. 33 | 34 | Checksums are also used when a new version of an artifact is created. If the 35 | checksum of the previous version is the same as the checksum for the proposed 36 | new version, the proposed new version is skipped. Artifact uploads from v0.7 can 37 | also skip actual upload of the contents of the artifact if there is already a 38 | blob in the cluster with a matching checksum. 39 | 40 | ## Sharing artifacts 41 | 42 | Artifacts in the system namespace can be shared with all other namespaces. 43 | Artifacts shared like this appear to the other namespaces as if they are local 44 | to the other namespace, although non-system namespaces should not be able to 45 | update such an artifact. This is useful if you have official or commonly used 46 | images which you want to provide all users of a cluster -- for example an 47 | official CentOS image that many users will want. 48 | 49 | ???+ info 50 | 51 | Another option for sharing artifacts is the "trusts" relationship between two 52 | namespaces, which is discussed in the [authentication section of the operator guide](authentication.md). 53 | 54 | To share an artifact, use the command line client like this: 55 | 56 | `sf-client artifact share ...uuid...` 57 | 58 | To unshare an artifact, do this: 59 | 60 | `sf-client artifact unshare ...uuid...` -------------------------------------------------------------------------------- /shakenfist/protos/event_pb2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by the protocol buffer compiler. DO NOT EDIT! 3 | # NO CHECKED-IN PROTOBUF GENCODE 4 | # source: event.proto 5 | # Protobuf Python Version: 5.29.0 6 | """Generated protocol buffer code.""" 7 | from google.protobuf import descriptor as _descriptor 8 | from google.protobuf import descriptor_pool as _descriptor_pool 9 | from google.protobuf import runtime_version as _runtime_version 10 | from google.protobuf import symbol_database as _symbol_database 11 | from google.protobuf.internal import builder as _builder 12 | _runtime_version.ValidateProtobufRuntimeVersion( 13 | _runtime_version.Domain.PUBLIC, 14 | 5, 15 | 29, 16 | 0, 17 | '', 18 | 'event.proto' 19 | ) 20 | # @@protoc_insertion_point(imports) 21 | 22 | _sym_db = _symbol_database.Default() 23 | 24 | 25 | 26 | 27 | DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x65vent.proto\x12\x11shakenfist.protos\"\xd7\x01\n\x0c\x45ventRequest\x12\x13\n\x0bobject_type\x18\x01 \x01(\t\x12\x13\n\x0bobject_uuid\x18\x02 \x01(\t\x12\x12\n\nevent_type\x18\x03 \x01(\t\x12\x1f\n\x12obsolete_timestamp\x18\x04 \x01(\x02H\x00\x88\x01\x01\x12\x0c\n\x04\x66qdn\x18\x05 \x01(\t\x12\x10\n\x08\x64uration\x18\x06 \x01(\x02\x12\x0f\n\x07message\x18\x07 \x01(\t\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x11\n\ttimestamp\x18\t \x01(\x01\x42\x15\n\x13_obsolete_timestamp\"7\n\x0b\x45ventObject\x12\x13\n\x0bobject_type\x18\x01 \x01(\t\x12\x13\n\x0bobject_uuid\x18\x02 \x01(\t\"\xab\x01\n\x11\x45ventMultiRequest\x12/\n\x07objects\x18\x01 \x03(\x0b\x32\x1e.shakenfist.protos.EventObject\x12\x12\n\nevent_type\x18\x02 \x01(\t\x12\x0c\n\x04\x66qdn\x18\x03 \x01(\t\x12\x10\n\x08\x64uration\x18\x04 \x01(\x02\x12\x0f\n\x07message\x18\x05 \x01(\t\x12\r\n\x05\x65xtra\x18\x06 \x01(\t\x12\x11\n\ttimestamp\x18\x07 \x01(\x01\"\x19\n\nEventReply\x12\x0b\n\x03\x61\x63k\x18\x01 \x01(\x08\x32\xba\x01\n\x0c\x45ventService\x12O\n\x0bRecordEvent\x12\x1f.shakenfist.protos.EventRequest\x1a\x1d.shakenfist.protos.EventReply\"\x00\x12Y\n\x10RecordMultiEvent\x12$.shakenfist.protos.EventMultiRequest\x1a\x1d.shakenfist.protos.EventReply\"\x00\x62\x06proto3') 28 | 29 | _globals = globals() 30 | _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) 31 | _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'event_pb2', _globals) 32 | if not _descriptor._USE_C_DESCRIPTORS: 33 | DESCRIPTOR._loaded_options = None 34 | _globals['_EVENTREQUEST']._serialized_start=35 35 | _globals['_EVENTREQUEST']._serialized_end=250 36 | _globals['_EVENTOBJECT']._serialized_start=252 37 | _globals['_EVENTOBJECT']._serialized_end=307 38 | _globals['_EVENTMULTIREQUEST']._serialized_start=310 39 | _globals['_EVENTMULTIREQUEST']._serialized_end=481 40 | _globals['_EVENTREPLY']._serialized_start=483 41 | _globals['_EVENTREPLY']._serialized_end=508 42 | _globals['_EVENTSERVICE']._serialized_start=511 43 | _globals['_EVENTSERVICE']._serialized_end=697 44 | # @@protoc_insertion_point(module_scope) 45 | -------------------------------------------------------------------------------- /shakenfist/protos/nodelock_pb2.pyi: -------------------------------------------------------------------------------- 1 | from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper 2 | from google.protobuf import descriptor as _descriptor 3 | from google.protobuf import message as _message 4 | from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union 5 | 6 | DESCRIPTOR: _descriptor.FileDescriptor 7 | 8 | class LockRequest(_message.Message): 9 | __slots__ = ("requester", "key") 10 | REQUESTER_FIELD_NUMBER: _ClassVar[int] 11 | KEY_FIELD_NUMBER: _ClassVar[int] 12 | requester: str 13 | key: str 14 | def __init__(self, requester: _Optional[str] = ..., key: _Optional[str] = ...) -> None: ... 15 | 16 | class UnlockRequest(_message.Message): 17 | __slots__ = ("requester", "key") 18 | REQUESTER_FIELD_NUMBER: _ClassVar[int] 19 | KEY_FIELD_NUMBER: _ClassVar[int] 20 | requester: str 21 | key: str 22 | def __init__(self, requester: _Optional[str] = ..., key: _Optional[str] = ...) -> None: ... 23 | 24 | class NodeLockRequest(_message.Message): 25 | __slots__ = ("lock_request", "unlock_request") 26 | LOCK_REQUEST_FIELD_NUMBER: _ClassVar[int] 27 | UNLOCK_REQUEST_FIELD_NUMBER: _ClassVar[int] 28 | lock_request: LockRequest 29 | unlock_request: UnlockRequest 30 | def __init__(self, lock_request: _Optional[_Union[LockRequest, _Mapping]] = ..., unlock_request: _Optional[_Union[UnlockRequest, _Mapping]] = ...) -> None: ... 31 | 32 | class LockReply(_message.Message): 33 | __slots__ = ("outcome",) 34 | class Outcome(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): 35 | __slots__ = () 36 | OK: _ClassVar[LockReply.Outcome] 37 | ALREADY_HELD: _ClassVar[LockReply.Outcome] 38 | DENIED: _ClassVar[LockReply.Outcome] 39 | OK: LockReply.Outcome 40 | ALREADY_HELD: LockReply.Outcome 41 | DENIED: LockReply.Outcome 42 | OUTCOME_FIELD_NUMBER: _ClassVar[int] 43 | outcome: LockReply.Outcome 44 | def __init__(self, outcome: _Optional[_Union[LockReply.Outcome, str]] = ...) -> None: ... 45 | 46 | class UnlockReply(_message.Message): 47 | __slots__ = ("outcome",) 48 | class Outcome(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): 49 | __slots__ = () 50 | OK: _ClassVar[UnlockReply.Outcome] 51 | NOT_HELD: _ClassVar[UnlockReply.Outcome] 52 | OK: UnlockReply.Outcome 53 | NOT_HELD: UnlockReply.Outcome 54 | OUTCOME_FIELD_NUMBER: _ClassVar[int] 55 | outcome: UnlockReply.Outcome 56 | def __init__(self, outcome: _Optional[_Union[UnlockReply.Outcome, str]] = ...) -> None: ... 57 | 58 | class NodeLockReply(_message.Message): 59 | __slots__ = ("lock_reply", "unlock_reply") 60 | LOCK_REPLY_FIELD_NUMBER: _ClassVar[int] 61 | UNLOCK_REPLY_FIELD_NUMBER: _ClassVar[int] 62 | lock_reply: LockReply 63 | unlock_reply: UnlockReply 64 | def __init__(self, lock_reply: _Optional[_Union[LockReply, _Mapping]] = ..., unlock_reply: _Optional[_Union[UnlockReply, _Mapping]] = ...) -> None: ... 65 | --------------------------------------------------------------------------------