├── .gitattributes ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ ├── add-to-sdk-team-project.yml │ ├── build_publish.yml │ ├── goth-nightly.yml │ ├── goth.yml │ └── tests.yml ├── .gitignore ├── .readthedocs.yaml ├── LICENSE ├── README.md ├── art ├── exposing-container.gif └── gc-nodes.svg ├── docs ├── Low-Level-Api.md ├── VM-Tutorial.md ├── diagrams │ ├── README.md │ ├── classes.puml │ ├── ctrl-c.puml │ ├── emit_events_async.puml │ ├── emit_events_summary.puml │ ├── execute_tasks.puml │ ├── execute_tasks.svg │ └── process_batches.puml └── sphinx │ ├── api.rst │ ├── conf.py │ └── index.rst ├── examples ├── blender │ ├── .gitignore │ ├── blender.Dockerfile │ ├── blender.py │ ├── blender_registry_usage.py │ ├── cubes.blend │ ├── run-blender.sh │ └── start_stop_blender.py ├── custom-usage-counter │ ├── README.md │ └── custom_usage_counter.py ├── custom_runtime │ └── custom_runtime.py ├── external-api-request │ ├── Dockerfile │ ├── external_api_request.py │ ├── external_api_request_partner.py │ ├── golem_sign.pem │ ├── manifest.json.base64.sha256.sig │ ├── manifest_partner_unrestricted.json │ ├── manifest_whitelist.json │ ├── partner_rule_poc.md │ ├── request.sh │ └── sign.sh ├── hello-world │ ├── Dockerfile │ ├── hello.py │ └── hello_service.py ├── http-proxy │ ├── Dockerfile │ └── http_proxy.py ├── low-level-api │ └── list-offers.py ├── market-strategy │ └── market_strategy.py ├── scan │ └── scan.py ├── simple-service-poc │ ├── simple_service.py │ └── simple_service │ │ ├── README.md │ │ ├── simple_service.Dockerfile │ │ ├── simple_service.py │ │ ├── simulate_observations.py │ │ └── simulate_observations_ctl.py ├── ssh │ ├── Dockerfile │ └── ssh.py ├── transfer-progress │ └── progress.py ├── utils │ └── __init__.py ├── webapp │ ├── README.md │ ├── db │ │ ├── Dockerfile │ │ └── run_rqlite.sh │ ├── http │ │ ├── Dockerfile │ │ ├── app.py │ │ └── templates │ │ │ └── index.html │ ├── webapp.py │ └── webapp_suspend_resume.py ├── webapp_fileupload │ ├── http-file-upload │ │ ├── Dockerfile │ │ ├── app.py │ │ └── templates │ │ │ ├── error.html │ │ │ └── index.html │ └── webapp_fileupload.py └── yacat │ ├── .gitignore │ ├── README.md │ ├── yacat.Dockerfile │ └── yacat.py ├── pydoc-markdown.yml ├── pyproject.toml ├── tests ├── __init__.py ├── conftest.py ├── contrib │ ├── __init__.py │ └── service │ │ ├── __init__.py │ │ └── test_chunk.py ├── drone │ ├── Dockerfile │ ├── __init__.py │ ├── drone.py │ └── task.sh ├── engine │ ├── __init__.py │ ├── test_debit_note_intervals.py │ └── test_engine.py ├── events │ ├── __init__.py │ ├── test_frozen.py │ └── test_repr.py ├── executor │ ├── __init__.py │ ├── test_smartq.py │ └── test_task.py ├── factories │ ├── __init__.py │ ├── agreements_pool.py │ ├── config.py │ ├── context.py │ ├── events.py │ ├── golem.py │ ├── network.py │ ├── props │ │ ├── __init__.py │ │ └── com.py │ └── rest │ │ ├── __init__.py │ │ ├── market.py │ │ └── payment.py ├── goth_tests │ ├── ___test_resubscription.py │ ├── __init__.py │ ├── _util.py │ ├── assertions.py │ ├── conftest.py │ ├── test_agreement_termination │ │ ├── __init__.py │ │ ├── requestor.py │ │ └── test_agreement_termination.py │ ├── test_async_task_generation │ │ ├── __init__.py │ │ ├── requestor.py │ │ └── test_async_task_generation.py │ ├── test_concurrent_executors │ │ ├── __init__.py │ │ ├── requestor.py │ │ └── test_concurrent_executors.py │ ├── test_instance_restart │ │ ├── __init__.py │ │ ├── requestor.py │ │ └── test_instance_restart.py │ ├── test_mid_agreement_payments │ │ ├── __init__.py │ │ ├── requestor_agent.py │ │ └── test_mid_agreement_payments.py │ ├── test_multiactivity_agreement │ │ ├── __init__.py │ │ ├── requestor.py │ │ └── test_multiactivity_agreement.py │ ├── test_power_outage.py │ ├── test_recycle_ip │ │ ├── __init__.py │ │ ├── ssh_recycle_ip.py │ │ └── test_recycle_ip.py │ ├── test_renegotiate_proposal │ │ ├── __init__.py │ │ ├── requestor.py │ │ └── test_renegotiate_proposal.py │ ├── test_run_blender.py │ ├── test_run_custom_usage_counter.py │ ├── test_run_scan.py │ ├── test_run_simple_service.py │ ├── test_run_ssh.py │ ├── test_run_webapp.py │ └── test_run_yacat.py ├── payload │ ├── __init__.py │ ├── test_manifest.py │ ├── test_payload.py │ └── test_repo.py ├── props │ ├── __init__.py │ ├── test_base.py │ ├── test_builder.py │ ├── test_com.py │ └── test_from_properties.py ├── rest │ ├── __init__.py │ ├── test_activity.py │ ├── test_allocation.py │ ├── test_demand_builder.py │ └── test_repeat_on_error.py ├── script │ ├── __init__.py │ └── test_script.py ├── services │ ├── __init__.py │ ├── test_cluster.py │ └── test_service_runner.py ├── storage │ ├── __init__.py │ ├── test_gftp.py │ └── test_storage.py ├── strategy │ ├── __init__.py │ ├── helpers.py │ ├── test_base.py │ ├── test_decrease_score_for_unconfirmed.py │ ├── test_default_strategies.py │ ├── test_provider_filter.py │ └── test_wrapping_strategy.py ├── test_add_event_consumer.py ├── test_agreements_pool.py ├── test_async_wrapper.py ├── test_config.py ├── test_ctx.py ├── test_log.py ├── test_network.py ├── test_payment_platforms.py ├── test_utils.py └── test_yapapi.py └── yapapi ├── __init__.py ├── agreements_pool.py ├── config.py ├── contrib ├── __init__.py ├── service │ ├── __init__.py │ ├── chunk.py │ ├── http_proxy.py │ └── socket_proxy.py └── strategy │ ├── __init__.py │ └── provider_filter.py ├── ctx.py ├── engine.py ├── event_dispatcher.py ├── events.py ├── executor ├── __init__.py ├── _smartq.py └── task.py ├── golem.py ├── invoice_manager.py ├── log.py ├── network.py ├── payload ├── __init__.py ├── manifest.py ├── package.py └── vm.py ├── props ├── __init__.py ├── base.py ├── builder.py ├── com.py └── inf.py ├── py.typed ├── rest ├── __init__.py ├── activity.py ├── common.py ├── configuration.py ├── market.py ├── net.py ├── payment.py └── resource.py ├── script ├── __init__.py ├── capture.py └── command.py ├── services ├── __init__.py ├── cluster.py ├── service.py ├── service_runner.py └── service_state.py ├── storage ├── __init__.py ├── gftp.py └── webdav.py ├── strategy ├── __init__.py ├── base.py ├── decrease_score_unconfirmed.py ├── dummy.py ├── least_expensive.py └── wrapping_strategy.py └── utils.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.gif filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @golemfactory/ya-sdk 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Description 11 | **OS**: 12 | [ e.g. Ubuntu 18.04, Windows 10, etc ] 13 | 14 | **yagna daemon version**: 15 | [ can be determined with `yagna --version`] 16 | 17 | **Python version**: 18 | [ can be determined with `python --version` ] 19 | 20 | **yapapi library version**: 21 | [ can be determined using: `python -c "import yapapi; print(yapapi.__version__)"`] 22 | 23 | **yapapi branch**: 24 | [ if you're using one of our included examples ] 25 | 26 | **Description of the issue**: 27 | _A clear and concise description of what went wrong, in which component, when and where._ 28 | 29 | **Actual result**: 30 | _What is the observed behavior and/or result in this issue_ 31 | 32 | **Screenshots**: 33 | _If applicable, add screenshots to help explain your problem._ 34 | 35 | ## Steps To Reproduce 36 | _Short description of steps to reproduce the behavior:_ 37 | e.g. 38 | 1. Launch the daemon with '...' 39 | 2. Run the example: '...' 40 | 3. See error 41 | 42 | ## Expected behavior 43 | _(What is the expected behavior and/or result in this scenario)_ 44 | 45 | ## Logs and any additional context 46 | _(Any additional information that could help to resolve the issue, which systems were checked to reproduce the issue)_ 47 | _Please upload your logs if possible_ 48 | -------------------------------------------------------------------------------- /.github/workflows/add-to-sdk-team-project.yml: -------------------------------------------------------------------------------- 1 | name: Add New Issues To SDK Team Project 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | 8 | jobs: 9 | add-to-project: 10 | name: Add issue to project 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/add-to-project@v0.1.0 14 | with: 15 | project-url: https://github.com/orgs/golemfactory/projects/32 16 | github-token: ${{ secrets.SDK_BOARD_ACTIONS_TOKEN }} 17 | -------------------------------------------------------------------------------- /.github/workflows/build_publish.yml: -------------------------------------------------------------------------------- 1 | name: Build and publish the release 2 | 3 | on: 4 | release: 5 | types: [prereleased, released] 6 | 7 | jobs: 8 | test: 9 | name: Run checks 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: actions/setup-python@v4 15 | with: 16 | python-version: '3.9' 17 | - uses: Gr1N/setup-poetry@v8 18 | - run: poetry install 19 | - run: poetry run poe tests_unit 20 | - run: poetry run poe checks_codestyle 21 | - run: poetry run poe checks_typing 22 | - run: poetry run poe checks_license 23 | 24 | build: 25 | needs: [test] 26 | name: Build the release 27 | runs-on: ubuntu-latest 28 | 29 | steps: 30 | - uses: actions/checkout@v3 31 | - uses: actions/setup-python@v4 32 | with: 33 | python-version: '3.9' 34 | - uses: Gr1N/setup-poetry@v8 35 | - name: Get git release tag 36 | run: echo "git-release-tag=yapapi $(git describe --tags)" >> $GITHUB_OUTPUT 37 | id: git_describe 38 | - name: Get package version 39 | run: echo "poetry-version=$(poetry version)" >> $GITHUB_OUTPUT 40 | id: poetry_version 41 | - name: Fail on version mismatch 42 | run: exit 1 43 | if: 44 | ${{ steps.git_describe.outputs.git-release-tag != 45 | steps.poetry_version.outputs.poetry-version }} 46 | - name: Build the release 47 | run: poetry build 48 | - name: Store the built package 49 | uses: actions/upload-artifact@v4 50 | with: 51 | name: dist 52 | path: dist 53 | 54 | test_publish: 55 | needs: [build] 56 | name: Publish the release to test.pypi 57 | runs-on: ubuntu-latest 58 | if: ${{ github.event.action == 'prereleased' }} 59 | 60 | steps: 61 | - uses: actions/checkout@v3 62 | - uses: actions/setup-python@v4 63 | with: 64 | python-version: '3.9' 65 | - uses: Gr1N/setup-poetry@v8 66 | - name: Retrieve the built package 67 | uses: actions/download-artifact@v4 68 | with: 69 | name: dist 70 | path: dist 71 | - name: Publish to pypi 72 | run: | 73 | poetry config repositories.testpypi https://test.pypi.org/legacy/ 74 | poetry publish -r testpypi -u __token__ -p ${{ secrets.TESTPYPI_TOKEN }} 75 | 76 | publish: 77 | needs: [build] 78 | name: Publish the release 79 | runs-on: ubuntu-latest 80 | if: ${{ github.event.action == 'released' }} 81 | 82 | steps: 83 | - uses: actions/checkout@v3 84 | - uses: actions/setup-python@v4 85 | with: 86 | python-version: '3.9' 87 | - uses: Gr1N/setup-poetry@v8 88 | - name: Retrieve the built package 89 | uses: actions/download-artifact@v4 90 | with: 91 | name: dist 92 | path: dist 93 | - name: Publish to pypi 94 | run: | 95 | poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }} 96 | -------------------------------------------------------------------------------- /.github/workflows/goth.yml: -------------------------------------------------------------------------------- 1 | name: Goth (PR and push) 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - master 8 | # - # put your branch name here to test it @ GH Actions 9 | pull_request: 10 | branches: 11 | - master 12 | - b0.* 13 | 14 | jobs: 15 | goth-tests: 16 | name: Run integration tests 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | 22 | - name: Configure python 23 | uses: actions/setup-python@v5 24 | with: 25 | python-version: '3.10' 26 | 27 | - name: Install Poetry 28 | run: curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2 29 | 30 | - name: Install dependencies 31 | run: | 32 | poetry install 33 | 34 | - name: Initialize the test suite 35 | run: poetry run poe tests_integration_init 36 | 37 | - name: Run test suite 38 | env: 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | run: poetry run poe tests_integration 41 | 42 | - name: Upload test logs 43 | uses: actions/upload-artifact@v4 44 | if: always() 45 | with: 46 | name: goth-logs 47 | path: /tmp/goth-tests 48 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Continuous integration 2 | on: 3 | push: 4 | branches: 5 | - master 6 | - b0.* 7 | pull_request: 8 | branches: 9 | - master 10 | - b0.* 11 | 12 | jobs: 13 | test: 14 | name: Run checks 15 | runs-on: ${{ matrix.os }} 16 | strategy: 17 | matrix: 18 | python-version: ["3.8", "3.9", "3.10"] 19 | os: 20 | - ubuntu-latest 21 | - macos-latest 22 | - windows-latest 23 | exclude: 24 | - os: windows-latest 25 | python-version: "3.10" 26 | - os: macos-latest 27 | python-version: "3.10" 28 | fail-fast: false 29 | 30 | steps: 31 | - uses: actions/checkout@v3 32 | - uses: actions/setup-python@v4 33 | with: 34 | python-version: ${{ matrix.python-version }} 35 | - uses: Gr1N/setup-poetry@v8 36 | 37 | - run: echo "ENABLE=1" >> $GITHUB_OUTPUT 38 | if: ${{ matrix.os == 'ubuntu-latest' }} 39 | name: Enable extended checks 40 | id: extended-checks 41 | - run: echo "ENABLE=1" >> $GITHUB_OUTPUT 42 | if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.9' }} 43 | name: Enable sphinx check 44 | id: extended-checks-sphinx 45 | 46 | - run: poetry install 47 | if: ${{ !steps.extended-checks-sphinx.outputs.ENABLE }} 48 | - run: poetry install -E docs 49 | if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }} 50 | 51 | - run: poetry run poe tests_unit 52 | - run: poetry run poe checks_codestyle 53 | - run: poetry run poe checks_typing 54 | if: ${{ steps.extended-checks.outputs.ENABLE }} 55 | - run: poetry run poe checks_license 56 | if: ${{ steps.extended-checks.outputs.ENABLE }} 57 | - run: poetry run poe sphinx -W 58 | if: ${{ steps.extended-checks-sphinx.outputs.ENABLE }} 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .mypy_cache 3 | .idea 4 | .vscode 5 | *.egg-info 6 | 7 | # files created by tests and examples 8 | *.log 9 | *.png 10 | .coverage 11 | requirements.txt 12 | 13 | tests/goth_tests/assets/ 14 | 15 | dist/ 16 | poetry.lock 17 | 18 | 19 | #Added by cargo 20 | 21 | /target 22 | 23 | # local config files 24 | .pre-commit-config.yaml 25 | 26 | /build 27 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | python: 4 | version: "3.8" 5 | install: 6 | - method: pip 7 | path: . 8 | extra_requirements: 9 | - docs 10 | -------------------------------------------------------------------------------- /art/exposing-container.gif: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:856d621390bb1207128e07387a0f2208002c3373146ff6e20cc9b89fc3ca1cdc 3 | size 8895229 4 | -------------------------------------------------------------------------------- /docs/Low-Level-Api.md: -------------------------------------------------------------------------------- 1 | # Low Level API 2 | 3 | ## General Concept 4 | 5 | ![](../art/gc-nodes.svg) 6 | 7 | ```python 8 | from yapapi.rest import Configuration, Market, Activity, Payment 9 | from yapapi import props as yp 10 | from yapapi.props.builder import DemandBuilder 11 | from datetime import datetime, timezone 12 | 13 | 14 | async def list_offers(conf: Configuration): 15 | async with conf.market() as client: 16 | market_api = Market(client) 17 | dbuild = DemandBuilder() 18 | dbuild.add(yp.NodeInfo(name="some scanning node")) 19 | dbuild.add(yp.Activity(expiration=datetime.now(timezone.utc))) 20 | 21 | async with market_api.subscribe( 22 | dbuild.properties, dbuild.constraints 23 | ) as subscription: 24 | async for event in subscription.events(): 25 | print("event=", event) 26 | print("done") 27 | ``` 28 | 29 | ## Access Configuration 30 | 31 | ### Class `yapapi.rest.Configuration(...)` 32 | 33 | **Initialization Arguments** 34 | 35 | `app_key`: (optional) str : Defines access token to API Gateway 36 | -------------------------------------------------------------------------------- /docs/VM-Tutorial.md: -------------------------------------------------------------------------------- 1 | # VM Runtime Enviroment 2 | 3 | ## Prerequisites 4 | 5 | - docker engine >= 19.03.6 6 | - python >=3.6 7 | - yagna (base package) >=0.3.0 8 | 9 | ## Development Setup 10 | 11 | TODO 12 | 13 | ## Exposing container 14 | 15 | ![Exposing Container](../art/exposing-container.gif) 16 | 17 | ## Example app 18 | -------------------------------------------------------------------------------- /docs/diagrams/README.md: -------------------------------------------------------------------------------- 1 | This directory contains [PlantUML](https://plantuml.com) files for UML sequence and class diagrams 2 | illustrating various aspect of `yapapi` code architecture and execution. 3 | 4 | To create diagrams from `.puml` files you need `java` and `plantuml.jar` 5 | which can be downloaded from https://sourceforge.net/projects/plantuml/files/plantuml.jar/download. 6 | 7 | Generate `diagram.png` from the source file `diagram.puml` with: 8 | ```shell script 9 | java -jar plantuml.jar diagram.puml 10 | ``` 11 | 12 | See https://plantuml.com/starting for more information on using PlantUML 13 | (for example, how to generate diagram files in formats other than PNG) 14 | and on the syntax of `.puml` files. 15 | -------------------------------------------------------------------------------- /docs/diagrams/classes.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title Main yapapi classes 3 | 4 | hide empty fields 5 | 6 | class Golem { 7 | execute_tasks() 8 | run_service() 9 | } 10 | 11 | together { 12 | class Executor <> { 13 | submit() 14 | add_job() 15 | process_batches() 16 | } 17 | class Cluster <> 18 | ' Cluster -[hidden] Executor 19 | } 20 | 21 | class _Engine <> { 22 | process_batches() 23 | start_worker() 24 | } 25 | 26 | class Job { 27 | find_offers() 28 | } 29 | 30 | class AgreementsPool 31 | 32 | 33 | 34 | Golem --|> _Engine 35 | Golem .. "creates" Executor 36 | Golem .. "creates" Cluster 37 | 38 | Executor -- "1" _Engine 39 | Cluster -- "1" _Engine 40 | ' Cluster -- "1" Job 41 | 42 | _Engine o-- "0..*" Job 43 | 44 | Job *-- "1" AgreementsPool 45 | 46 | class Payload 47 | ' Executor --> Payload 48 | ' Cluster --> Payload 49 | Job "0..*" -- "1" Payload 50 | 51 | class Service 52 | class MyService 53 | MyService --|> Service 54 | 55 | Cluster *-- "0..*" Service 56 | (Cluster, Service) .. ServiceInstance 57 | 58 | MyService -[hidden]-- AgreementsPool 59 | 60 | @enduml 61 | -------------------------------------------------------------------------------- /docs/diagrams/ctrl-c.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title Handling Ctrl+C 3 | 4 | hide footbox 5 | 6 | participant "_ _main_ _" as main 7 | 8 | participant ":EventLoop" as loop 9 | 10 | participant "task:asyncio.Task" as task 11 | 12 | activate main 13 | 14 | create loop 15 | main -> loop : get_event_loop() 16 | 17 | create task 18 | main -> task : loop.create_task() 19 | note right: task runs yapapi code 20 | 21 | main -> loop : run_until_complete(task) 22 | 23 | activate loop 24 | loop -> task ++ : <> 25 | deactivate 26 | 27 | loop -> task ++ : <> 28 | deactivate 29 | 30 | 'user -> loop : Ctrl+C 31 | 32 | == Ctrl+C pressed == 33 | 34 | return raise KeyboardInterrupt 35 | 36 | main -> main : except KeyboardInterrupt 37 | main -> task ++ : cancel() 38 | deactivate 39 | 40 | main -> loop ++ : run_until_complete(task) 41 | loop -> task ++ : throw CancelledError 42 | task -> task : except CancelledError 43 | note over task: handle CancelledError\ne.g. peform Executor shutdown 44 | return 45 | return 46 | 47 | @enduml 48 | -------------------------------------------------------------------------------- /docs/diagrams/emit_events_async.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title Emitting events using AsyncWrapper 3 | hide footbox 4 | skinparam BoxPadding 10 5 | 6 | box "yapapi 'core'" 7 | participant ":_Engine" as engine 8 | participant ":AsyncWrapper" as wrapper 9 | participant ":asyncio.Queue" as queue 10 | participant "worker" as worker <> 11 | end box 12 | 13 | box "user code" 14 | participant "event_consumer" as consumer 15 | end box 16 | 17 | activate engine 18 | create wrapper 19 | engine -> wrapper : <> 20 | 21 | create queue 22 | wrapper -> queue : <> 23 | 24 | create worker 25 | wrapper -> worker : <> 26 | activate worker 27 | 28 | worker -> queue : get() 29 | 30 | engine -> wrapper ++ : emit(event) 31 | 32 | wrapper -> queue : put(event) 33 | return 34 | 35 | queue -> worker : return event 36 | 37 | worker -> consumer ++ : callback(event) 38 | ||| 39 | 40 | @enduml 41 | -------------------------------------------------------------------------------- /docs/diagrams/emit_events_summary.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title Processing events by SummaryLogger 3 | 4 | hide footbox 5 | skinparam BoxPadding 10 6 | 7 | box "yapapi 'core'" 8 | participant ":_Engine" as engine 9 | participant ":AsyncWrapper" as wrapper 10 | participant ":asyncio.Queue" as queue 11 | participant "worker" as worker <> 12 | end box 13 | 14 | 15 | box "yapapi 'metro area'" 16 | participant ":SummaryLogger" as consumer 17 | end box 18 | 19 | participant ":logging.Logger" as logger 20 | 21 | activate engine 22 | create wrapper 23 | engine -> wrapper : <> 24 | 25 | create queue 26 | wrapper -> queue : <> 27 | 28 | create worker 29 | wrapper -> worker : <> 30 | activate worker 31 | 32 | worker -> queue : get() 33 | 34 | engine -> wrapper ++ : emit(event) 35 | 36 | wrapper -> queue : put(event) 37 | return 38 | 39 | queue -> worker : return event 40 | 41 | worker -> consumer ++ : callback(event) 42 | consumer -> consumer ++ : handle() 43 | consumer -> logger : info() 44 | ||| 45 | 46 | @enduml 47 | -------------------------------------------------------------------------------- /docs/diagrams/execute_tasks.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title Golem.execute_tasks() 3 | 4 | ' Define colors for activation rectangles of 5 | !$task_1 = "#White" 6 | !$task_2 = "#White" 7 | !$task_3 = "#White" 8 | 9 | hide footbox 10 | skinparam BoxPadding 10 11 | 12 | box "main task" #ffffee 13 | participant "Client code" 14 | participant "golem:Golem" as engine #Turquoise 15 | participant "executor:Executor" as executor #Violet 16 | end box 17 | 18 | box "worker_starter() task" #ffeeff 19 | participant "executor:Executor" as executor2 #Violet 20 | participant "golem:Golem" as engine2 #Turquoise 21 | participant ":AgreementPool" as pool 22 | end box 23 | 24 | box "worker_task() task" #eeffff 25 | collections "golem:Golem" as engine3 #Turquoise 26 | collections "executor:Executor" as executor3 #Violet 27 | 28 | participant "batch_generator:AsyncGenerator" as batch_generator 29 | end box 30 | 31 | create engine 32 | "Client code" -> engine : <> 33 | "Client code" -> engine ++ $task_1 : execute_tasks(worker) 34 | 35 | create executor 36 | engine -> executor : <> 37 | engine -> executor ++ $task_1 : submit(worker) 38 | 39 | create pool 40 | executor -> pool : <> 41 | 42 | executor -> executor2 ++ $task_2 : <> 43 | note right: worker_starter() 44 | 45 | loop executed every 2 seconds if there is unassigned work, each iteration can create new worker_task 46 | executor2 -> engine2 ++ $task_2 : start_worker(run_worker) 47 | 48 | engine2 -> pool ++ $task_2: use_agreement(worker_task) 49 | deactivate engine2 50 | ?<- pool : agr = create_agreement() 51 | 52 | pool -> engine3 ++ $task_3: <> \n <> 53 | note right: worker_task(agr) 54 | deactivate pool 55 | end 56 | 57 | engine3 ->? : act = new_activity() 58 | engine3 ->? : ctx = WorkContext() 59 | 60 | engine3 -> executor3 ++ $task_3: <> 61 | note right: run_worker(act, ctx) 62 | 63 | create batch_generator 64 | executor3 -> batch_generator : <> 65 | note over batch_generator: worker(ctx) 66 | executor3 -> engine3++ : process_batches(batch_generator) 67 | 68 | ||| 69 | 70 | ref over engine3, executor3, batch_generator 71 | See the diagram for process_batches() 72 | end ref 73 | 74 | ||| 75 | 76 | return 77 | return 78 | deactivate engine3 79 | 80 | legend left 81 | * Three background boxes represent different asyncio tasks. 82 | * Participants with the same name represent the same object activated by different asyncio tasks 83 | * <> message represents loop.create_task() call 84 | endlegend 85 | 86 | @enduml 87 | -------------------------------------------------------------------------------- /docs/diagrams/process_batches.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title Creating and executing command batches 3 | hide footbox 4 | 5 | participant "executor:Executor" as executor #Violet 6 | participant ":Activity" as act 7 | participant "golem:[[../../yapapi/golem.py Golem]]" as engine #Turquoise 8 | participant "batch_generator:AsyncGenerator" as batch_generator 9 | participant ":WorkContext" as ctx 10 | 11 | executor -> engine ++ : process_batches(batch_generator) 12 | 13 | ' batch 1 14 | engine -> batch_generator : anext() 15 | activate batch_generator 16 | batch_generator -> ctx : send_file() 17 | activate ctx 18 | deactivate ctx 19 | batch_generator -> ctx : run() 20 | activate ctx 21 | deactivate ctx 22 | batch_generator -> ctx : commit() 23 | activate ctx 24 | return batch_1 25 | batch_generator --> engine : yield batch_1 26 | 27 | engine -> act : send(batch_1) 28 | activate act 29 | return batch_results_1 30 | 31 | ' batch 2 32 | engine -> batch_generator : asend(batch_results_1) 33 | batch_generator -> ctx : run() 34 | activate ctx 35 | deactivate ctx 36 | batch_generator -> ctx : download_file() 37 | activate ctx 38 | deactivate ctx 39 | batch_generator -> ctx: commit() 40 | activate ctx 41 | return batch_2 42 | batch_generator --> engine : yield batch_2 43 | 44 | engine -> act : send(batch_2) 45 | activate act 46 | return batch_results_2 47 | 48 | engine -> batch_generator : asend(batch_results_2) 49 | return StopIteration 50 | 51 | return 52 | 53 | deactivate executor 54 | deactivate executor 55 | 56 | deactivate engine 57 | 58 | @enduml 59 | -------------------------------------------------------------------------------- /docs/sphinx/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | from typing import List 16 | 17 | sys.path.insert(0, os.path.abspath("../../")) 18 | 19 | from yapapi import get_version 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = "yapapi" 24 | copyright = "2020-2021, Golem Factory" 25 | author = "Golem Factory" 26 | 27 | # The full version, including alpha/beta/rc tags 28 | release = get_version() 29 | 30 | 31 | # -- General configuration --------------------------------------------------- 32 | 33 | # Add any Sphinx extension module names here, as strings. They can be 34 | # extensions coming with Sphinx (named "sphinx.ext.*") or your custom 35 | # ones. 36 | extensions = [ 37 | "sphinx.ext.autodoc", 38 | "sphinx_autodoc_typehints", 39 | ] 40 | 41 | # Add any paths that contain templates here, relative to this directory. 42 | templates_path = ["_templates"] 43 | 44 | # List of patterns, relative to source directory, that match files and 45 | # directories to ignore when looking for source files. 46 | # This pattern also affects html_static_path and html_extra_path. 47 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 48 | 49 | 50 | # -- Options for HTML output ------------------------------------------------- 51 | 52 | # The theme to use for HTML and HTML Help pages. See the documentation for 53 | # a list of builtin themes. 54 | # 55 | html_theme = "sphinx_rtd_theme" 56 | 57 | # Add any paths that contain custom static files (such as style sheets) here, 58 | # relative to this directory. They are copied after the builtin static files, 59 | # so a file named "default.css" will overwrite the builtin "default.css". 60 | html_static_path: List[str] = [] 61 | 62 | 63 | # This removes the `yapapi.log` docstrings. 64 | # There are two reasons: 65 | # * there are some sections/subsections declared there and it messes up the docs 66 | # * it's obsolete, because encourages usage of Executor 67 | # Solution from https://stackoverflow.com/a/18031024/15851655 68 | def remove_module_docstring(app, what, name, obj, options, lines): 69 | if what == "module" and name == "yapapi.log": 70 | del lines[:] 71 | 72 | 73 | def setup(app): 74 | app.connect("autodoc-process-docstring", remove_module_docstring) 75 | 76 | 77 | autodoc_member_order = "bysource" 78 | -------------------------------------------------------------------------------- /docs/sphinx/index.rst: -------------------------------------------------------------------------------- 1 | ************************** 2 | Golem Python API Reference 3 | ************************** 4 | 5 | .. toctree:: 6 | 7 | api 8 | 9 | Golem Handbook 10 | Github 11 | Pypi 12 | -------------------------------------------------------------------------------- /examples/blender/.gitignore: -------------------------------------------------------------------------------- 1 | output_*.png 2 | -------------------------------------------------------------------------------- /examples/blender/blender.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golemfactory/blender:1.13 2 | COPY run-blender.sh /golem/entrypoints/ 3 | VOLUME /golem/work /golem/output /golem/resource 4 | -------------------------------------------------------------------------------- /examples/blender/cubes.blend: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/examples/blender/cubes.blend -------------------------------------------------------------------------------- /examples/blender/run-blender.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | cd /golem/work 4 | python3 /golem/entrypoints/render_entrypoint.py 5 | -------------------------------------------------------------------------------- /examples/custom-usage-counter/README.md: -------------------------------------------------------------------------------- 1 | # Custom Usage Counters Example 2 | 3 | The example includes `CustomCounterService` class; the main method of this class does the following things in the loop: 4 | 5 | - Sends a `sleep` command (`self._ctx.run("sleep", "1000")`) that is recognized by a custom runtime. Other runtimes may recognize different commands, e.g. a `process_file` command may process a file and increase the custom counter by the size of this file or the time it takes to process it. 6 | - Gets total cost from the provider. 7 | - Gets usage stats from the provider (default: `golem.usage.duration_sec`, `golem.usage.cpu_sec`; custom: `golem.usage.custom.counter`). 8 | 9 | Custom runtime used by this example is specified in the CustomCounterServicePayload class: 10 | ```py 11 | class CustomCounterServicePayload(Payload): 12 | runtime: str = constraint(inf.INF_RUNTIME_NAME, default="test-counters") 13 | ``` 14 | 15 | `test-counters` source code: https://github.com/golemfactory/ya-test-runtime-counters 16 | 17 | The main loop of this example runs the service on Golem and then prints status for each cluster instance every three seconds until all instances are no longer running. 18 | -------------------------------------------------------------------------------- /examples/custom_runtime/custom_runtime.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from dataclasses import dataclass 4 | 5 | from yapapi import Golem 6 | from yapapi.payload import Payload 7 | from yapapi.props import inf 8 | from yapapi.props.base import constraint, prop 9 | from yapapi.services import Service 10 | 11 | RUNTIME_NAME = "my-runtime" 12 | SOME_CUSTOM_PROPERTY = "golem.srv.app.eth.network" 13 | 14 | # The `CustomPayload` in this example is an arbitrary definition of some demand 15 | # that the requestor might wish to be fulfilled by the providers on the Golem network 16 | # 17 | # This payload must correspond with a runtime running on providers, either a runtime 18 | # written by some other party, or developed alongside its requestor counterpart using 19 | # the Runtime SDK (https://github.com/golemfactory/ya-runtime-sdk/) 20 | # 21 | # It is up to the author of said runtime to define any additional properties that would 22 | # describe the requestor's demand and `custom_property` is just an example. 23 | 24 | 25 | @dataclass 26 | class CustomPayload(Payload): 27 | custom_property: str = prop(SOME_CUSTOM_PROPERTY) 28 | 29 | runtime: str = constraint(inf.INF_RUNTIME_NAME, default=RUNTIME_NAME) 30 | min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) 31 | min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) 32 | 33 | 34 | class CustomRuntimeService(Service): 35 | @staticmethod 36 | async def get_payload(): 37 | return CustomPayload(custom_property="whatever") 38 | 39 | 40 | async def main(subnet_tag, driver=None, network=None): 41 | async with Golem( 42 | budget=10.0, 43 | subnet_tag=subnet_tag, 44 | payment_driver=driver, 45 | payment_network=network, 46 | ) as golem: 47 | cluster = await golem.run_service( 48 | CustomRuntimeService, 49 | ) 50 | 51 | def instances(): 52 | return [f"{s.provider_name}: {s.state.value}" for s in cluster.instances] 53 | 54 | cnt = 0 55 | while cnt < 10: 56 | print(f"instances: {instances()}") 57 | await asyncio.sleep(3) 58 | 59 | cluster.stop() 60 | 61 | cnt = 0 62 | while cnt < 10 and any(s.is_available for s in cluster.instances): 63 | print(f"instances: {instances()}") 64 | await asyncio.sleep(1) 65 | 66 | print(f"instances: {instances()}") 67 | 68 | 69 | asyncio.run(main(None)) 70 | -------------------------------------------------------------------------------- /examples/external-api-request/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | RUN apk update \ 4 | && apk add curl \ 5 | && apk add jq \ 6 | && rm -rf /var/cache/apk/* 7 | 8 | COPY request.sh /golem/entrypoints/ 9 | VOLUME /golem/work /golem/output /golem/resource 10 | -------------------------------------------------------------------------------- /examples/external-api-request/external_api_request.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | import pathlib 4 | import sys 5 | from datetime import datetime 6 | 7 | from utils import build_parser, print_env_info, run_golem_example 8 | 9 | from yapapi import Golem 10 | from yapapi.payload import vm 11 | from yapapi.services import Service 12 | 13 | examples_dir = pathlib.Path(__file__).resolve().parent.parent 14 | sys.path.append(str(examples_dir)) 15 | 16 | 17 | class ApiCallService(Service): 18 | @staticmethod 19 | async def get_payload(): 20 | manifest = open("manifest_whitelist.json", "rb").read() 21 | manifest_sig = open("manifest.json.base64.sha256.sig", "rb").read() 22 | manifest_sig_algorithm = "sha256" 23 | # DER, PEM and PEM chain formats are supported 24 | manifest_cert = open("golem_sign.pem", "rb").read() 25 | 26 | return await vm.manifest( 27 | manifest=manifest, 28 | manifest_sig=manifest_sig, 29 | manifest_sig_algorithm=manifest_sig_algorithm, 30 | manifest_cert=manifest_cert, 31 | min_mem_gib=0.5, 32 | min_cpu_threads=0.5, 33 | capabilities=["inet", "manifest-support"], 34 | ) 35 | 36 | async def run(self): 37 | script = self._ctx.new_script() 38 | future_result = script.run( 39 | "/bin/sh", 40 | "-c", 41 | "GOLEM_PRICE=`curl -X 'GET' \ 42 | 'https://api.coingecko.com/api/v3/simple/price?ids=golem&vs_currencies=usd' \ 43 | -H 'accept: application/json' | jq .golem.usd`; \ 44 | echo ---; \ 45 | echo \"Golem price: $GOLEM_PRICE USD\"; \ 46 | echo ---;", 47 | ) 48 | yield script 49 | 50 | result = (await future_result).stdout 51 | print(result.strip() if result else "") 52 | 53 | 54 | async def main(subnet_tag, payment_driver, payment_network): 55 | async with Golem( 56 | budget=1.0, 57 | subnet_tag=subnet_tag, 58 | payment_driver=payment_driver, 59 | payment_network=payment_network, 60 | ) as golem: 61 | print_env_info(golem) 62 | 63 | cluster = await golem.run_service(ApiCallService, num_instances=1) 64 | 65 | while True: 66 | print(cluster.instances) 67 | try: 68 | await asyncio.sleep(10) 69 | except (KeyboardInterrupt, asyncio.CancelledError): 70 | break 71 | 72 | 73 | if __name__ == "__main__": 74 | parser = build_parser("External API request example") 75 | now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") 76 | parser.set_defaults(log_file=f"external-api-request-yapapi-{now}.log") 77 | args = parser.parse_args() 78 | 79 | run_golem_example( 80 | main( 81 | subnet_tag=args.subnet_tag, 82 | payment_driver=args.payment_driver, 83 | payment_network=args.payment_network, 84 | ), 85 | log_file=args.log_file, 86 | ) 87 | -------------------------------------------------------------------------------- /examples/external-api-request/external_api_request_partner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | import json 4 | import pathlib 5 | import sys 6 | from datetime import datetime 7 | 8 | from utils import build_parser, print_env_info, run_golem_example 9 | 10 | from yapapi import Golem 11 | from yapapi.payload import vm 12 | from yapapi.services import Service 13 | 14 | examples_dir = pathlib.Path(__file__).resolve().parent.parent 15 | sys.path.append(str(examples_dir)) 16 | 17 | 18 | class ApiCallService(Service): 19 | @staticmethod 20 | async def get_payload(): 21 | # Replace with manifest_whitelist.json for whitelist mode 22 | manifest = open("manifest_partner_unrestricted.json", "rb").read() 23 | node_descriptor = json.loads(open("node-descriptor.signed.json", "r").read()) 24 | 25 | return await vm.manifest( 26 | manifest=manifest, 27 | node_descriptor=node_descriptor, 28 | min_mem_gib=0.5, 29 | min_cpu_threads=0.5, 30 | capabilities=[ 31 | "inet", 32 | ], 33 | ) 34 | 35 | async def run(self): 36 | script = self._ctx.new_script() 37 | future_result = script.run( 38 | "/bin/sh", 39 | "-c", 40 | "GOLEM_PRICE=`curl -X 'GET' \ 41 | 'https://api.coingecko.com/api/v3/simple/price?ids=golem&vs_currencies=usd' \ 42 | -H 'accept: application/json' | jq .golem.usd`; \ 43 | echo ---; \ 44 | echo \"Golem price: $GOLEM_PRICE USD\"; \ 45 | echo ---;", 46 | ) 47 | yield script 48 | 49 | result = (await future_result).stdout 50 | print(result.strip() if result else "") 51 | 52 | 53 | async def main(subnet_tag, payment_driver, payment_network): 54 | async with Golem( 55 | budget=1.0, 56 | subnet_tag=subnet_tag, 57 | payment_driver=payment_driver, 58 | payment_network=payment_network, 59 | ) as golem: 60 | print_env_info(golem) 61 | 62 | cluster = await golem.run_service(ApiCallService, num_instances=1) 63 | 64 | while True: 65 | print(cluster.instances) 66 | try: 67 | await asyncio.sleep(10) 68 | except (KeyboardInterrupt, asyncio.CancelledError): 69 | break 70 | 71 | 72 | if __name__ == "__main__": 73 | parser = build_parser("External API request example") 74 | now = datetime.now().strftime("%Y-%m-%d_%H.%M.%S") 75 | parser.set_defaults(log_file=f"external-api-request-yapapi-{now}.log") 76 | args = parser.parse_args() 77 | 78 | run_golem_example( 79 | main( 80 | subnet_tag=args.subnet_tag, 81 | payment_driver=args.payment_driver, 82 | payment_network=args.payment_network, 83 | ), 84 | log_file=args.log_file, 85 | ) 86 | -------------------------------------------------------------------------------- /examples/external-api-request/golem_sign.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIB/zCCAYWgAwIBAgIUL+slWOaCypC0+VZM96VMW3lN3RkwCgYIKoZIzj0EAwIw 3 | ODEWMBQGA1UECgwNR29sZW0gRmFjdG9yeTEeMBwGA1UEAwwVR29sZW0gRmFjdG9y 4 | eSBST09UIENBMB4XDTIyMTEyMzExNDA1M1oXDTI1MTEyMjExNDA1M1owQDEWMBQG 5 | A1UECgwNR29sZW0gRmFjdG9yeTEmMCQGA1UEAwwdR29sZW0gRmFjdG9yeSBJTlRF 6 | Uk1FRElBVEUgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASSFvpQd9Q0hE33 7 | VcdWXx57BNYfxL82IechAhcOFZMfuxlUIh3scIrvVu7IKK/5WZEeMw3SlU7zQrHf 8 | xDQhW7FZo2UwYzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBATA9 9 | BgNVHR8BAf8EMzAxMC+gLaArhilodHRwczovL2NhLmdvbGVtLm5ldHdvcmsvY3Js 10 | L0dPTEVNLUNBLmNybDAKBggqhkjOPQQDAgNoADBlAjAMz6iyHqudwY0WwfOGSEWu 11 | QtJ+/x0EztMzHVKIc1sFrNFhKTm1rNozlX1k5dGaMZ0CMQDrCcCBs9XiVCjRIjsG 12 | UqSjwyXJprSAQn4g3kFNOLCHrzOo/5IS9MZUH0lnekbaa1o= 13 | -----END CERTIFICATE----- 14 | -------------------------------------------------------------------------------- /examples/external-api-request/manifest.json.base64.sha256.sig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/examples/external-api-request/manifest.json.base64.sha256.sig -------------------------------------------------------------------------------- /examples/external-api-request/manifest_partner_unrestricted.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.0", 3 | "createdAt": "2022-07-26T12:51:00.000000Z", 4 | "expiresAt": "2100-01-01T00:01:00.000000Z", 5 | "metadata": { 6 | "name": "External API call example", 7 | "description": "Example manifest of a service making an outbound call to the external API", 8 | "version": "0.1.0" 9 | }, 10 | "payload": [ 11 | { 12 | "platform": { 13 | "arch": "x86_64", 14 | "os": "linux" 15 | }, 16 | "urls": ["http://yacn2.dev.golem.network:8000/docker-golem-script-curl-latest-d75268e752.gvmi"], 17 | "hash": "sha3:e5f5ddfd649525dbe25d93d9ed51d1bdd0849933d9a5720adb4b5810" 18 | } 19 | ], 20 | "compManifest": { 21 | "version": "0.1.0", 22 | "net": { 23 | "inet": { 24 | "out": { 25 | "protocols": ["https"], 26 | "unrestricted": { 27 | "urls": true 28 | } 29 | } 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/external-api-request/manifest_whitelist.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.0", 3 | "createdAt": "2022-07-26T12:51:00.000000Z", 4 | "expiresAt": "2100-01-01T00:01:00.000000Z", 5 | "metadata": { 6 | "name": "External API call example", 7 | "description": "Example manifest of a service making an outbound call to the external API", 8 | "version": "0.1.0" 9 | }, 10 | "payload": [ 11 | { 12 | "platform": { 13 | "arch": "x86_64", 14 | "os": "linux" 15 | }, 16 | "urls": [ 17 | "http://yacn2.dev.golem.network:8000/docker-golem-script-curl-latest-d75268e752.gvmi" 18 | ], 19 | "hash": "sha3:e5f5ddfd649525dbe25d93d9ed51d1bdd0849933d9a5720adb4b5810" 20 | } 21 | ], 22 | "compManifest": { 23 | "version": "0.1.0", 24 | "script": { 25 | "commands": [ 26 | "run .*curl.*", 27 | "run .*request.sh.*", 28 | "transfer .*output.txt" 29 | ], 30 | "match": "regex" 31 | }, 32 | "net": { 33 | "inet": { 34 | "out": { 35 | "protocols": ["https"], 36 | "urls": ["https://api.coingecko.com", "https://httpbin.org"] 37 | } 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /examples/external-api-request/partner_rule_poc.md: -------------------------------------------------------------------------------- 1 | # HOW TO RUN PARTNER RULE POC 2 | 3 | ## Obtain necessary certificates & node_descriptor 4 | 5 | To match Partner Rule in provider, some signed files are needed. 6 | 7 | Example ones are put here: [https://drive.google.com/drive/folders/1LpCrnsatthcD1w4BPVOYcV2IxZ1ty9_V](https://drive.google.com/drive/folders/1LpCrnsatthcD1w4BPVOYcV2IxZ1ty9_V) 8 | 9 | Download following ones: 10 | 11 | - `root-certificate.signed.json` 12 | - `partner-certificate.signed.json` 13 | - `partner-keypair.key` 14 | - `node-descriptor.json` 15 | 16 | Then edit `nodeId` of `node-descriptor.json` accordingly, and sign it with partner certificate with [golem-certificate-cli](https://github.com/golemfactory/golem-certificate) like so: 17 | 18 | ```bash 19 | cargo run -p golem-certificate-cli sign node-descriptor.json partner-certificate.signed.json partner-keypair.key 20 | ``` 21 | 22 | Then, following files will be needed in next steps: 23 | 24 | - `root-certificate.signed.json` 25 | - `node-descriptor.signed.json` 26 | 27 | ## Set-up provider 28 | 29 | Set up provider as always but with follwing rule set command: 30 | 31 | ```bash 32 | cargo run -p ya-provider -- rule set outbound partner import-cert root-certificate.signed.json --mode all 33 | ``` 34 | 35 | ## Run task 36 | 37 | Copy `node-descriptor.signed.json` to directory with `external_api_request.py` and run task: 38 | 39 | ```bash 40 | poetry run python3 external_api_request.py 41 | ``` 42 | 43 | > **Note** 44 | > Remember to set YAGNA_APPKEY and other env variables accordingly 45 | -------------------------------------------------------------------------------- /examples/external-api-request/request.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | out=$(curl -X 'GET' 'https://httpbin.org/get') && echo "$out" > /golem/output/output.txt 4 | -------------------------------------------------------------------------------- /examples/external-api-request/sign.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 1 ]; then 4 | echo 1>&2 "$0: Add path to private key as a param" 5 | exit 2 6 | fi 7 | 8 | PRIVATE_KEY=$1 9 | base64 manifest.json --wrap=0 > manifest.json.base64 10 | openssl dgst -sha256 -sign $PRIVATE_KEY -out manifest.json.base64.sha256.sig manifest.json.base64 11 | -------------------------------------------------------------------------------- /examples/hello-world/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | VOLUME /golem/input 3 | WORKDIR /golem/work 4 | -------------------------------------------------------------------------------- /examples/hello-world/hello.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | from typing import AsyncIterable 4 | 5 | from yapapi import Golem, Task, WorkContext 6 | from yapapi.log import enable_default_logger 7 | from yapapi.payload import vm 8 | 9 | 10 | async def worker(context: WorkContext, tasks: AsyncIterable[Task]): 11 | async for task in tasks: 12 | script = context.new_script() 13 | future_result = script.run("/bin/sh", "-c", "date") 14 | 15 | yield script 16 | 17 | task.accept_result(result=await future_result) 18 | 19 | 20 | async def main(): 21 | package = await vm.repo( 22 | image_hash="d646d7b93083d817846c2ae5c62c72ca0507782385a2e29291a3d376", 23 | ) 24 | 25 | tasks = [Task(data=None)] 26 | 27 | async with Golem(budget=1.0, subnet_tag="public") as golem: 28 | async for completed in golem.execute_tasks(worker, tasks, payload=package): 29 | print(completed.result.stdout) 30 | 31 | 32 | if __name__ == "__main__": 33 | enable_default_logger(log_file="hello.log") 34 | 35 | loop = asyncio.get_event_loop() 36 | task = loop.create_task(main()) 37 | loop.run_until_complete(task) 38 | -------------------------------------------------------------------------------- /examples/hello-world/hello_service.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | from datetime import datetime, timedelta 4 | 5 | from yapapi import Golem 6 | from yapapi.log import enable_default_logger 7 | from yapapi.payload import vm 8 | from yapapi.services import Service 9 | 10 | DATE_OUTPUT_PATH = "/golem/work/date.txt" 11 | REFRESH_INTERVAL_SEC = 5 12 | 13 | 14 | class DateService(Service): 15 | @staticmethod 16 | async def get_payload(): 17 | return await vm.repo( 18 | image_hash="d646d7b93083d817846c2ae5c62c72ca0507782385a2e29291a3d376", 19 | ) 20 | 21 | async def start(self): 22 | async for script in super().start(): 23 | yield script 24 | 25 | # every `DATE_POLL_INTERVAL` write output of `date` to `DATE_OUTPUT_PATH` 26 | script = self._ctx.new_script() 27 | script.run( 28 | "/bin/sh", 29 | "-c", 30 | f"while true; do date > {DATE_OUTPUT_PATH}; sleep {REFRESH_INTERVAL_SEC}; done &", 31 | ) 32 | yield script 33 | 34 | async def run(self): 35 | while True: 36 | await asyncio.sleep(REFRESH_INTERVAL_SEC) 37 | script = self._ctx.new_script() 38 | future_result = script.run( 39 | "/bin/sh", 40 | "-c", 41 | f"cat {DATE_OUTPUT_PATH}", 42 | ) 43 | 44 | yield script 45 | 46 | result = (await future_result).stdout 47 | print(result.strip() if result else "") 48 | 49 | 50 | async def main(): 51 | async with Golem(budget=1.0, subnet_tag="public") as golem: 52 | cluster = await golem.run_service(DateService, num_instances=1) 53 | start_time = datetime.now() 54 | 55 | while datetime.now() < start_time + timedelta(minutes=1): 56 | for num, instance in enumerate(cluster.instances): 57 | print(f"Instance {num} is {instance.state.value} on {instance.provider_name}") 58 | await asyncio.sleep(REFRESH_INTERVAL_SEC) 59 | 60 | 61 | if __name__ == "__main__": 62 | enable_default_logger(log_file="hello.log") 63 | 64 | loop = asyncio.get_event_loop() 65 | task = loop.create_task(main()) 66 | loop.run_until_complete(task) 67 | -------------------------------------------------------------------------------- /examples/http-proxy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:stable-alpine 2 | -------------------------------------------------------------------------------- /examples/low-level-api/list-offers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | import json 4 | import pathlib 5 | import sys 6 | from asyncio import TimeoutError 7 | from datetime import datetime, timezone 8 | 9 | from yapapi import props as yp 10 | from yapapi.config import ApiConfig 11 | from yapapi.log import enable_default_logger 12 | from yapapi.props.builder import DemandBuilder 13 | from yapapi.rest import Activity, Configuration, Market, Payment # noqa 14 | 15 | examples_dir = pathlib.Path(__file__).resolve().parent.parent 16 | sys.path.append(str(examples_dir)) 17 | import utils 18 | 19 | 20 | async def list_offers(conf: Configuration, subnet_tag: str): 21 | async with conf.market() as client: 22 | market_api = Market(client) 23 | dbuild = DemandBuilder() 24 | dbuild.add(yp.NodeInfo(name="some scanning node", subnet_tag=subnet_tag)) 25 | dbuild.add(yp.Activity(expiration=datetime.now(timezone.utc))) 26 | 27 | async with market_api.subscribe(dbuild.properties, dbuild.constraints) as subscription: 28 | async for event in subscription.events(): 29 | print(f"Offer: {event.id}") 30 | print(f"from {event.issuer}") 31 | print(f"props {json.dumps(event.props, indent=4)}") 32 | print("\n\n") 33 | print("done") 34 | 35 | 36 | def main(): 37 | parser = utils.build_parser("List offers") 38 | args = parser.parse_args() 39 | 40 | subnet = args.subnet_tag 41 | sys.stderr.write(f"Using subnet: {utils.TEXT_COLOR_YELLOW}{subnet}{utils.TEXT_COLOR_DEFAULT}\n") 42 | 43 | enable_default_logger() 44 | try: 45 | asyncio.get_event_loop().run_until_complete( 46 | asyncio.wait_for( 47 | list_offers( 48 | Configuration(api_config=ApiConfig()), # YAGNA_APPKEY will be loaded from env 49 | subnet_tag=subnet, 50 | ), 51 | timeout=4, 52 | ) 53 | ) 54 | except TimeoutError: 55 | pass 56 | 57 | 58 | if __name__ == "__main__": 59 | main() 60 | -------------------------------------------------------------------------------- /examples/simple-service-poc/simple_service/README.md: -------------------------------------------------------------------------------- 1 | This directory contains files used to construct the application Docker image 2 | that's then converted to a GVMI file (a Golem Virtual Machine Image file) and uploaded 3 | to the Yagna image repository. 4 | 5 | All Python scripts here are run within a VM on the Provider's end. 6 | 7 | The example (`../simple_service.py`) already contains the appropriate image hash 8 | but if you'd like to experiment with it, feel free to re-build it. 9 | 10 | ## Building the image 11 | 12 | You'll need: 13 | 14 | * Docker: https://www.docker.com/products/docker-desktop 15 | * gvmkit-build: `pip install gvmkit-build` 16 | 17 | Once you have those installed, run the following from this directory: 18 | 19 | ```bash 20 | docker build -f simple_service.Dockerfile -t simple-service . 21 | gvmkit-build simple-service:latest 22 | gvmkit-build simple-service:latest --push 23 | ``` 24 | 25 | Note the hash link that's presented after the upload finishes. 26 | 27 | e.g. `b742b6cb04123d07bacb36a2462f8b2347b20c32223c1ac49664635f` 28 | 29 | and update the service's `get_payload` method to point to this image: 30 | 31 | ```python 32 | async def get_payload(): 33 | return await vm.repo( 34 | image_hash="b742b6cb04123d07bacb36a2462f8b2347b20c32223c1ac49664635f", 35 | min_mem_gib=0.5, 36 | min_storage_gib=2.0, 37 | ) 38 | ``` 39 | -------------------------------------------------------------------------------- /examples/simple-service-poc/simple_service/simple_service.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-slim 2 | VOLUME /golem/in /golem/out 3 | COPY simple_service.py /golem/run/simple_service.py 4 | COPY simulate_observations.py /golem/run/simulate_observations.py 5 | COPY simulate_observations_ctl.py /golem/run/simulate_observations_ctl.py 6 | RUN pip install numpy matplotlib 7 | RUN chmod +x /golem/run/* 8 | RUN /golem/run/simple_service.py --init 9 | ENTRYPOINT ["sh"] 10 | -------------------------------------------------------------------------------- /examples/simple-service-poc/simple_service/simulate_observations.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | """The "hello world" service here just adds randomized numbers with normal distribution. 3 | 4 | in a real-world example, this could be e.g. a thermometer connected to the provider's 5 | machine providing its inputs into the database or some other piece of information 6 | from some external source that changes over time and which can be expressed as a 7 | singular value 8 | 9 | [ part of the VM image that's deployed by the runtime on the Provider's end. ] 10 | """ 11 | import os 12 | import random 13 | import time 14 | from pathlib import Path 15 | 16 | MU = 14 17 | SIGMA = 3 18 | 19 | SERVICE_PATH = Path(__file__).absolute().parent / "simple_service.py" 20 | 21 | 22 | while True: 23 | v = random.normalvariate(MU, SIGMA) 24 | os.system(f"{SERVICE_PATH} --add {v}") 25 | time.sleep(1) 26 | -------------------------------------------------------------------------------- /examples/simple-service-poc/simple_service/simulate_observations_ctl.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | """a helper, control script that starts and stops our example `simulate_observations` service. 3 | 4 | [ part of the VM image that's deployed by the runtime on the Provider's end. ] 5 | """ 6 | import argparse 7 | import os 8 | import signal 9 | import subprocess 10 | 11 | PIDFILE = "/var/run/simulate_observations.pid" 12 | SCRIPT_FILE = "/golem/run/simulate_observations.py" 13 | 14 | parser = argparse.ArgumentParser("start/stop simulation") 15 | group = parser.add_mutually_exclusive_group(required=True) 16 | group.add_argument("--start", action="store_true") 17 | group.add_argument("--stop", action="store_true") 18 | 19 | args = parser.parse_args() 20 | 21 | if args.start: 22 | if os.path.exists(PIDFILE): 23 | raise Exception(f"Cannot start process, {PIDFILE} exists.") 24 | p = subprocess.Popen([SCRIPT_FILE]) 25 | with open(PIDFILE, "w") as pidfile: 26 | pidfile.write(str(p.pid)) 27 | elif args.stop: 28 | if not os.path.exists(PIDFILE): 29 | raise Exception(f"Could not find pidfile: {PIDFILE}.") 30 | with open(PIDFILE, "r") as pidfile: 31 | pid = int(pidfile.read()) 32 | 33 | os.kill(pid, signal.SIGKILL) 34 | os.remove(PIDFILE) 35 | -------------------------------------------------------------------------------- /examples/ssh/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | RUN apk add --no-cache --update bash openssh iproute2 tcpdump net-tools screen 4 | RUN echo "UseDNS no" >> /etc/ssh/sshd_config && \ 5 | echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \ 6 | echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config 7 | -------------------------------------------------------------------------------- /examples/webapp/README.md: -------------------------------------------------------------------------------- 1 | # A minimal web app example for yapapi 2 | 3 | This example demonstrates using yapapi's Services API to run a web application composed of two services deployed to separate provider hosts and each one using its own vm image. 4 | 5 | ## Components 6 | 7 | ### DB backend: rqlite 8 | 9 | For the backend, we decided to use **[rqlite](https://github.com/rqlite/rqlite)** - an "easy-to-use, lightweight, distributed relational database, which uses SQLite as its storage engine." 10 | 11 | One reason to use `rqlite` is that it's very easy to set-up, and another - more important one - is that its clustering capabilities make it extremely useful for distributed deployments on Golem. 12 | 13 | ### HTTP frontend: a "oneliner" status app 14 | 15 | For the frontend, we're using a trivial `Flask` app, connected to the DB with `SQLAlchemy`. 16 | 17 | The app uses a single HTML template and allows the user to send a single line of text that gets appended to an existing log of messages and at the same time, presents the list of previous messages to the user. 18 | 19 | ### Local HTTP proxy 20 | 21 | Finally, to allow the user to connect to the HTTP front-end running on a provider's end, we're launching a simple HTTP proxy service that listens on a local HTTP port, forwards all requests to the provider's HTTP instance and all responses back to the local port. 22 | -------------------------------------------------------------------------------- /examples/webapp/db/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | ENV RQLITE_VERSION=7.3.1 4 | 5 | RUN apk update && \ 6 | apk --no-cache add curl tar bash && \ 7 | curl -L https://github.com/rqlite/rqlite/releases/download/v${RQLITE_VERSION}/rqlite-v${RQLITE_VERSION}-linux-amd64-musl.tar.gz -o rqlite-v${RQLITE_VERSION}-linux-amd64-musl.tar.gz && \ 8 | tar xvfz rqlite-v${RQLITE_VERSION}-linux-amd64-musl.tar.gz && \ 9 | cp rqlite-v${RQLITE_VERSION}-linux-amd64-musl/rqlited /bin && \ 10 | cp rqlite-v${RQLITE_VERSION}-linux-amd64-musl/rqlite /bin && \ 11 | rm -fr rqlite-v${RQLITE_VERSION}-linux-amd64-musl rqlite-v${RQLITE_VERSION}-linux-amd64-musl.tar.gz && \ 12 | apk del curl tar 13 | 14 | RUN mkdir -p /rqlite/file 15 | 16 | EXPOSE 4001 4002 17 | COPY run_rqlite.sh /bin/run_rqlite.sh 18 | RUN chmod a+x /bin/run_rqlite.sh 19 | -------------------------------------------------------------------------------- /examples/webapp/db/run_rqlite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | /bin/rqlited -http-addr 0.0.0.0:4001 /rqlite/file/data > /run/out 2> /run/err & 3 | -------------------------------------------------------------------------------- /examples/webapp/http/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim 2 | RUN pip install Flask flask-sqlalchemy sqlalchemy_rqlite 3 | 4 | RUN mkdir -p /webapp/templates 5 | 6 | COPY app.py /webapp/app.py 7 | COPY templates/index.html /webapp/templates/index.html 8 | -------------------------------------------------------------------------------- /examples/webapp/http/app.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from flask import Flask, redirect, render_template, request, url_for 4 | from flask_sqlalchemy import SQLAlchemy 5 | 6 | parser = argparse.ArgumentParser("simple flask app") 7 | parser.add_argument("--db-address", help="the address of the rqlite database", default="localhost") 8 | parser.add_argument("--db-port", help="the of the rqlite database", default="4001") 9 | 10 | subparsers = parser.add_subparsers(dest="cmd", required=True) 11 | 12 | subparsers.add_parser("initdb", help="initialize the database") 13 | run_parser = subparsers.add_parser("run", help="run the app") 14 | args = parser.parse_args() 15 | 16 | app = Flask(__name__) 17 | app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False 18 | app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"echo": True} 19 | app.config["SQLALCHEMY_DATABASE_URI"] = f"rqlite+pyrqlite://{args.db_address}:{args.db_port}/" 20 | 21 | db = SQLAlchemy(app) 22 | 23 | 24 | class Line(db.Model): 25 | id = db.Column(db.Integer, primary_key=True) 26 | message = db.Column(db.String(255)) 27 | 28 | 29 | @app.route("/", methods=["get"]) 30 | def root_get(): 31 | return render_template("index.html", messages=Line.query.order_by(Line.id.desc()).limit(16)) 32 | 33 | 34 | @app.route("/", methods=["post"]) 35 | def root_post(): 36 | db.session.add(Line(message=request.form["message"])) 37 | db.session.commit() 38 | return redirect(url_for("root_get")) 39 | 40 | 41 | if args.cmd == "initdb": 42 | db.create_all() 43 | elif args.cmd == "run": 44 | app.run(host="0.0.0.0") 45 | -------------------------------------------------------------------------------- /examples/webapp/http/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Onliner 6 | 7 | 8 |
9 | 10 | 11 |
12 |
13 | {%- for msg in messages %} 14 |

15 | {{ msg.message }} 16 |

17 | {%- endfor %} 18 | 19 | -------------------------------------------------------------------------------- /examples/webapp_fileupload/http-file-upload/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim 2 | RUN pip install Flask flask-sqlalchemy sqlalchemy_rqlite 3 | 4 | RUN mkdir -p /webapp/templates 5 | RUN mkdir -p /webapp/uploads 6 | 7 | COPY app.py /webapp/app.py 8 | COPY templates/index.html /webapp/templates/index.html 9 | COPY templates/error.html /webapp/templates/error.html 10 | 11 | VOLUME /logs 12 | -------------------------------------------------------------------------------- /examples/webapp_fileupload/http-file-upload/app.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pathlib import Path 3 | 4 | from flask import Flask, Request, redirect, render_template, request, send_from_directory, url_for 5 | from flask_sqlalchemy import SQLAlchemy 6 | from werkzeug.datastructures import FileStorage 7 | from werkzeug.utils import secure_filename 8 | 9 | parser = argparse.ArgumentParser("simple flask app") 10 | parser.add_argument("--db-address", help="the address of the rqlite database", default="localhost") 11 | parser.add_argument("--db-port", help="the of the rqlite database", default="4001") 12 | 13 | subparsers = parser.add_subparsers(dest="cmd", required=True) 14 | 15 | subparsers.add_parser("initdb", help="initialize the database") 16 | run_parser = subparsers.add_parser("run", help="run the app") 17 | args = parser.parse_args() 18 | 19 | app = Flask(__name__) 20 | app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False 21 | app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"echo": True} 22 | app.config["SQLALCHEMY_DATABASE_URI"] = f"rqlite+pyrqlite://{args.db_address}:{args.db_port}/" 23 | app.config["UPLOAD_FOLDER"] = "uploads" 24 | 25 | 26 | db = SQLAlchemy(app) 27 | 28 | request: Request 29 | 30 | 31 | class Img(db.Model): 32 | id = db.Column(db.Integer, primary_key=True) 33 | filename = db.Column(db.String(255)) 34 | 35 | 36 | @app.route("/", methods=["get"]) 37 | def root_get(): 38 | return render_template( 39 | "index.html", 40 | images_url=url_for("download_image", filename=""), 41 | images=Img.query.order_by(Img.id.desc()).limit(64), 42 | ) 43 | 44 | 45 | @app.route("/images/") 46 | def download_image(filename): 47 | return send_from_directory(app.config["UPLOAD_FOLDER"], filename) 48 | 49 | 50 | @app.route("/", methods=["post"]) 51 | def root_post(): 52 | file: FileStorage = request.files.get("file", None) 53 | if not file: 54 | return render_template("error.html", back_url=url_for("root_get")) 55 | 56 | filename = secure_filename(file.filename) 57 | filepath = Path(app.config["UPLOAD_FOLDER"]) / filename 58 | file.save(filepath) 59 | 60 | db.session.add(Img(filename=filename)) 61 | db.session.commit() 62 | return redirect(url_for("root_get")) 63 | 64 | 65 | if args.cmd == "initdb": 66 | db.create_all() 67 | elif args.cmd == "run": 68 | app.run(host="0.0.0.0") 69 | -------------------------------------------------------------------------------- /examples/webapp_fileupload/http-file-upload/templates/error.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Error 6 | 7 | 8 | Something went wrong... perhaps no file selected? 9 | Back 10 | 11 | 12 | -------------------------------------------------------------------------------- /examples/webapp_fileupload/http-file-upload/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Image chat 6 | 7 | 8 |
9 | 10 | 11 |
12 |
13 | {%- for img in images %} 14 |

15 | 16 | {{img.filename}} 17 | 18 |

19 | {%- endfor %} 20 | 21 | -------------------------------------------------------------------------------- /examples/yacat/.gitignore: -------------------------------------------------------------------------------- 1 | hashcat_*.potfile 2 | in.hash 3 | keyspace.* 4 | -------------------------------------------------------------------------------- /examples/yacat/README.md: -------------------------------------------------------------------------------- 1 | 2 | to run the example use the following command: 3 | 4 | ``` 5 | python yacat.py --mask '?a?a?a' --hash '$P$5ZDzPE45CLLhEx/72qt3NehVzwN2Ry/' 6 | ``` 7 | 8 | you can also try with a heavier password: 9 | 10 | ``` 11 | python yacat.py --mask '?a?a?a?a' --hash '$H$5ZDzPE45C.e3TjJ2Qi58Aaozha6cs30' --max-workers 4 12 | ``` 13 | 14 | or a lighter one: 15 | 16 | ``` 17 | python3 yacat.py --mask '?a?a' --hash '$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0' 18 | ``` 19 | 20 | -------------------------------------------------------------------------------- /examples/yacat/yacat.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM dizcza/docker-hashcat:intel-cpu 2 | 3 | VOLUME /golem/input /golem/output 4 | WORKDIR /golem/entrypoint 5 | -------------------------------------------------------------------------------- /pydoc-markdown.yml: -------------------------------------------------------------------------------- 1 | loaders: 2 | - type: python 3 | processors: 4 | - type: filter 5 | - type: smart 6 | - type: crossref 7 | renderer: 8 | type: mkdocs 9 | pages: 10 | - title: API Reference 11 | contents: 12 | - '*' 13 | mkdocs_config: 14 | site_name: yapapi 15 | theme: readthedocs 16 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | 6 | def pytest_addoption(parser): 7 | parser.addoption("--ya-api-key", type=str, help="instance api key", dest="yaApiKey") 8 | 9 | 10 | @pytest.fixture 11 | def dummy_yagna_engine(monkeypatch): 12 | """Use this fixture to call `_Engine.start()` in unit tests, without yagna/gftp. 13 | 14 | So also e.g. `async with Golem(..., APP_KEY='FAKE_APP_KEY')`, or Golem.start(). 15 | 16 | But first check if monkeypatches done here don't interefere with 17 | the thing you want to test ofc. 18 | """ 19 | from yapapi.engine import _Engine 20 | from yapapi.storage.gftp import GftpProvider 21 | 22 | async def _engine_create_allocations(self): 23 | pass 24 | 25 | async def _gftp_aenter(self): 26 | return self 27 | 28 | async def _gftp_aexit(self, *args): 29 | return None 30 | 31 | monkeypatch.setattr(_Engine, "_create_allocations", _engine_create_allocations) 32 | monkeypatch.setattr(GftpProvider, "__aenter__", _gftp_aenter) 33 | monkeypatch.setattr(GftpProvider, "__aexit__", _gftp_aexit) 34 | 35 | 36 | @pytest.fixture 37 | def purge_yagna_os_env() -> None: 38 | for key in [ 39 | "YAGNA_APPKEY", 40 | "YAGNA_API_URL", 41 | "YAGNA_MARKET_URL", 42 | "YAGNA_PAYMENT_URL", 43 | "YAGNA_NET_URL", 44 | "YAGNA_ACTIVITY_URL", 45 | "YAGNA_SUBNET", 46 | "YAGNA_PAYMENT_DRIVER", 47 | "YAGNA_PAYMENT_NETWORK", 48 | ]: 49 | os.environ.pop(key, None) 50 | -------------------------------------------------------------------------------- /tests/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/contrib/__init__.py -------------------------------------------------------------------------------- /tests/contrib/service/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/contrib/service/__init__.py -------------------------------------------------------------------------------- /tests/contrib/service/test_chunk.py: -------------------------------------------------------------------------------- 1 | import secrets 2 | 3 | import pytest 4 | 5 | from yapapi.contrib.service.chunk import chunks 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "data, chunk_limit, num_chunks_expected, pass_as_memoryview", 10 | ( 11 | (secrets.token_bytes(16), 16, 1, True), 12 | (secrets.token_bytes(8), 16, 1, True), 13 | (secrets.token_bytes(17), 16, 2, True), 14 | (secrets.token_bytes(31), 16, 2, True), 15 | (secrets.token_bytes(256), 16, 16, True), 16 | (secrets.token_bytes(257), 16, 17, True), 17 | (secrets.token_bytes(16), 16, 1, False), 18 | (secrets.token_bytes(8), 16, 1, False), 19 | (secrets.token_bytes(17), 16, 2, False), 20 | (secrets.token_bytes(31), 16, 2, False), 21 | (secrets.token_bytes(256), 16, 16, False), 22 | (secrets.token_bytes(257), 16, 17, False), 23 | ), 24 | ) 25 | def test_chunks(data, chunk_limit, num_chunks_expected, pass_as_memoryview): 26 | num_chunks_received = 0 27 | data_out = b"" 28 | data_in = memoryview(data) if pass_as_memoryview else data 29 | for chunk in chunks(data_in, chunk_limit): 30 | data_out += chunk 31 | num_chunks_received += 1 32 | 33 | assert num_chunks_received == num_chunks_expected 34 | assert data_out == data 35 | -------------------------------------------------------------------------------- /tests/drone/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | RUN apk add --no-cache --update bash stress-ng 4 | RUN rm /lib/libcrypto* 5 | RUN mkdir -p /golem/input /golem/output 6 | 7 | COPY task.sh /golem/ 8 | 9 | VOLUME /golem/input /golem/output 10 | -------------------------------------------------------------------------------- /tests/drone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/drone/__init__.py -------------------------------------------------------------------------------- /tests/drone/task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_NAME=$0 4 | LC_NUMERIC=C 5 | 6 | DEFAULT_TIME="0" 7 | DEFAULT_STDOUT_RATE="0" 8 | DEFAULT_STDERR_RATE="0" 9 | DEFAULT_EXIT_CODE="0" 10 | DEFAULT_INTERVAL="0.25" 11 | 12 | function usage() { 13 | echo "Usage:" 14 | echo " $SCRIPT_NAME [-t time] [-o rate] [-e rate] [-c code] [-f path(,size)] [-i interval]" 15 | echo " -h,--help display help message" 16 | echo " -t,--time running time [default: 0]" 17 | echo " -o,--stdout stdout output rate (B/s) [default: 0]" 18 | echo " -e,--stderr stderr output rate (B/s) [default: 0]" 19 | echo " -c,--code exit code [default: 0]" 20 | echo " -i,--interval sleep interval in s [default: 0.25]" 21 | echo " -f,--file generate output file (opt. with N random bytes)" 22 | exit 201 23 | } 24 | 25 | function read_tuple() { 26 | declare -n result=$2 27 | local f=$(echo $(echo $1 | cut -d',' -f1)) 28 | local s=$(echo $(echo $1 | cut -d',' -f2 -s)) 29 | result=($f $s) 30 | } 31 | 32 | function read_uint() { 33 | declare -n result=$2 34 | if [[ $1 =~ ^[0-9]+$ ]]; then 35 | result=$(num $1) 36 | else 37 | >&2 echo "invalid unsigned integer" 38 | exit 202; 39 | fi 40 | } 41 | 42 | function num() { 43 | [[ -z "$1" ]] && echo "0" || echo "$1" 44 | } 45 | 46 | function max() { 47 | local a=$(num $1) 48 | local b=$(num $2) 49 | (( $a > $b )) && echo "$a" || echo "$b" 50 | } 51 | 52 | function round() { 53 | local n=$(num $1) 54 | local f=$(echo "scale=1; $n + 0.5" | bc) 55 | echo ${f%.*} 56 | } 57 | 58 | function random_str() { 59 | tr -dc A-Za-z0-9 < /dev/urandom | head -c $1 60 | } 61 | 62 | function timestamp() { 63 | if [[ -z "$1" ]]; then 64 | date +"%s" 65 | else 66 | local now=$(date +"%s") 67 | echo $(( $now + $1 )) 68 | fi 69 | } 70 | 71 | function main() { 72 | local p_time=$DEFAULT_TIME 73 | local p_stdout=$DEFAULT_STDOUT_RATE 74 | local p_stderr=$DEFAULT_STDERR_RATE 75 | local p_code=$DEFAULT_EXIT_CODE 76 | local p_interval=$DEFAULT_INTERVAL 77 | local p_file 78 | 79 | while [[ "$#" -gt 0 ]]; do 80 | case $1 in 81 | -h|--help) usage; exit 0 ;; 82 | -t|--time) read_uint $2 p_time; shift; shift ;; 83 | -o|--stdout) read_uint $2 p_stdout; shift; shift ;; 84 | -e|--stderr) read_uint $2 p_stderr; shift; shift ;; 85 | -c|--code) read_uint $2 p_code; shift; shift ;; 86 | -i|--interval) p_interval=$2; shift; shift ;; 87 | -f|--file) read_tuple $2 p_file; shift; shift ;; 88 | -*) usage ;; 89 | esac 90 | done 91 | 92 | local deadline=$( timestamp $p_time ) 93 | local stdout_csz=$( round $(echo "scale=4; ${p_stdout} * $p_interval" | bc) ) 94 | local stderr_csz=$( round $(echo "scale=4; ${p_stderr} * $p_interval" | bc) ) 95 | 96 | if [[ ! -z $p_file ]]; then 97 | mkdir -p $(dirname "${p_file[0]}") 98 | echo -n $( random_str $(num ${p_file[1]}) ) > "${p_file[0]}" 99 | fi 100 | 101 | while true; do 102 | if [[ $stdout_csz -gt 0 ]]; then 103 | echo -n $(random_str $stdout_csz) 104 | fi 105 | if [[ $stderr_csz -gt 0 ]]; then 106 | >&2 echo -n $(random_str $stderr_csz) 107 | fi 108 | if [[ $(timestamp) -ge $deadline ]]; then 109 | break 110 | fi 111 | sleep $p_interval 112 | done 113 | 114 | exit $p_code 115 | } 116 | 117 | main "$@" 118 | -------------------------------------------------------------------------------- /tests/engine/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/engine/__init__.py -------------------------------------------------------------------------------- /tests/engine/test_engine.py: -------------------------------------------------------------------------------- 1 | """Unit tests for `yapapi.engine` module.""" 2 | 3 | from unittest.mock import Mock 4 | 5 | import pytest 6 | 7 | import yapapi.engine 8 | import yapapi.rest 9 | from tests.factories.golem import GolemFactory 10 | from yapapi.engine import Job 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "default_subnet, subnet_arg, expected_subnet", 15 | [ 16 | (None, None, None), 17 | ("my-little-subnet", None, "my-little-subnet"), 18 | (None, "whole-golem", "whole-golem"), 19 | ("my-little-subnet", "whole-golem", "whole-golem"), 20 | ], 21 | ) 22 | def test_set_subnet_tag(default_subnet, subnet_arg, expected_subnet, monkeypatch): 23 | """Check that `subnet_tag` argument takes precedence over `yapapi.engine.DEFAULT_SUBNET`.""" 24 | 25 | monkeypatch.setattr(yapapi.engine, "DEFAULT_SUBNET", default_subnet) 26 | 27 | if subnet_arg is not None: 28 | golem = GolemFactory(subnet_tag=subnet_arg) 29 | else: 30 | golem = GolemFactory() 31 | assert golem.subnet_tag == expected_subnet 32 | 33 | 34 | def test_job_id(monkeypatch): 35 | """Test automatic generation of job ids.""" 36 | 37 | used_ids = [] 38 | 39 | job_1 = Job(engine=Mock(), expiration_time=Mock(), payload=Mock()) 40 | assert job_1.id 41 | used_ids.append(job_1.id) 42 | 43 | job_2 = Job(engine=Mock(), expiration_time=Mock(), payload=Mock()) 44 | assert job_2.id 45 | assert job_2.id not in used_ids 46 | used_ids.append(job_2.id) 47 | 48 | user_id_3 = f"{job_1.id}:{job_2.id}" 49 | job_3 = Job(engine=Mock(), expiration_time=Mock(), payload=Mock(), id=user_id_3) 50 | assert job_3.id == user_id_3 51 | used_ids.append(user_id_3) 52 | 53 | job_4 = Job(engine=Mock(), expiration_time=Mock(), payload=Mock()) 54 | assert job_4.id 55 | assert job_4.id not in used_ids 56 | used_ids.append(job_4.id) 57 | 58 | # Assuming generated ids are just numbers: pass str(N+1) as the user-specified id, 59 | # where N is the numeric value of the last autogenerated id, and make sure the next 60 | # autogenerated id is not str(N+1) (a duplicate). 61 | numeric_ids = set() 62 | for id in used_ids: 63 | try: 64 | numeric_ids.add(int(id)) 65 | except ValueError: 66 | pass 67 | if numeric_ids: 68 | max_id = max(numeric_ids) 69 | next_id = str(max_id + 1) 70 | job_5 = Job(engine=Mock(), expiration_time=Mock(), payload=Mock(), id=next_id) 71 | used_ids.append(job_5.id) 72 | job_6 = Job(engine=Mock(), expiration_time=Mock(), payload=Mock()) 73 | assert job_6.id not in used_ids 74 | 75 | # Passing an already used id should raise a ValueError 76 | with pytest.raises(ValueError): 77 | duplicate_id = used_ids[0] 78 | Job(engine=Mock(), expiration_time=Mock(), payload=Mock(), id=duplicate_id) 79 | -------------------------------------------------------------------------------- /tests/events/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/events/__init__.py -------------------------------------------------------------------------------- /tests/events/test_frozen.py: -------------------------------------------------------------------------------- 1 | import attr 2 | import pytest 3 | 4 | from yapapi.events import TaskStarted 5 | 6 | 7 | def test_frozen_is_inherited(): 8 | test_event = TaskStarted(job="a-job", agreement="an-agr", activity="an-act", task="a-task") 9 | with pytest.raises(attr.exceptions.FrozenInstanceError): 10 | test_event.job = "some-other-job" 11 | -------------------------------------------------------------------------------- /tests/events/test_repr.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from yapapi.events import ( 4 | AgreementCreated, 5 | CollectFailed, 6 | CommandExecuted, 7 | CommandStarted, 8 | Event, 9 | ExecutionInterrupted, 10 | ProposalReceived, 11 | ScriptSent, 12 | ServiceFinished, 13 | SubscriptionFailed, 14 | TaskStarted, 15 | ) 16 | 17 | 18 | @pytest.mark.parametrize( 19 | "event, expected_str", 20 | [ 21 | ( 22 | SubscriptionFailed(job="a-job", reason="the-reason"), 23 | "SubscriptionFailed(job='a-job', reason='the-reason')", 24 | ), 25 | ( 26 | CollectFailed(job="a-job", subscription="a-sub", reason="the-reason"), 27 | "CollectFailed(job='a-job', subscription='a-sub', reason='the-reason')", 28 | ), 29 | ( 30 | ProposalReceived(job="a-job", proposal="a-prop"), 31 | "ProposalReceived(job='a-job', proposal='a-prop')", 32 | ), 33 | ( 34 | AgreementCreated(job="a-job", agreement="an-agr"), 35 | "AgreementCreated(job='a-job', agreement='an-agr')", 36 | ), 37 | ( 38 | TaskStarted(job="a-job", agreement="an-agr", activity="an-act", task="a-task"), 39 | "TaskStarted(job='a-job', agreement='an-agr', activity='an-act', task='a-task')", 40 | ), 41 | ( 42 | ServiceFinished( 43 | job="a-job", agreement="an-agr", activity="an-act", service="a-service" 44 | ), 45 | "ServiceFinished(job='a-job', agreement='an-agr', activity='an-act'," 46 | " service='a-service')", 47 | ), 48 | ( 49 | ScriptSent(job="a-job", agreement="an-agr", activity="an-act", script="a-script"), 50 | "ScriptSent(job='a-job', agreement='an-agr', activity='an-act', script='a-script')", 51 | ), 52 | ( 53 | CommandStarted( 54 | job="a-job", 55 | agreement="an-agr", 56 | activity="an-act", 57 | script="a-script", 58 | command="the-command", 59 | ), 60 | "CommandStarted(job='a-job', agreement='an-agr', activity='an-act', " 61 | "script='a-script', command='the-command')", 62 | ), 63 | ( 64 | CommandExecuted( 65 | job="a-job", 66 | agreement="an-agr", 67 | activity="an-act", 68 | script="a-script", 69 | command="the-command", 70 | success=True, 71 | message="the-message", 72 | ), 73 | "CommandExecuted(job='a-job', agreement='an-agr', activity='an-act', " 74 | "script='a-script', command='the-command', success=True, message='the-message', " 75 | "stdout=None, stderr=None)", 76 | ), 77 | ( 78 | ExecutionInterrupted(exc_info=(RuntimeError.__class__, RuntimeError(), None)), 79 | "ExecutionInterrupted(exception=RuntimeError())", 80 | ), 81 | ], 82 | ) 83 | def test_event_to_str(event: Event, expected_str: str): 84 | assert str(event) == expected_str 85 | assert repr(event) == str(event) 86 | -------------------------------------------------------------------------------- /tests/executor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/executor/__init__.py -------------------------------------------------------------------------------- /tests/executor/test_task.py: -------------------------------------------------------------------------------- 1 | from yapapi.executor import Task 2 | 3 | 4 | def test_task(): 5 | t: Task[int, None] = Task(data=1) 6 | 7 | assert t.data == 1 8 | -------------------------------------------------------------------------------- /tests/factories/__init__.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | 4 | def dataclass_fields_dict(data_class): 5 | return {field.name: field for field in dataclasses.fields(data_class)} 6 | -------------------------------------------------------------------------------- /tests/factories/agreements_pool.py: -------------------------------------------------------------------------------- 1 | import factory 2 | 3 | from yapapi.agreements_pool import BufferedAgreement 4 | 5 | from .rest.market import AgreementFactory 6 | 7 | 8 | class BufferedAgreementFactory(factory.Factory): 9 | class Meta: 10 | model = BufferedAgreement 11 | 12 | agreement = factory.SubFactory(AgreementFactory) 13 | agreement_details = factory.lazy_attribute(lambda o: o.agreement._details) # noqa 14 | worker_task = None 15 | has_multi_activity = False 16 | -------------------------------------------------------------------------------- /tests/factories/config.py: -------------------------------------------------------------------------------- 1 | import factory 2 | 3 | import yapapi.config 4 | 5 | 6 | class ApiConfigFactory(factory.Factory): 7 | class Meta: 8 | model = yapapi.config.ApiConfig 9 | 10 | app_key = "yagna-app-key" 11 | -------------------------------------------------------------------------------- /tests/factories/context.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import factory 4 | 5 | from yapapi.ctx import WorkContext 6 | 7 | 8 | def mock_emitter(event_class, **kwargs): 9 | kwargs["job"] = mock.MagicMock() 10 | return event_class(**kwargs) 11 | 12 | 13 | class WorkContextFactory(factory.Factory): 14 | class Meta: 15 | model = WorkContext 16 | 17 | activity = factory.LazyFunction(mock.MagicMock) 18 | agreement = factory.LazyFunction(mock.MagicMock) 19 | storage = factory.LazyFunction(mock.AsyncMock) 20 | emitter = mock_emitter 21 | -------------------------------------------------------------------------------- /tests/factories/events.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import factory 4 | 5 | from yapapi.events import AgreementConfirmed, AgreementEvent, AgreementRejected 6 | 7 | 8 | class _JobFactory(factory.Factory): 9 | class Meta: 10 | model = mock.Mock 11 | 12 | 13 | class _AgreementFactory(factory.Factory): 14 | class Meta: 15 | model = mock.Mock 16 | 17 | @factory.post_generation 18 | def provider_id(obj: mock.Mock, create, extracted, **kwargs): 19 | obj.details.raw_details.offer.provider_id = extracted 20 | 21 | 22 | class AgreementEventFactory(factory.Factory): 23 | class Meta: 24 | model = AgreementEvent 25 | 26 | job = factory.SubFactory(_JobFactory) 27 | agreement = factory.SubFactory(_AgreementFactory) 28 | 29 | 30 | class AgreementConfirmedFactory(AgreementEventFactory): 31 | class Meta: 32 | model = AgreementConfirmed 33 | 34 | 35 | class AgreementRejectedFactory(AgreementEventFactory): 36 | class Meta: 37 | model = AgreementRejected 38 | -------------------------------------------------------------------------------- /tests/factories/golem.py: -------------------------------------------------------------------------------- 1 | import factory 2 | 3 | import yapapi.golem 4 | from tests.factories.config import ApiConfigFactory 5 | 6 | 7 | class GolemFactory(factory.Factory): 8 | class Meta: 9 | model = yapapi.golem.Golem 10 | 11 | budget = 10.0 12 | api_config = factory.SubFactory(ApiConfigFactory) 13 | -------------------------------------------------------------------------------- /tests/factories/network.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from concurrent import futures 3 | from unittest import mock 4 | 5 | import factory 6 | import faker 7 | 8 | from yapapi.network import Network 9 | 10 | 11 | class NetworkFactory(factory.Factory): 12 | class Meta: 13 | model = Network 14 | 15 | ip = factory.Faker("ipv4", network=True) 16 | owner_id = factory.LazyFunction(lambda: "0x" + faker.Faker().binary(length=20).hex()) 17 | 18 | @classmethod 19 | def _create(cls, model_class, *args, **kwargs): 20 | if "net_api" not in kwargs: 21 | net_api = mock.AsyncMock() 22 | net_api.create_network = mock.AsyncMock( 23 | return_value=faker.Faker().binary(length=16).hex() 24 | ) 25 | net_api.remove_network = mock.AsyncMock() 26 | kwargs["net_api"] = net_api 27 | 28 | # we're using `futures.ThreadPoolExecutor` here 29 | # to run an async awaitable in a synchronous manner 30 | pool = futures.ThreadPoolExecutor() 31 | return pool.submit(asyncio.run, model_class.create(*args, **kwargs)).result() 32 | -------------------------------------------------------------------------------- /tests/factories/props/__init__.py: -------------------------------------------------------------------------------- 1 | import factory 2 | 3 | from yapapi.props import NodeInfo 4 | 5 | 6 | class NodeInfoFactory(factory.Factory): 7 | class Meta: 8 | model = NodeInfo 9 | 10 | name = factory.Faker("pystr") 11 | subnet_tag = "public" 12 | -------------------------------------------------------------------------------- /tests/factories/props/com.py: -------------------------------------------------------------------------------- 1 | import factory 2 | 3 | from tests.factories import dataclass_fields_dict 4 | from yapapi.props import com 5 | 6 | _com_linear = dataclass_fields_dict(com.ComLinear) 7 | 8 | 9 | class ComLinearPropsFactory(factory.DictFactory): 10 | class Meta: 11 | rename = { 12 | "price_model": _com_linear["price_model"].metadata["key"], 13 | "linear_coeffs": com.LINEAR_COEFFS, 14 | "usage_vector": com.DEFINED_USAGES, 15 | "scheme": _com_linear["scheme"].metadata["key"], 16 | } 17 | 18 | price_model = com.PriceModel.LINEAR.value 19 | scheme = com.BillingScheme.PAYU.value 20 | linear_coeffs = (0.001, 0.002, 0.1) 21 | usage_vector = [com.Counter.CPU.value, com.Counter.TIME.value] 22 | 23 | 24 | class ComLinearFactory(factory.Factory): 25 | class Meta: 26 | model = com.ComLinear 27 | 28 | price_model = com.PriceModel.LINEAR.value 29 | scheme = com.BillingScheme.PAYU.value 30 | linear_coeffs = (0.001, 0.002, 0.1) 31 | usage_vector = [com.Counter.CPU.value, com.Counter.TIME.value] 32 | -------------------------------------------------------------------------------- /tests/factories/rest/__init__.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import factory 4 | 5 | from ya_market.api.requestor_api import RequestorApi 6 | 7 | 8 | class RestApiModelFactory(factory.Factory): 9 | api = factory.LazyFunction(lambda: RequestorApi(mock.Mock())) 10 | -------------------------------------------------------------------------------- /tests/factories/rest/market.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import factory 4 | 5 | from ya_market import models as market_models 6 | 7 | from tests.factories.props.com import ComLinearPropsFactory 8 | from tests.factories.rest import RestApiModelFactory 9 | from yapapi.rest.market import Agreement, AgreementDetails, OfferProposal, Subscription 10 | 11 | 12 | class SubscriptionFactory(RestApiModelFactory): 13 | class Meta: 14 | model = Subscription 15 | 16 | subscription_id = factory.Faker("pystr") 17 | 18 | 19 | class RestProposalFactory(factory.Factory): 20 | class Meta: 21 | model = market_models.Proposal 22 | 23 | properties = factory.SubFactory(ComLinearPropsFactory) 24 | constraints = "" 25 | proposal_id = factory.Faker("pystr") 26 | issuer_id = factory.Faker("pystr") 27 | state = "Initial" 28 | timestamp = factory.LazyFunction(datetime.datetime.now) 29 | 30 | 31 | class RestProposalEventFactory(factory.Factory): 32 | class Meta: 33 | model = market_models.ProposalEvent 34 | 35 | proposal = factory.SubFactory(RestProposalFactory) 36 | 37 | 38 | class OfferProposalFactory(factory.Factory): 39 | class Meta: 40 | model = OfferProposal 41 | 42 | @classmethod 43 | def create(cls, provider_id=None, coeffs=None, **kwargs): 44 | if provider_id: 45 | kwargs["proposal__proposal__issuer_id"] = provider_id 46 | if coeffs: 47 | kwargs["proposal__proposal__properties__linear_coeffs"] = list(coeffs) 48 | return super().create(**kwargs) 49 | 50 | subscription = factory.SubFactory(SubscriptionFactory) 51 | proposal = factory.SubFactory(RestProposalEventFactory) 52 | 53 | 54 | class RestDemandFactory(factory.Factory): 55 | class Meta: 56 | model = market_models.Demand 57 | 58 | properties = factory.DictFactory() 59 | constraints = "" 60 | demand_id = factory.Faker("pystr") 61 | requestor_id = factory.Faker("pystr") 62 | timestamp = factory.LazyFunction(datetime.datetime.now) 63 | 64 | 65 | class RestOfferFactory(factory.Factory): 66 | class Meta: 67 | model = market_models.Offer 68 | 69 | properties = factory.DictFactory() 70 | constraints = "" 71 | offer_id = factory.Faker("pystr") 72 | provider_id = factory.Faker("pystr") 73 | timestamp = factory.LazyFunction(datetime.datetime.now) 74 | 75 | 76 | class RestAgreementFactory(factory.Factory): 77 | class Meta: 78 | model = market_models.Agreement 79 | 80 | agreement_id = factory.Faker("pystr") 81 | demand = factory.SubFactory(RestDemandFactory) 82 | offer = factory.SubFactory(RestOfferFactory) 83 | valid_to = factory.LazyFunction(lambda: object()) 84 | state = "Approved" 85 | timestamp = factory.LazyFunction(datetime.datetime.now) 86 | 87 | 88 | class AgreementDetailsFactory(factory.Factory): 89 | class Meta: 90 | model = AgreementDetails 91 | 92 | _ref = factory.SubFactory(RestAgreementFactory) 93 | 94 | 95 | class AgreementFactory(RestApiModelFactory): 96 | class Meta: 97 | model = Agreement 98 | 99 | @factory.post_generation 100 | def details(obj: Agreement, created, extracted, **kwargs): 101 | if extracted: 102 | obj._details = extracted 103 | else: 104 | obj._details = AgreementDetailsFactory(**kwargs) 105 | 106 | @factory.post_generation 107 | def terminated(obj: Agreement, created, extracted, **kwargs): 108 | assert not kwargs 109 | if extracted: 110 | obj._terminated = extracted 111 | 112 | agreement_id = factory.Faker("pystr") 113 | subscription = factory.SubFactory(SubscriptionFactory) 114 | -------------------------------------------------------------------------------- /tests/factories/rest/payment.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from unittest import mock 3 | 4 | import factory 5 | 6 | from ya_payment import models as payment_models 7 | from ya_payment.api.requestor_api import RequestorApi 8 | 9 | from yapapi.rest.payment import DebitNote 10 | 11 | 12 | class RestDebitNoteFactory(factory.Factory): 13 | class Meta: 14 | model = payment_models.DebitNote 15 | 16 | debit_note_id = factory.Faker("pystr") 17 | issuer_id = factory.Faker("pystr") 18 | recipient_id = factory.Faker("pystr") 19 | payee_addr = factory.Faker("pystr") 20 | payer_addr = factory.Faker("pystr") 21 | payment_platform = factory.Faker("pystr") 22 | timestamp = factory.LazyFunction(datetime.datetime.now) 23 | agreement_id = factory.Faker("pystr") 24 | activity_id = factory.Faker("pystr") 25 | total_amount_due = factory.Sequence(lambda n: str(n)) 26 | status = "RECEIVED" 27 | 28 | 29 | class DebitNoteFactory(factory.Factory): 30 | class Meta: 31 | model = DebitNote 32 | 33 | _api = factory.LazyFunction(lambda: RequestorApi(mock.Mock())) 34 | _base = factory.SubFactory(RestDebitNoteFactory) 35 | -------------------------------------------------------------------------------- /tests/goth_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/_util.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | 4 | def get_free_port(range_start: int = 8080, range_end: int = 9090) -> int: 5 | """Get the first available port on localhost within the specified range. 6 | 7 | The range is inclusive on both sides (i.e. `range_end` will be included). 8 | Raises `RuntimeError` when no free port could be found. 9 | """ 10 | for port in range(range_start, range_end + 1): 11 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 12 | try: 13 | s.bind(("", port)) 14 | return port 15 | except OSError: 16 | pass 17 | -------------------------------------------------------------------------------- /tests/goth_tests/assertions.py: -------------------------------------------------------------------------------- 1 | """Temporal assertions used in `goth` integration tests. 2 | 3 | The assertions express properties of lines of output printed by requestor 4 | scripts (e.g. `blender.py`) to their stdout or stderr. For example, one such 5 | property would be that if the requestor script prints a line containing 6 | the string "Agreement confirmed by provider P", for any name P, then 7 | eventually it will also print a line containing "Accepted invoice from P". 8 | """ 9 | 10 | import logging 11 | import re 12 | from typing import Set 13 | 14 | from goth.assertions import EventStream 15 | 16 | logger = logging.getLogger("goth.test.assertions") 17 | 18 | 19 | async def assert_no_errors(output_lines: EventStream[str]): 20 | """Assert that no output line contains the substring `ERROR`.""" 21 | async for line in output_lines: 22 | if "ERROR" in line: 23 | raise AssertionError("Command reported ERROR") 24 | 25 | 26 | async def assert_all_invoices_accepted(output_lines: EventStream[str]): 27 | """Assert that an invoice is accepted for every provider that confirmed an agreement.""" 28 | unpaid_agreement_providers = list() 29 | 30 | async for line in output_lines: 31 | m = re.search("Agreement confirmed by provider '([^']*)'", line) 32 | if m: 33 | prov_name = m.group(1) 34 | logger.debug("assert_all_invoices_accepted: adding provider '%s'", prov_name) 35 | unpaid_agreement_providers.append(prov_name) 36 | m = re.search("Accepted invoice from '([^']*)'", line) 37 | if m: 38 | prov_name = m.group(1) 39 | logger.debug("assert_all_invoices_accepted: adding invoice for '%s'", prov_name) 40 | unpaid_agreement_providers.remove(prov_name) 41 | 42 | if unpaid_agreement_providers: 43 | raise AssertionError(f"Unpaid agreements for: {','.join(unpaid_agreement_providers)}") 44 | 45 | 46 | async def assert_tasks_processed(tasks: Set[str], status: str, output_lines: EventStream[str]): 47 | """Assert that for every task in `tasks` a line with `Task {status}` will appear.""" 48 | remaining_tasks = tasks.copy() 49 | 50 | async for line in output_lines: 51 | m = re.search(rf".*Task {status} .* task data: (.+)$", line) 52 | if m: 53 | task_data = m.group(1) 54 | logger.debug("assert_tasks_processed: Task %s: %s", status, task_data) 55 | remaining_tasks.discard(task_data) 56 | if not remaining_tasks: 57 | return 58 | 59 | raise AssertionError(f"Tasks not {status}: {remaining_tasks}") 60 | -------------------------------------------------------------------------------- /tests/goth_tests/test_agreement_termination/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_agreement_termination/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_agreement_termination/requestor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """A requestor script used for testing agreement termination.""" 3 | import asyncio 4 | import logging 5 | from datetime import timedelta 6 | 7 | from yapapi import Golem, Task, WorkContext 8 | from yapapi.log import enable_default_logger 9 | from yapapi.payload import vm 10 | 11 | 12 | async def main(): 13 | package = await vm.repo( 14 | image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", 15 | min_mem_gib=0.5, 16 | min_storage_gib=2.0, 17 | ) 18 | 19 | first_worker = True 20 | 21 | async def worker(ctx: WorkContext, tasks): 22 | """Execute `Golem.execute_tasks()` as a worker. 23 | 24 | The first call to this function will produce a worker 25 | that sends an invalid `run` command to the provider. 26 | This should cause `yield script` to fail with 27 | `CommandExecutionError`. 28 | 29 | The remaining calls will just send `sleep 5` to the 30 | provider to simulate some work. 31 | """ 32 | 33 | nonlocal first_worker 34 | should_fail = first_worker 35 | first_worker = False 36 | 37 | async for task in tasks: 38 | script = ctx.new_script() 39 | 40 | if should_fail: 41 | # Send a command that will fail on the provider 42 | script.run("xyz") 43 | yield script 44 | else: 45 | # Simulate some work 46 | script.run("/bin/sleep", "1") 47 | yield script 48 | 49 | task.accept_result() 50 | 51 | async with Golem( 52 | budget=10.0, 53 | subnet_tag="goth", 54 | payment_network="holesky", 55 | ) as golem: 56 | tasks = [Task(data=n) for n in range(6)] 57 | async for task in golem.execute_tasks( 58 | worker, 59 | tasks, 60 | package, 61 | max_workers=1, 62 | timeout=timedelta(minutes=6), 63 | ): 64 | print(f"Task computed: {task}, time: {task.running_time}") 65 | 66 | print("All tasks computed") 67 | 68 | 69 | if __name__ == "__main__": 70 | enable_default_logger(log_file="test.log") 71 | 72 | console_handler = logging.StreamHandler() 73 | console_handler.setLevel(logging.DEBUG) 74 | logging.getLogger("yapapi.events").addHandler(console_handler) 75 | 76 | loop = asyncio.get_event_loop() 77 | task = loop.create_task(main()) 78 | 79 | try: 80 | loop.run_until_complete(task) 81 | except KeyboardInterrupt: 82 | print("Shutting down gracefully...") 83 | task.cancel() 84 | try: 85 | loop.run_until_complete(task) 86 | print("Shutdown completed") 87 | except (asyncio.CancelledError, KeyboardInterrupt): 88 | pass 89 | -------------------------------------------------------------------------------- /tests/goth_tests/test_async_task_generation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_async_task_generation/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_async_task_generation/requestor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """A requestor script for testing asynchronous generation of input tasks.""" 3 | import asyncio 4 | import pathlib 5 | import sys 6 | from datetime import timedelta 7 | 8 | from yapapi import Golem, Task 9 | from yapapi.log import enable_default_logger, log_event_repr 10 | from yapapi.payload import vm 11 | 12 | 13 | async def main(): 14 | vm_package = await vm.repo( 15 | image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", 16 | min_mem_gib=0.5, 17 | min_storage_gib=2.0, 18 | ) 19 | 20 | async def worker(work_ctx, tasks): 21 | async for task in tasks: 22 | script = work_ctx.new_script() 23 | print("task data:", task.data, file=sys.stderr) 24 | script.run("/bin/sleep", "1") 25 | yield script 26 | task.accept_result(result=task.data) 27 | 28 | async with Golem( 29 | budget=10.0, 30 | subnet_tag="goth", 31 | event_consumer=log_event_repr, 32 | payment_network="holesky", 33 | ) as golem: 34 | # We use an async task generator that yields tasks removed from 35 | # an async queue. Each computed task will potentially spawn 36 | # new tasks -- this is made possible thanks to using async task 37 | # generator as an input to `executor.submit()`. 38 | 39 | task_queue = asyncio.Queue() 40 | 41 | # Seed the queue with the first task: 42 | await task_queue.put(Task(data=3)) 43 | 44 | async def input_generator(): 45 | """Task generator yields tasks removed from `queue`.""" 46 | while True: 47 | task = await task_queue.get() 48 | if task.data == 0: 49 | break 50 | yield task 51 | 52 | async for task in golem.execute_tasks( 53 | worker, 54 | input_generator(), 55 | vm_package, 56 | max_workers=1, 57 | timeout=timedelta(minutes=6), 58 | ): 59 | print("task result:", task.result, file=sys.stderr) 60 | for n in range(task.result): 61 | await task_queue.put(Task(data=task.result - 1)) 62 | 63 | print("all done!", file=sys.stderr) 64 | 65 | 66 | if __name__ == "__main__": 67 | test_dir = pathlib.Path(__file__).parent.name 68 | enable_default_logger(log_file=f"{test_dir}.log") 69 | asyncio.run(main()) 70 | -------------------------------------------------------------------------------- /tests/goth_tests/test_async_task_generation/test_async_task_generation.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from pathlib import Path 4 | from typing import List 5 | 6 | import pytest 7 | 8 | import goth.configuration 9 | from goth.runner import Runner 10 | from goth.runner.log import configure_logging 11 | from goth.runner.probe import RequestorProbe 12 | 13 | logger = logging.getLogger("goth.test.async_task_generation") 14 | 15 | 16 | @pytest.mark.asyncio 17 | async def test_async_task_generation( 18 | log_dir: Path, 19 | goth_config_path: Path, 20 | config_overrides: List[goth.configuration.Override], 21 | single_node_override: goth.configuration.Override, 22 | ) -> None: 23 | """Run the `requestor.py` and make sure that it's standard output is as expected.""" 24 | 25 | configure_logging(log_dir) 26 | 27 | # Override the default test configuration to create only one provider node 28 | goth_config = goth.configuration.load_yaml( 29 | goth_config_path, config_overrides + [single_node_override] 30 | ) 31 | 32 | runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) 33 | 34 | async with runner(goth_config.containers): 35 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 36 | 37 | async with requestor.run_command_on_host( 38 | str(Path(__file__).parent / "requestor.py"), env=os.environ 39 | ) as (_cmd_task, cmd_monitor, _process_monitor): 40 | # The requestor should print "task result: 3" once ... 41 | await cmd_monitor.wait_for_pattern("task result: 3", timeout=60) 42 | # ... then "task result: 2" twice ... 43 | for _ in range(3): 44 | await cmd_monitor.wait_for_pattern("task result: 2", timeout=10) 45 | # ... and "task result: 1" six times. 46 | for _ in range(6): 47 | await cmd_monitor.wait_for_pattern("task result: 1", timeout=10) 48 | await cmd_monitor.wait_for_pattern("all done!", timeout=10) 49 | -------------------------------------------------------------------------------- /tests/goth_tests/test_concurrent_executors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_concurrent_executors/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_concurrent_executors/requestor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """A requestor script for testing concurrent execution of Golem.execute_tasks(). 3 | 4 | It creates a pipeline consisting of two `Golem.execute_tasks()` calls, 5 | with the results of the tasks computed by the first one (A) given as input data 6 | for the tasks computed by the second one (B). 7 | """ 8 | import asyncio 9 | import pathlib 10 | import sys 11 | 12 | from yapapi import Golem, Task 13 | from yapapi.log import enable_default_logger 14 | from yapapi.payload import vm 15 | 16 | 17 | async def main(): 18 | vm_package = await vm.repo( 19 | image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", 20 | min_mem_gib=0.5, 21 | min_storage_gib=2.0, 22 | ) 23 | 24 | first_task = True 25 | 26 | async def duplicator(work_ctx, tasks): 27 | """Execute `echo {task.data} {task.data}` command on provider as a worker. 28 | 29 | The command's output is the result of the whole task. 30 | 31 | The first ever task computed by an instance of this worker fails. 32 | It succeeds when re-tried. This may be used to test re-trying tasks, 33 | creating new agreements and displaying summary information on task/activity 34 | failures. 35 | """ 36 | async for task in tasks: 37 | nonlocal first_task 38 | 39 | script = work_ctx.new_script() 40 | 41 | if first_task: 42 | first_task = False 43 | script.run("/bin/sleep", "1") 44 | script.run("/command/not/found") 45 | 46 | script.run("/bin/sleep", "1") 47 | future_result = script.run("/bin/echo", task.data, task.data) 48 | yield script 49 | result = await future_result 50 | output = result.stdout.strip() 51 | task.accept_result(output) 52 | 53 | async with Golem(budget=1.0, subnet_tag="goth", payment_network="holesky") as golem: 54 | # Construct a pipeline: 55 | # 56 | # input_tasks 57 | # | 58 | # V 59 | # [ Job ALEF ] 60 | # | 61 | # V 62 | # intermediate_tasks 63 | # | 64 | # V 65 | # [ Job BET ] 66 | # | 67 | # V 68 | # output_tasks 69 | 70 | input_tasks = [Task(s) for s in "01234567"] 71 | 72 | computed_input_tasks = golem.execute_tasks( 73 | duplicator, 74 | input_tasks, 75 | payload=vm_package, 76 | max_workers=1, 77 | job_id="ALEF", 78 | ) 79 | 80 | async def intermediate_tasks(): 81 | async for task in computed_input_tasks: 82 | print(f"ALEF computed task: {task.data} -> {task.result}", file=sys.stderr) 83 | yield Task(data=task.result) 84 | 85 | output_tasks = golem.execute_tasks( 86 | duplicator, 87 | intermediate_tasks(), 88 | payload=vm_package, 89 | max_workers=1, 90 | job_id="BET", 91 | ) 92 | 93 | async for task in output_tasks: 94 | print(f"BET computed task: {task.data} -> {task.result}", file=sys.stderr) 95 | 96 | 97 | if __name__ == "__main__": 98 | test_dir = pathlib.Path(__file__).parent.name 99 | enable_default_logger(log_file=f"{test_dir}.log") 100 | asyncio.run(main()) 101 | -------------------------------------------------------------------------------- /tests/goth_tests/test_concurrent_executors/test_concurrent_executors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Test concurrent job execution and that SummaryLogger handles it correctly.""" 3 | import logging 4 | import os 5 | from pathlib import Path 6 | from typing import List 7 | 8 | import pytest 9 | 10 | import goth.configuration 11 | from goth.runner import Runner 12 | from goth.runner.log import configure_logging 13 | from goth.runner.probe import RequestorProbe 14 | 15 | logger = logging.getLogger("goth.test.async_concurrent_executors") 16 | 17 | 18 | @pytest.mark.asyncio 19 | async def test_concurrent_executors( 20 | log_dir: Path, 21 | goth_config_path: Path, 22 | config_overrides: List[goth.configuration.Override], 23 | ) -> None: 24 | """Run the `requestor.py` and make sure that it's standard output is as expected.""" 25 | 26 | configure_logging(log_dir) 27 | 28 | goth_config = goth.configuration.load_yaml(goth_config_path, config_overrides) 29 | 30 | runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) 31 | 32 | async with runner(goth_config.containers): 33 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 34 | 35 | async with requestor.run_command_on_host( 36 | str(Path(__file__).parent / "requestor.py"), env=os.environ 37 | ) as (_cmd_task, cmd_monitor, _process_monitor): 38 | # Wait for job ALEF summary 39 | await cmd_monitor.wait_for_pattern(".*ALEF.* Job finished", timeout=60) 40 | await cmd_monitor.wait_for_pattern(".*ALEF.* Negotiated 2 agreements", timeout=5) 41 | await cmd_monitor.wait_for_pattern(".*ALEF.* Provider .* computed 8 tasks", timeout=5) 42 | await cmd_monitor.wait_for_pattern(".*ALEF.* Activity failed 1 time", timeout=5) 43 | 44 | # Wait for job BET summary 45 | await cmd_monitor.wait_for_pattern(".*BET.* Job finished", timeout=60) 46 | await cmd_monitor.wait_for_pattern(".*BET.* Negotiated 1 agreement", timeout=5) 47 | await cmd_monitor.wait_for_pattern(".*BET.* Provider .* computed 8 tasks", timeout=5) 48 | 49 | await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) 50 | -------------------------------------------------------------------------------- /tests/goth_tests/test_instance_restart/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_instance_restart/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_instance_restart/test_instance_restart.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Test if a Cluster respawns an instance if an existing instance fails in the `starting` state.""" 3 | import logging 4 | import os 5 | from pathlib import Path 6 | from typing import List 7 | 8 | import pytest 9 | 10 | import goth.configuration 11 | from goth.runner import Runner 12 | from goth.runner.log import configure_logging 13 | from goth.runner.probe import RequestorProbe 14 | 15 | logger = logging.getLogger("goth.test.async_task_generation") 16 | 17 | 18 | instances_started = set() 19 | instances_running = set() 20 | 21 | 22 | async def count_instances(events): 23 | global instances_started, instances_running 24 | 25 | async for line in events: 26 | line = line.strip() 27 | try: 28 | word, num = line.split() 29 | if word == "STARTING": 30 | instances_started.add(int(num)) 31 | elif word == "RUNNING": 32 | instances_running.add(int(num)) 33 | except ValueError: 34 | pass 35 | 36 | 37 | @pytest.mark.asyncio 38 | async def test_instance_restart( 39 | log_dir: Path, 40 | goth_config_path: Path, 41 | config_overrides: List[goth.configuration.Override], 42 | single_node_override: goth.configuration.Override, 43 | ) -> None: 44 | """Run the `requestor.py` and make sure that it's standard output is as expected.""" 45 | 46 | configure_logging(log_dir) 47 | 48 | goth_config = goth.configuration.load_yaml( 49 | goth_config_path, config_overrides + [single_node_override] 50 | ) 51 | 52 | runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) 53 | 54 | async with runner(goth_config.containers): 55 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 56 | 57 | async with requestor.run_command_on_host( 58 | str(Path(__file__).parent / "requestor.py"), env=os.environ 59 | ) as (_cmd_task, cmd_monitor, _process_monitor): 60 | cmd_monitor.add_assertion(count_instances) 61 | 62 | # The first attempt to create an instance should fail 63 | await cmd_monitor.wait_for_pattern("STARTING 1$", timeout=60) 64 | await cmd_monitor.wait_for_pattern(".*CommandExecutionError", timeout=20) 65 | 66 | # The second one should successfully start and fail in `running` state 67 | await cmd_monitor.wait_for_pattern("STARTING 2$", timeout=20) 68 | await cmd_monitor.wait_for_pattern("RUNNING 2$", timeout=20) 69 | await cmd_monitor.wait_for_pattern(".*CommandExecutionError", timeout=20) 70 | await cmd_monitor.wait_for_pattern("STOPPING 2$", timeout=20) 71 | 72 | # The third instance should be started, but not running 73 | await cmd_monitor.wait_for_pattern("STARTING 3$", timeout=20) 74 | await cmd_monitor.wait_for_pattern("Cluster stopped$", timeout=60) 75 | 76 | assert instances_started == {1, 2, 3}, ( 77 | "Expected to see instances 1, 2, 3 starting, saw instances " 78 | f"{', '.join(str(n) for n in instances_started)} instead" 79 | ) 80 | assert instances_running == {2}, ( 81 | "Expected to see only instance 2 running, saw instances " 82 | f"{', '.join(str(n) for n in instances_running)} instead" 83 | ) 84 | -------------------------------------------------------------------------------- /tests/goth_tests/test_mid_agreement_payments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_mid_agreement_payments/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_mid_agreement_payments/requestor_agent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import asyncio 4 | import logging 5 | from datetime import timedelta 6 | from typing import AsyncIterable, List 7 | 8 | from yapapi import Golem, Task, WorkContext 9 | from yapapi.log import enable_default_logger, log_event_repr 10 | from yapapi.payload import vm 11 | from yapapi.strategy import ( 12 | PROP_DEBIT_NOTE_INTERVAL_SEC, 13 | PROP_PAYMENT_TIMEOUT_SEC, 14 | LeastExpensiveLinearPayuMS, 15 | PropValueRange, 16 | ) 17 | 18 | 19 | class ShortDebitNoteIntervalAndPaymentTimeout(LeastExpensiveLinearPayuMS): 20 | acceptable_prop_value_range_overrides = { 21 | PROP_DEBIT_NOTE_INTERVAL_SEC: PropValueRange(25, 30), 22 | PROP_PAYMENT_TIMEOUT_SEC: PropValueRange(60, 70), 23 | } 24 | 25 | 26 | async def worker(context: WorkContext, tasks: AsyncIterable[Task]): 27 | async for task in tasks: 28 | script = context.new_script() 29 | future_result = script.run("/bin/sh", "-c", "sleep 105") 30 | yield script 31 | task.accept_result(result=await future_result) 32 | 33 | 34 | async def main(): 35 | package = await vm.repo(image_hash="d646d7b93083d817846c2ae5c62c72ca0507782385a2e29291a3d376") 36 | 37 | tasks: List[Task] = [Task(data=None)] 38 | timeout = timedelta(hours=24) 39 | 40 | async with Golem( 41 | budget=10.0, 42 | strategy=ShortDebitNoteIntervalAndPaymentTimeout(), 43 | subnet_tag="goth", 44 | event_consumer=log_event_repr, 45 | payment_network="holesky", 46 | ) as golem: 47 | logger = logging.getLogger("yapapi") 48 | logger.handlers[0].setLevel(logging.DEBUG) 49 | async for completed in golem.execute_tasks( 50 | worker, tasks, payload=package, max_workers=1, timeout=timeout 51 | ): 52 | print(f"Task finished: {completed}.") 53 | 54 | 55 | if __name__ == "__main__": 56 | parser = argparse.ArgumentParser() 57 | parser.add_argument( 58 | "--log-file", 59 | default=str("mid_agreement_payments.log"), 60 | ) 61 | args = parser.parse_args() 62 | enable_default_logger(log_file=args.log_file) 63 | 64 | loop = asyncio.get_event_loop() 65 | task = loop.create_task(main()) 66 | loop.run_until_complete(task) 67 | -------------------------------------------------------------------------------- /tests/goth_tests/test_mid_agreement_payments/test_mid_agreement_payments.py: -------------------------------------------------------------------------------- 1 | """A goth test scenario for mid-agreement payments.""" 2 | 3 | import logging 4 | import os 5 | import re 6 | from datetime import datetime 7 | from pathlib import Path 8 | from typing import List, Optional 9 | 10 | import pytest 11 | 12 | from goth.assertions import EventStream 13 | from goth.configuration import Override, load_yaml 14 | from goth.runner import Runner 15 | from goth.runner.log import configure_logging 16 | from goth.runner.probe import RequestorProbe 17 | 18 | DEBIT_NOTE_INTERVAL_GRACE_PERIOD = 30 19 | 20 | logger = logging.getLogger("goth.test.mid_agreement_payments") 21 | 22 | 23 | async def assert_debit_note_freq(events: EventStream) -> str: 24 | expected_note_num: int = 1 25 | received_freq: Optional[int] = None 26 | note_num: int = 0 27 | async for log_line in events: 28 | if received_freq is None: 29 | m = re.search(r"Debit notes interval: ([0-9]+)", log_line) 30 | if m: 31 | received_freq = int(m.group(1)) 32 | 33 | m = re.search( 34 | r"Debit notes for activity.* ([0-9]+) notes/([0-9]+)", 35 | log_line, 36 | ) 37 | if m and "Payable" not in log_line: 38 | note_num = int(m.group(1)) 39 | total_time = int(m.group(2)) 40 | 41 | assert note_num == expected_note_num, "Unexpected debit note number" 42 | expected_note_num += 1 43 | assert ( 44 | received_freq is not None 45 | ), "Expected debit note frequency message before a debit note" 46 | assert ( 47 | total_time + DEBIT_NOTE_INTERVAL_GRACE_PERIOD > note_num * received_freq 48 | ), "Too many notes" 49 | assert note_num >= 2, "Expected at least two debit notes" 50 | return f"{note_num} debit notes processed" 51 | 52 | 53 | @pytest.mark.asyncio 54 | async def test_mid_agreement_payments( 55 | project_dir: Path, 56 | log_dir: Path, 57 | goth_config_path: Path, 58 | config_overrides: List[Override], 59 | single_node_override: Override, 60 | ) -> None: 61 | # goth setup 62 | config = load_yaml(goth_config_path, config_overrides + [single_node_override]) 63 | configure_logging(log_dir) 64 | 65 | logfile = f"mid-agreement-payments-{datetime.now().strftime('%Y-%m-%d_%H.%M.%S')}.log" 66 | 67 | requestor_path = Path(__file__).parent / "requestor_agent.py" 68 | 69 | runner = Runner(base_log_dir=log_dir, compose_config=config.compose_config) 70 | async with runner(config.containers): 71 | # given 72 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 73 | # when 74 | async with requestor.run_command_on_host( 75 | f"{requestor_path} --log-file {(log_dir / logfile).resolve()}", env=os.environ 76 | ) as ( 77 | _, 78 | cmd_monitor, 79 | _, 80 | ): 81 | # then 82 | cmd_monitor.add_assertion(assert_debit_note_freq) 83 | # assert mid-agreement payments were enabled 84 | await cmd_monitor.wait_for_pattern(".*Enabling mid-agreement payments.*", timeout=60) 85 | # Wait for executor shutdown 86 | await cmd_monitor.wait_for_pattern(".*ShutdownFinished.*", timeout=200) 87 | logger.info("Requestor script finished") 88 | -------------------------------------------------------------------------------- /tests/goth_tests/test_multiactivity_agreement/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_multiactivity_agreement/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_multiactivity_agreement/requestor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """A requestor script for testing if multiple workers are run for an agreement.""" 3 | import asyncio 4 | import logging 5 | from datetime import timedelta 6 | 7 | from yapapi import Golem, Task 8 | from yapapi.log import enable_default_logger, log_event_repr # noqa 9 | from yapapi.payload import vm 10 | 11 | 12 | async def main(): 13 | vm_package = await vm.repo( 14 | image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", 15 | min_mem_gib=0.5, 16 | min_storage_gib=2.0, 17 | ) 18 | 19 | async def worker(work_ctx, tasks): 20 | """Compute just one task and exit.""" 21 | async for task in tasks: 22 | script = work_ctx.new_script() 23 | script.run("/bin/sleep", "1") 24 | yield script 25 | task.accept_result() 26 | return 27 | 28 | async with Golem( 29 | budget=10.0, 30 | subnet_tag="goth", 31 | event_consumer=log_event_repr, 32 | payment_network="holesky", 33 | ) as golem: 34 | tasks = [Task(data=n) for n in range(3)] 35 | async for task in golem.execute_tasks( 36 | worker, 37 | tasks, 38 | vm_package, 39 | max_workers=1, 40 | timeout=timedelta(minutes=6), 41 | ): 42 | print(f"Task computed: {task}") 43 | 44 | 45 | if __name__ == "__main__": 46 | enable_default_logger() 47 | console_handler = logging.StreamHandler() 48 | console_handler.setLevel(logging.DEBUG) 49 | logging.getLogger("yapapi.events").addHandler(console_handler) 50 | 51 | asyncio.run(main()) 52 | -------------------------------------------------------------------------------- /tests/goth_tests/test_multiactivity_agreement/test_multiactivity_agreement.py: -------------------------------------------------------------------------------- 1 | """A goth test scenario for multi-activity agreements.""" 2 | 3 | import logging 4 | import os 5 | import re 6 | from functools import partial 7 | from pathlib import Path 8 | from typing import List 9 | 10 | import pytest 11 | 12 | import goth.configuration 13 | from goth.runner import Runner 14 | from goth.runner.log import configure_logging 15 | from goth.runner.probe import RequestorProbe 16 | 17 | logger = logging.getLogger("goth.test.multiactivity_agreement") 18 | 19 | 20 | async def assert_agreement_created(events): 21 | """Assert that `AgreementCreated` event occurs.""" 22 | 23 | async for line in events: 24 | m = re.match(r"AgreementCreated\(.*Agreement\(id=([0-9a-f]+)", line) 25 | if m: 26 | return m.group(1) 27 | raise AssertionError("Expected AgreementCreated event") 28 | 29 | 30 | async def assert_multiple_workers_run(agr_id, events): 31 | """Assert that more than one worker is run with given `agr_id`. 32 | 33 | Fails if a worker failure is detected or if a worker has run for another agreement. 34 | """ 35 | workers_finished = 0 36 | 37 | async for line in events: 38 | m = re.match(r"WorkerFinished\(.*Agreement\(id=([0-9a-f]+)", line) 39 | if m: 40 | worker_agr_id = m.group(1) 41 | assert worker_agr_id == agr_id, "Worker run for another agreement" 42 | assert "exception" not in line, "Worker finished with error" 43 | workers_finished += 1 44 | elif re.match("JobFinished", line): 45 | break 46 | 47 | assert workers_finished > 1, ( 48 | f"Only {workers_finished} worker(s) run for agreement {agr_id}, " "expected more than one" 49 | ) 50 | 51 | 52 | @pytest.mark.asyncio 53 | async def test_multiactivity_agreement( 54 | log_dir: Path, 55 | goth_config_path: Path, 56 | config_overrides: List[goth.configuration.Override], 57 | single_node_override: goth.configuration.Override, 58 | ) -> None: 59 | configure_logging(log_dir) 60 | 61 | goth_config = goth.configuration.load_yaml( 62 | goth_config_path, config_overrides + [single_node_override] 63 | ) 64 | 65 | runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) 66 | 67 | async with runner(goth_config.containers): 68 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 69 | 70 | async with requestor.run_command_on_host( 71 | str(Path(__file__).parent / "requestor.py"), env=os.environ 72 | ) as (_cmd_task, cmd_monitor, _process_monitor): 73 | # Wait for agreement 74 | assertion = cmd_monitor.add_assertion(assert_agreement_created) 75 | agr_id = await assertion.wait_for_result(timeout=30) 76 | 77 | # Wait for multiple workers run for the agreement 78 | assertion = cmd_monitor.add_assertion( 79 | partial(assert_multiple_workers_run, agr_id), 80 | name=f"assert_multiple_workers_run({agr_id})", 81 | ) 82 | await assertion.wait_for_result(timeout=60) 83 | -------------------------------------------------------------------------------- /tests/goth_tests/test_recycle_ip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_recycle_ip/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_recycle_ip/ssh_recycle_ip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | import random 4 | import string 5 | from datetime import timedelta 6 | 7 | from yapapi import Golem 8 | from yapapi.contrib.service.socket_proxy import SocketProxy, SocketProxyService 9 | from yapapi.payload import vm 10 | 11 | first_time = True 12 | 13 | 14 | class SshService(SocketProxyService): 15 | remote_port = 22 16 | 17 | def __init__(self, proxy: SocketProxy): 18 | super().__init__() 19 | self.proxy = proxy 20 | 21 | @staticmethod 22 | async def get_payload(): 23 | return await vm.repo( 24 | image_hash="1e06505997e8bd1b9e1a00bd10d255fc6a390905e4d6840a22a79902", # ssh example 25 | capabilities=[vm.VM_CAPS_VPN], 26 | ) 27 | 28 | async def start(self): 29 | global first_time 30 | 31 | async for script in super().start(): 32 | yield script 33 | 34 | password = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) 35 | 36 | script = self._ctx.new_script(timeout=timedelta(seconds=10)) 37 | 38 | if first_time: 39 | first_time = False 40 | raise Exception("intentional failure on the first run") 41 | 42 | script.run("/bin/bash", "-c", "syslogd") 43 | script.run("/bin/bash", "-c", "ssh-keygen -A") 44 | script.run("/bin/bash", "-c", f'echo -e "{password}\n{password}" | passwd') 45 | script.run("/bin/bash", "-c", "/usr/sbin/sshd") 46 | yield script 47 | 48 | server = await self.proxy.run_server(self, self.remote_port) 49 | 50 | print( 51 | f"connect with:\n" 52 | f"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " 53 | f"-p {server.local_port} root@{server.local_address}" 54 | ) 55 | print(f"password: {password}") 56 | 57 | 58 | async def main(): 59 | async with Golem( 60 | budget=1.0, 61 | subnet_tag="goth", 62 | payment_network="holesky", 63 | ) as golem: 64 | network = await golem.create_network("192.168.0.1/24") 65 | proxy = SocketProxy(ports=[2222]) 66 | 67 | async with network: 68 | cluster = await golem.run_service( 69 | SshService, 70 | network=network, 71 | instance_params=[{"proxy": proxy}], 72 | network_addresses=["192.168.0.2"], 73 | ) 74 | instances = cluster.instances 75 | 76 | while True: 77 | print(instances) 78 | print(".....", network._nodes) 79 | try: 80 | await asyncio.sleep(5) 81 | except (KeyboardInterrupt, asyncio.CancelledError): 82 | break 83 | 84 | await proxy.stop() 85 | cluster.stop() 86 | 87 | cnt = 0 88 | while cnt < 3 and any(s.is_available for s in instances): 89 | print(instances) 90 | await asyncio.sleep(5) 91 | cnt += 1 92 | 93 | 94 | if __name__ == "__main__": 95 | try: 96 | loop = asyncio.get_event_loop() 97 | task = loop.create_task(main()) 98 | loop.run_until_complete(task) 99 | except KeyboardInterrupt: 100 | pass 101 | -------------------------------------------------------------------------------- /tests/goth_tests/test_renegotiate_proposal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/goth_tests/test_renegotiate_proposal/__init__.py -------------------------------------------------------------------------------- /tests/goth_tests/test_renegotiate_proposal/test_renegotiate_proposal.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from pathlib import Path 4 | from typing import List 5 | 6 | import pytest 7 | 8 | from goth.configuration import Override, load_yaml 9 | from goth.runner import Runner 10 | from goth.runner.log import configure_logging 11 | from goth.runner.probe import RequestorProbe 12 | 13 | logger = logging.getLogger("goth.test.renegotiate_proposal") 14 | 15 | 16 | @pytest.mark.asyncio 17 | async def test_renegotiation( 18 | log_dir: Path, 19 | goth_config_path: Path, 20 | config_overrides: List[Override], 21 | ) -> None: 22 | # This is the default configuration with 2 wasm/VM providers 23 | goth_config = load_yaml(goth_config_path, config_overrides) 24 | test_script_path = str(Path(__file__).parent / "requestor.py") 25 | 26 | configure_logging(log_dir) 27 | 28 | runner = Runner( 29 | base_log_dir=log_dir, 30 | compose_config=goth_config.compose_config, 31 | ) 32 | 33 | async with runner(goth_config.containers): 34 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 35 | 36 | async with requestor.run_command_on_host(test_script_path, env=os.environ) as ( 37 | _cmd_task, 38 | cmd_monitor, 39 | _process_monitor, 40 | ): 41 | await cmd_monitor.wait_for_pattern(r"\[.+\] Renegotiating", timeout=50) 42 | await cmd_monitor.wait_for_pattern(r"agreement.terminate\(\): True", timeout=50) 43 | # assert not "Main timeout triggered :(" 44 | await cmd_monitor.wait_for_pattern(r"All done", timeout=50) 45 | -------------------------------------------------------------------------------- /tests/goth_tests/test_run_blender.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from pathlib import Path 4 | from typing import List 5 | 6 | import pytest 7 | 8 | from goth.assertions import EventStream 9 | from goth.configuration import Override, load_yaml 10 | from goth.runner import Runner 11 | from goth.runner.log import configure_logging 12 | from goth.runner.probe import RequestorProbe 13 | 14 | from .assertions import assert_all_invoices_accepted, assert_no_errors, assert_tasks_processed 15 | 16 | logger = logging.getLogger("goth.test.run_blender") 17 | 18 | ALL_TASKS = {"0", "10", "20", "30", "40", "50"} 19 | 20 | 21 | # Temporal assertions expressing properties of sequences of "events". In this case, each "event" 22 | # is just a line of output from `blender.py`. 23 | 24 | 25 | async def assert_all_tasks_started(output_lines: EventStream[str]): 26 | """Assert that for every task a line with `Task started on provider` will appear.""" 27 | await assert_tasks_processed(ALL_TASKS, "started on provider", output_lines) 28 | 29 | 30 | async def assert_all_tasks_computed(output_lines: EventStream[str]): 31 | """Assert that for every task a line with `Task computed by provider` will appear.""" 32 | await assert_tasks_processed(ALL_TASKS, "finished by provider", output_lines) 33 | 34 | 35 | @pytest.mark.asyncio 36 | async def test_run_blender( 37 | project_dir: Path, log_dir: Path, goth_config_path: Path, config_overrides: List[Override] 38 | ) -> None: 39 | # This is the default configuration with 2 wasm/VM providers 40 | goth_config = load_yaml(goth_config_path, config_overrides) 41 | 42 | blender_path = project_dir / "examples" / "blender" / "blender.py" 43 | 44 | configure_logging(log_dir) 45 | 46 | runner = Runner( 47 | base_log_dir=log_dir, 48 | compose_config=goth_config.compose_config, 49 | ) 50 | 51 | async with runner(goth_config.containers): 52 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 53 | 54 | async with requestor.run_command_on_host( 55 | f"{blender_path} --subnet-tag goth --min-cpu-threads 1", 56 | env=os.environ, 57 | ) as (_cmd_task, cmd_monitor, _process_monitor): 58 | # Add assertions to the command output monitor `cmd_monitor`: 59 | cmd_monitor.add_assertion(assert_no_errors) 60 | cmd_monitor.add_assertion(assert_all_invoices_accepted) 61 | all_sent = cmd_monitor.add_assertion(assert_all_tasks_started) 62 | all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) 63 | 64 | await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ", timeout=30) 65 | logger.info("Received proposals") 66 | 67 | await cmd_monitor.wait_for_pattern(".*Agreement proposed ", timeout=20) 68 | logger.info("Agreement proposed") 69 | 70 | await cmd_monitor.wait_for_pattern(".*Agreement confirmed ", timeout=20) 71 | logger.info("Agreement confirmed") 72 | 73 | await all_sent.wait_for_result(timeout=120) 74 | logger.info("All tasks sent") 75 | 76 | await all_computed.wait_for_result(timeout=120) 77 | logger.info("All tasks computed, waiting for Golem shutdown") 78 | 79 | await cmd_monitor.wait_for_pattern(".*Golem engine has shut down", timeout=120) 80 | 81 | logger.info("Requestor script finished") 82 | -------------------------------------------------------------------------------- /tests/goth_tests/test_run_scan.py: -------------------------------------------------------------------------------- 1 | """An integration test scenario that runs the `scan` example.""" 2 | 3 | import asyncio 4 | import logging 5 | import os 6 | import re 7 | import signal 8 | from pathlib import Path 9 | from typing import List 10 | 11 | import pytest 12 | 13 | from goth.configuration import Override, load_yaml 14 | from goth.runner import Runner 15 | from goth.runner.log import configure_logging 16 | from goth.runner.probe import RequestorProbe 17 | 18 | from .assertions import assert_all_invoices_accepted, assert_no_errors 19 | 20 | logger = logging.getLogger("goth.test.run_test") 21 | 22 | SUBNET_TAG = "goth" 23 | 24 | 25 | @pytest.mark.asyncio 26 | async def test_run_scan( 27 | log_dir: Path, 28 | project_dir: Path, 29 | goth_config_path: Path, 30 | config_overrides: List[Override], 31 | ) -> None: 32 | configure_logging(log_dir) 33 | 34 | # This is the default configuration with 2 wasm/VM providers 35 | goth_config = load_yaml(goth_config_path, config_overrides) 36 | 37 | requestor_path = project_dir / "examples" / "scan" / "scan.py" 38 | 39 | runner = Runner( 40 | base_log_dir=log_dir, 41 | compose_config=goth_config.compose_config, 42 | ) 43 | 44 | async with runner(goth_config.containers): 45 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 46 | 47 | async with requestor.run_command_on_host( 48 | f"{requestor_path} --subnet-tag {SUBNET_TAG} --scan-size 3", 49 | env=os.environ, 50 | ) as (_cmd_task, cmd_monitor, process_monitor): 51 | cmd_monitor.add_assertion(assert_no_errors) 52 | cmd_monitor.add_assertion(assert_all_invoices_accepted) 53 | 54 | # ensure this line is produced twice with a differing provider and task info: 55 | # Task finished by provider 'provider-N', task data: M 56 | 57 | providers = set() 58 | tasks = set() 59 | 60 | for i in range(2): 61 | output = await cmd_monitor.wait_for_pattern( 62 | ".*Task finished by provider", timeout=120 63 | ) 64 | matches = re.match(r".*by provider 'provider-(\d)', task data: (\d)", output) 65 | providers.add(matches.group(1)) 66 | tasks.add(matches.group(2)) 67 | 68 | assert providers == {"1", "2"} 69 | assert tasks == {"0", "1"} 70 | logger.info("Scanner tasks completed for the two providers in the network.") 71 | 72 | # ensure no more tasks are executed by the two providers 73 | logger.info("Waiting to see if another task gets started...") 74 | await asyncio.sleep(30) 75 | 76 | tasks_finished = [ 77 | e for e in cmd_monitor._events if re.match(".*Task finished by provider", e) 78 | ] 79 | 80 | assert len(tasks_finished) == 2 81 | logger.info("As expected, no more tasks started. Issuing a break...") 82 | 83 | proc: asyncio.subprocess.Process = await process_monitor.get_process() 84 | proc.send_signal(signal.SIGINT) 85 | 86 | logger.info("SIGINT sent...") 87 | 88 | await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) 89 | logger.info("Requestor script finished.") 90 | -------------------------------------------------------------------------------- /tests/goth_tests/test_run_simple_service.py: -------------------------------------------------------------------------------- 1 | """An integration test scenario that runs the Simple Service example requestor app.""" 2 | 3 | import logging 4 | import os 5 | import time 6 | from pathlib import Path 7 | from typing import List 8 | 9 | import pytest 10 | 11 | from goth.configuration import Override, load_yaml 12 | from goth.runner import Runner 13 | from goth.runner.log import configure_logging 14 | from goth.runner.probe import RequestorProbe 15 | 16 | from .assertions import assert_all_invoices_accepted, assert_no_errors 17 | 18 | logger = logging.getLogger("goth.test.run_simple_service") 19 | 20 | RUNNING_TIME = 40 # in seconds 21 | SUBNET_TAG = "goth" 22 | 23 | 24 | @pytest.mark.asyncio 25 | async def test_run_simple_service( 26 | log_dir: Path, 27 | project_dir: Path, 28 | goth_config_path: Path, 29 | config_overrides: List[Override], 30 | ) -> None: 31 | configure_logging(log_dir) 32 | 33 | # This is the default configuration with 2 wasm/VM providers 34 | goth_config = load_yaml(goth_config_path, config_overrides) 35 | 36 | requestor_path = project_dir / "examples" / "simple-service-poc" / "simple_service.py" 37 | 38 | runner = Runner( 39 | base_log_dir=log_dir, 40 | compose_config=goth_config.compose_config, 41 | ) 42 | 43 | async with runner(goth_config.containers): 44 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 45 | 46 | async with requestor.run_command_on_host( 47 | f"{requestor_path} --running-time {RUNNING_TIME} --subnet-tag {SUBNET_TAG}", 48 | env=os.environ, 49 | ) as (_cmd_task, cmd_monitor, _process_monitor): 50 | start_time = time.time() 51 | 52 | def elapsed_time(): 53 | return f"time: {(time.time() - start_time):.1f}" 54 | 55 | cmd_monitor.add_assertion(assert_no_errors) 56 | cmd_monitor.add_assertion(assert_all_invoices_accepted) 57 | 58 | await cmd_monitor.wait_for_pattern( 59 | "Starting Cluster 1: 1x\\[Service: SimpleService", timeout=20 60 | ) 61 | # A longer timeout to account for downloading a VM image 62 | await cmd_monitor.wait_for_pattern("All instances started", timeout=120) 63 | logger.info(f"The instance was started successfully ({elapsed_time()})") 64 | 65 | for _ in range(3): 66 | await cmd_monitor.wait_for_pattern("instances:.*running", timeout=20) 67 | logger.info("The instance is running") 68 | 69 | await cmd_monitor.wait_for_pattern( 70 | "Stopping Cluster 1: 1x\\[Service: SimpleService", timeout=60 71 | ) 72 | logger.info(f"The instance is stopping ({elapsed_time()})") 73 | 74 | await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) 75 | logger.info(f"Requestor script finished ({elapsed_time()})") 76 | -------------------------------------------------------------------------------- /tests/goth_tests/test_run_webapp.py: -------------------------------------------------------------------------------- 1 | """An integration test scenario that runs the `webapp` example.""" 2 | 3 | import asyncio 4 | import logging 5 | import os 6 | import signal 7 | import time 8 | from pathlib import Path 9 | from typing import List 10 | 11 | import pytest 12 | import requests 13 | 14 | from goth.configuration import Override, load_yaml 15 | from goth.runner import Runner 16 | from goth.runner.log import configure_logging 17 | from goth.runner.probe import RequestorProbe 18 | 19 | from ._util import get_free_port 20 | from .assertions import assert_all_invoices_accepted, assert_no_errors 21 | 22 | logger = logging.getLogger("goth.test") 23 | 24 | SUBNET_TAG = "goth" 25 | 26 | ONELINER_ENTRY = "hello from goth" 27 | 28 | port = get_free_port() 29 | 30 | ONELINER_URL = f"http://localhost:{port}/" 31 | 32 | 33 | @pytest.mark.asyncio 34 | async def test_run_webapp( 35 | log_dir: Path, 36 | project_dir: Path, 37 | goth_config_path: Path, 38 | config_overrides: List[Override], 39 | ) -> None: 40 | configure_logging(log_dir) 41 | 42 | # This is the default configuration with 2 wasm/VM providers 43 | goth_config = load_yaml(goth_config_path, config_overrides) 44 | 45 | # disable the mitm proxy used to capture the requestor agent -> daemon API calls 46 | # because it doesn't support websockets which are needed by the VPN (and the Local HTTP Proxy) 47 | requestor = [c for c in goth_config.containers if c.name == "requestor"][0] 48 | requestor.use_proxy = False 49 | 50 | requestor_path = project_dir / "examples" / "webapp" / "webapp.py" 51 | 52 | runner = Runner( 53 | base_log_dir=log_dir, 54 | compose_config=goth_config.compose_config, 55 | ) 56 | 57 | async with runner(goth_config.containers): 58 | requestor = runner.get_probes(probe_type=RequestorProbe)[0] 59 | 60 | async with requestor.run_command_on_host( 61 | f"{requestor_path} --subnet-tag {SUBNET_TAG} --port {port}", 62 | env=os.environ, 63 | ) as (_cmd_task, cmd_monitor, process_monitor): 64 | start_time = time.time() 65 | 66 | def elapsed_time(): 67 | return f"time: {(time.time() - start_time):.1f}" 68 | 69 | cmd_monitor.add_assertion(assert_no_errors) 70 | cmd_monitor.add_assertion(assert_all_invoices_accepted) 71 | 72 | logger.info("Waiting for the instances to start") 73 | 74 | # A longer timeout to account for downloading a VM image 75 | await cmd_monitor.wait_for_pattern("DB instance started.*", timeout=240) 76 | logger.info("Db instance started") 77 | 78 | await cmd_monitor.wait_for_pattern("Local HTTP server listening on.*", timeout=120) 79 | logger.info("HTTP instance started") 80 | 81 | requests.post(ONELINER_URL, data={"message": ONELINER_ENTRY}) 82 | r = requests.get(ONELINER_URL) 83 | assert r.status_code == 200 84 | assert ONELINER_ENTRY in r.text 85 | logger.info("DB write confirmed :)") 86 | 87 | proc: asyncio.subprocess.Process = await process_monitor.get_process() 88 | proc.send_signal(signal.SIGINT) 89 | logger.info("Sent SIGINT...") 90 | 91 | for i in range(2): 92 | await cmd_monitor.wait_for_pattern(".*Service terminated.*", timeout=20) 93 | 94 | logger.info(f"The instances have been terminated ({elapsed_time()})") 95 | 96 | await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) 97 | logger.info(f"Requestor script finished ({elapsed_time()})") 98 | -------------------------------------------------------------------------------- /tests/payload/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/payload/__init__.py -------------------------------------------------------------------------------- /tests/payload/test_payload.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dataclasses import dataclass 3 | 4 | from yapapi.payload import Payload 5 | from yapapi.props import constraint, inf, prop 6 | from yapapi.props.builder import DemandBuilder 7 | 8 | 9 | @dataclass 10 | class _FooPayload(Payload): 11 | port: int = prop("golem.srv.app.foo.port", None) 12 | 13 | runtime: str = constraint(inf.INF_RUNTIME_NAME, "=", "foo") 14 | min_mem_gib: float = constraint(inf.INF_MEM, ">=", 16) 15 | min_storage_gib: float = constraint(inf.INF_STORAGE, ">=", 1024) 16 | 17 | 18 | @pytest.mark.asyncio 19 | async def test_payload(): 20 | builder = DemandBuilder() 21 | await builder.decorate(_FooPayload(port=1234, min_mem_gib=32)) 22 | assert builder.properties == {"golem.srv.app.foo.port": 1234} 23 | assert ( 24 | builder.constraints 25 | == "(&(golem.runtime.name=foo)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024))" 26 | ) 27 | -------------------------------------------------------------------------------- /tests/payload/test_repo.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import AsyncMock 2 | 3 | import pytest 4 | 5 | from yapapi.payload import vm 6 | from yapapi.payload.package import PackageException 7 | 8 | _MOCK_HTTP_ADDR = "http://test.address/" 9 | _MOCK_HTTPS_ADDR = "https://test.address/" 10 | _MOCK_SHA3 = "abcdef124356789" 11 | _MOCK_SIZE = 2**24 12 | 13 | 14 | async def _mock_response(*args, **kwargs): 15 | mock = AsyncMock() 16 | mock.status = 200 17 | mock.json.return_value = { 18 | "http": _MOCK_HTTP_ADDR, 19 | "https": _MOCK_HTTPS_ADDR, 20 | "sha3": _MOCK_SHA3, 21 | "size": _MOCK_SIZE, 22 | } 23 | return mock 24 | 25 | 26 | @pytest.mark.parametrize( 27 | "image_hash, image_tag, image_url, image_use_https, " 28 | "expected_url, expected_error, expected_error_msg", 29 | ( 30 | ("testhash", None, None, False, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTP_ADDR}", None, ""), 31 | (None, "testtag", None, False, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTP_ADDR}", None, ""), 32 | ("testhash", None, None, True, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTPS_ADDR}", None, ""), 33 | (None, "testtag", None, True, f"hash:sha3:{_MOCK_SHA3}:{_MOCK_HTTPS_ADDR}", None, ""), 34 | ("testhash", None, "http://image", False, "hash:sha3:testhash:http://image", None, ""), 35 | ( 36 | None, 37 | None, 38 | None, 39 | False, 40 | None, 41 | PackageException, 42 | "Either an image_hash or an image_tag is required " 43 | "to resolve an image URL from the Golem Registry", 44 | ), 45 | ( 46 | None, 47 | None, 48 | "http://image", 49 | False, 50 | None, 51 | PackageException, 52 | "An image_hash is required when using a direct image_url", 53 | ), 54 | ( 55 | None, 56 | "testtag", 57 | "http://image", 58 | False, 59 | None, 60 | PackageException, 61 | "An image_tag can only be used when resolving " 62 | "from Golem Registry, not with a direct image_url", 63 | ), 64 | ( 65 | "testhash", 66 | "testtag", 67 | None, 68 | False, 69 | None, 70 | PackageException, 71 | "Golem Registry images can be resolved by either " 72 | "an image_hash or by an image_tag but not both", 73 | ), 74 | ), 75 | ) 76 | @pytest.mark.asyncio 77 | async def test_repo( 78 | monkeypatch, 79 | image_hash, 80 | image_tag, 81 | image_url, 82 | image_use_https, 83 | expected_url, 84 | expected_error, 85 | expected_error_msg, 86 | ): 87 | monkeypatch.setattr("aiohttp.ClientSession.get", _mock_response) 88 | monkeypatch.setattr("aiohttp.ClientSession.head", _mock_response) 89 | 90 | package_awaitable = vm.repo( 91 | image_hash=image_hash, 92 | image_tag=image_tag, 93 | image_url=image_url, 94 | image_use_https=image_use_https, 95 | ) 96 | 97 | if expected_error: 98 | with pytest.raises(expected_error) as e: 99 | _ = await package_awaitable 100 | assert expected_error_msg in str(e) 101 | else: 102 | package = await package_awaitable 103 | url = await package.resolve_url() 104 | assert url == expected_url 105 | -------------------------------------------------------------------------------- /tests/props/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/props/__init__.py -------------------------------------------------------------------------------- /tests/props/test_builder.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dataclasses import dataclass 3 | 4 | from yapapi.props import constraint, prop 5 | from yapapi.props.builder import AutodecoratingModel, DemandBuilder 6 | 7 | 8 | @pytest.mark.asyncio 9 | async def test_autodecorating_model(): 10 | @dataclass 11 | class Foo(AutodecoratingModel): 12 | bar: str = prop("some.bar") 13 | max_baz: int = constraint("baz", "<=", 100) 14 | 15 | foo = Foo(bar="a nice one", max_baz=50) 16 | demand = DemandBuilder() 17 | await foo.decorate_demand(demand) 18 | assert demand.properties == {"some.bar": "a nice one"} 19 | assert demand.constraints == "(baz<=50)" 20 | 21 | 22 | def test_add_properties(): 23 | demand = DemandBuilder() 24 | assert demand.properties == {} 25 | demand.add_properties({"golem.foo": 667}) 26 | demand.add_properties({"golem.bar": "blah"}) 27 | assert demand.properties == { 28 | "golem.foo": 667, 29 | "golem.bar": "blah", 30 | } 31 | -------------------------------------------------------------------------------- /tests/props/test_com.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from tests.factories.props.com import ComLinearFactory 4 | from yapapi.props.com import ComLinear, Counter 5 | 6 | LINEAR_COEFFS = [0.1, 0.2, 0.3] 7 | DEFINED_USAGES = [Counter.CPU.value, Counter.TIME.value] 8 | 9 | 10 | def test_com_linear_fixed_price(): 11 | com: ComLinear = ComLinearFactory(linear_coeffs=LINEAR_COEFFS) 12 | assert com.fixed_price == LINEAR_COEFFS[-1] 13 | 14 | 15 | def test_com_linear_price_for(): 16 | com: ComLinear = ComLinearFactory(linear_coeffs=LINEAR_COEFFS, usage_vector=DEFINED_USAGES) 17 | assert com.price_for[Counter.CPU.value] == LINEAR_COEFFS[0] 18 | assert com.price_for[Counter.TIME.value] == LINEAR_COEFFS[1] 19 | 20 | 21 | @pytest.mark.parametrize( 22 | "usage, cost", 23 | [ 24 | ( 25 | [0.0, 0.0], 26 | LINEAR_COEFFS[-1], 27 | ), 28 | ( 29 | [2.0, 0.0], 30 | LINEAR_COEFFS[0] * 2.0 + LINEAR_COEFFS[-1], 31 | ), 32 | ( 33 | [0.0, 2.0], 34 | LINEAR_COEFFS[1] * 2.0 + LINEAR_COEFFS[-1], 35 | ), 36 | ( 37 | [3.0, 5.0], 38 | LINEAR_COEFFS[0] * 3.0 + LINEAR_COEFFS[1] * 5.0 + LINEAR_COEFFS[-1], 39 | ), 40 | ], 41 | ) 42 | def test_com_linear_calculate_cost(usage, cost): 43 | com: ComLinear = ComLinearFactory(linear_coeffs=LINEAR_COEFFS, usage_vector=DEFINED_USAGES) 44 | assert com.calculate_cost(usage) == cost 45 | -------------------------------------------------------------------------------- /tests/rest/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/rest/__init__.py -------------------------------------------------------------------------------- /tests/rest/test_allocation.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | import pytest 4 | 5 | from yapapi.config import ApiConfig 6 | from yapapi.rest import Configuration, Payment 7 | 8 | 9 | @pytest.fixture 10 | async def yapapi_payment(request): 11 | conf = Configuration(api_config=ApiConfig(app_key=request.config.getvalue("yaApiKey"))) 12 | async with conf.payment() as p: 13 | yield Payment(p) 14 | 15 | 16 | @pytest.mark.skipif("not config.getvalue('yaApiKey')") 17 | @pytest.mark.asyncio 18 | async def test_allocation(yapapi_payment: Payment): 19 | async for a in yapapi_payment.allocations(): 20 | print("a=", a) 21 | 22 | async with yapapi_payment.new_allocation(Decimal(40), "NGNT", "mockaddress") as allocation: 23 | found = False 24 | async for a in yapapi_payment.allocations(): 25 | if a.id == allocation.id: 26 | found = True 27 | break 28 | assert found 29 | -------------------------------------------------------------------------------- /tests/rest/test_demand_builder.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta, timezone 2 | 3 | from yapapi import props 4 | from yapapi.payload import vm 5 | from yapapi.props.builder import DemandBuilder 6 | 7 | 8 | def test_builder(): 9 | print("inf.cores=", vm.InfVmKeys.names()) 10 | b = DemandBuilder() 11 | e = datetime.now(timezone.utc) + timedelta(days=1) 12 | b.add(props.Activity(expiration=e)) 13 | b.add(vm.VmRequest(package_url="", package_format=vm.VmPackageFormat.GVMKIT_SQUASH)) 14 | print(b) 15 | -------------------------------------------------------------------------------- /tests/script/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/script/__init__.py -------------------------------------------------------------------------------- /tests/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/services/__init__.py -------------------------------------------------------------------------------- /tests/services/test_service_runner.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Optional 3 | from unittest import mock 4 | 5 | import pytest 6 | from statemachine import State 7 | 8 | from ya_activity.exceptions import ApiException 9 | 10 | from yapapi.ctx import WorkContext 11 | from yapapi.services.service import Service, ServiceState 12 | from yapapi.services.service_runner import ServiceRunner, ServiceRunnerError 13 | 14 | 15 | def mock_service(init_state: Optional[State] = None): 16 | service = Service() 17 | if init_state: 18 | service.service_instance.service_state = ServiceState(start_value=init_state.name) 19 | service._ctx = WorkContext(mock.AsyncMock(), mock.Mock(), mock.Mock(), mock.Mock()) 20 | return service 21 | 22 | 23 | @pytest.mark.asyncio 24 | async def test_ensure_alive_no_interval(): 25 | with mock.patch("asyncio.Future", mock.AsyncMock()) as future: 26 | service_runner = ServiceRunner(mock.Mock(), health_check_interval=None) 27 | await service_runner._ensure_alive(mock.AsyncMock()) 28 | 29 | future.assert_awaited() 30 | 31 | 32 | @pytest.mark.parametrize( 33 | "service_state, side_effect, num_retries, expected_alive, expected_num_calls", 34 | ( 35 | (ServiceState.pending, None, None, True, 0), 36 | (ServiceState.running, None, None, True, 10), 37 | (ServiceState.running, ApiException(), None, False, 3), 38 | (ServiceState.running, ApiException(), 2, False, 2), 39 | (ServiceState.running, ApiException(), 5, False, 5), 40 | ), 41 | ) 42 | @pytest.mark.asyncio 43 | async def test_ensure_alive( 44 | service_state, 45 | side_effect, 46 | num_retries, 47 | expected_alive, 48 | expected_num_calls, 49 | ): 50 | service = mock_service(service_state) 51 | with mock.patch( 52 | "yapapi.ctx.WorkContext.get_raw_state", mock.AsyncMock(side_effect=side_effect) 53 | ) as grs_mock: 54 | service_runner = ServiceRunner( 55 | mock.Mock(), 56 | health_check_interval=0.001, 57 | **({"health_check_retries": num_retries} if num_retries else {}), 58 | ) 59 | 60 | loop = asyncio.get_event_loop() 61 | ensure_alive = loop.create_task(service_runner._ensure_alive(service)) 62 | sentinel = loop.create_task(asyncio.sleep(0.1)) 63 | 64 | done, pending = await asyncio.wait( 65 | ( 66 | ensure_alive, 67 | sentinel, 68 | ), 69 | return_when=asyncio.FIRST_COMPLETED, 70 | ) 71 | 72 | if expected_alive: 73 | assert ensure_alive in pending 74 | else: 75 | assert ensure_alive in done 76 | 77 | with pytest.raises(ServiceRunnerError): 78 | ensure_alive.result() 79 | 80 | sentinel.cancel() 81 | ensure_alive.cancel() 82 | 83 | assert len(grs_mock.mock_calls) >= expected_num_calls 84 | -------------------------------------------------------------------------------- /tests/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/storage/__init__.py -------------------------------------------------------------------------------- /tests/storage/test_storage.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from yapapi.storage import Content, Destination 4 | 5 | 6 | class _TestDestination(Destination): 7 | def __init__(self, test_data: bytes): 8 | self._test_data = test_data 9 | 10 | def upload_url(self): 11 | return "" 12 | 13 | async def download_stream(self) -> Content: 14 | async def data(): 15 | for c in [self._test_data]: 16 | yield self._test_data 17 | 18 | return Content(len(self._test_data), data()) 19 | 20 | 21 | @pytest.mark.asyncio 22 | async def test_download_bytes(): 23 | expected = b"some test data" 24 | destination = _TestDestination(expected) 25 | assert await destination.download_bytes() == expected 26 | 27 | 28 | @pytest.mark.asyncio 29 | async def test_download_bytes_with_limit(): 30 | destination = _TestDestination(b"some test data") 31 | assert await destination.download_bytes(limit=4) == b"some" 32 | -------------------------------------------------------------------------------- /tests/strategy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/tests/strategy/__init__.py -------------------------------------------------------------------------------- /tests/strategy/helpers.py: -------------------------------------------------------------------------------- 1 | from yapapi.strategy import MarketStrategy 2 | 3 | DEFAULT_OFFER_SCORE = 77.51937984496124 # this matches the OfferProposalFactory default 4 | 5 | 6 | class Always6(MarketStrategy): 7 | async def score_offer(self, offer): 8 | return 6 9 | -------------------------------------------------------------------------------- /tests/strategy/test_decrease_score_for_unconfirmed.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from tests.factories.events import AgreementConfirmedFactory as AgreementConfirmed 6 | from tests.factories.events import AgreementRejectedFactory as AgreementRejected 7 | from tests.factories.rest.market import OfferProposalFactory 8 | from yapapi import Golem 9 | from yapapi.strategy import DecreaseScoreForUnconfirmedAgreement 10 | 11 | from .helpers import DEFAULT_OFFER_SCORE, Always6 12 | 13 | # (events, providers_with_decreased_scores) 14 | sample_data = ( 15 | ((), ()), 16 | (((AgreementRejected, 1),), (1,)), 17 | (((AgreementRejected, 2),), (2,)), 18 | (((AgreementConfirmed, 1),), ()), 19 | (((AgreementRejected, 1), (AgreementConfirmed, 1)), ()), 20 | (((AgreementRejected, 2), (AgreementConfirmed, 1)), (2,)), 21 | (((AgreementRejected, 1), (AgreementConfirmed, 2)), (1,)), 22 | (((AgreementRejected, 1), (AgreementRejected, 1)), (1,)), 23 | (((AgreementRejected, 1), (AgreementConfirmed, 1), (AgreementRejected, 1)), (1,)), 24 | ) 25 | 26 | 27 | @pytest.mark.asyncio 28 | async def test_6(): 29 | strategy = DecreaseScoreForUnconfirmedAgreement(Always6(), 0.5) 30 | offer = OfferProposalFactory() 31 | assert 6 == await strategy.score_offer(offer) 32 | 33 | 34 | @pytest.mark.asyncio 35 | @pytest.mark.parametrize("events_def, decreased_providers", sample_data) 36 | async def test_decrease_score_for(events_def, decreased_providers): 37 | """Test if DecreaseScoreForUnconfirmedAgreement works as expected.""" 38 | strategy = DecreaseScoreForUnconfirmedAgreement(Always6(), 0.5) 39 | 40 | for event_cls, event_provider_id in events_def: 41 | event = event_cls(agreement__provider_id=event_provider_id) 42 | strategy.on_event(event) 43 | 44 | for provider_id in (1, 2): 45 | offer = OfferProposalFactory(provider_id=provider_id) 46 | expected_score = 3 if provider_id in decreased_providers else 6 47 | assert expected_score == await strategy.score_offer(offer) 48 | 49 | 50 | def empty_event_consumer(event): 51 | """To silience the default logger - it doesn't work with mocked events.""" 52 | 53 | 54 | @pytest.mark.asyncio 55 | @pytest.mark.parametrize("events_def, decreased_providers", sample_data) 56 | async def test_full_DSFUA_workflow(dummy_yagna_engine, events_def, decreased_providers): 57 | """Test if DecreaseScoreForUnconfirmedAgreement is correctly initialized as a default strategy \ 58 | that is - if events emitted by the engine reach the event consumer of the default strategy.""" 59 | 60 | golem = Golem(budget=1, event_consumer=empty_event_consumer, app_key="NOT_A_REAL_APPKEY") 61 | async with golem: 62 | for event_cls, event_provider_id in events_def: 63 | event = event_cls(agreement__provider_id=event_provider_id) 64 | golem._engine._emit_event(event) 65 | 66 | await asyncio.sleep(0.1) # let the events propagate 67 | 68 | for provider_id in (1, 2): 69 | offer = OfferProposalFactory(provider_id=provider_id) 70 | expected_score = ( 71 | DEFAULT_OFFER_SCORE / 2 72 | if provider_id in decreased_providers 73 | else DEFAULT_OFFER_SCORE 74 | ) 75 | assert expected_score == await golem._engine._strategy.score_offer(offer) 76 | -------------------------------------------------------------------------------- /tests/strategy/test_provider_filter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from tests.factories.rest.market import OfferProposalFactory 4 | from yapapi import Golem 5 | from yapapi.contrib.strategy import ProviderFilter 6 | from yapapi.strategy import SCORE_REJECTED 7 | 8 | from .helpers import DEFAULT_OFFER_SCORE, Always6 9 | 10 | 11 | @pytest.mark.asyncio 12 | @pytest.mark.parametrize( 13 | "bad_providers", 14 | ((), (1,), (2,), (1, 2)), 15 | ) 16 | async def test_restricted_providers(bad_providers): 17 | """Test if the strategy restricts correct providers.""" 18 | 19 | strategy = ProviderFilter(Always6(), lambda provider_id: provider_id not in bad_providers) 20 | 21 | for provider_id in (1, 2, 3): 22 | offer = OfferProposalFactory(provider_id=provider_id) 23 | expected_score = SCORE_REJECTED if provider_id in bad_providers else 6 24 | assert expected_score == await strategy.score_offer(offer) 25 | 26 | 27 | @pytest.mark.asyncio 28 | async def test_dynamic_change(): 29 | """Test if changes in the is_allowed function are reflected in scores.""" 30 | 31 | something_happened = False 32 | 33 | def is_allowed(provider_id): 34 | if something_happened: 35 | return False 36 | return True 37 | 38 | strategy = ProviderFilter(Always6(), is_allowed) 39 | 40 | for i in range(0, 5): 41 | offer = OfferProposalFactory(provider_id=1) 42 | expected_score = SCORE_REJECTED if i > 3 else 6 43 | assert expected_score == await strategy.score_offer(offer) 44 | 45 | if i == 3: 46 | something_happened = True 47 | 48 | 49 | @pytest.mark.asyncio 50 | async def test_default_strategy_update(dummy_yagna_engine): 51 | """Test if default strategy extended with ProviderFilter works as expected.""" 52 | 53 | golem = Golem(budget=1, app_key="NOT_A_REAL_APPKEY") 54 | golem.strategy = ProviderFilter(golem.strategy, lambda provider_id: provider_id == 2) 55 | 56 | async with golem: 57 | for provider_id in (1, 2, 3): 58 | offer = OfferProposalFactory(provider_id=provider_id) 59 | expected_score = DEFAULT_OFFER_SCORE if provider_id == 2 else SCORE_REJECTED 60 | assert expected_score == await golem._engine._strategy.score_offer(offer) 61 | -------------------------------------------------------------------------------- /tests/strategy/test_wrapping_strategy.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from yapapi.strategy import WrappingMarketStrategy 4 | 5 | 6 | class MyBaseStrategy(mock.Mock): 7 | some_value = 123 8 | some_other_value = 456 9 | 10 | 11 | class MyStrategy(WrappingMarketStrategy): 12 | some_value = 789 13 | 14 | def some_method(self): 15 | return True 16 | 17 | 18 | def test_wrapping_strategy(): 19 | base_strategy = MyBaseStrategy() 20 | strategy = MyStrategy(base_strategy) 21 | 22 | assert strategy.some_value == MyStrategy.some_value 23 | assert strategy.some_other_value == MyBaseStrategy.some_other_value 24 | 25 | strategy.some_method() 26 | assert not base_strategy.some_method.called 27 | 28 | strategy.some_other_method() 29 | assert base_strategy.some_other_method.called 30 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | import yapapi.config 6 | 7 | 8 | def test_api_config_missing_configuration_error(purge_yagna_os_env): 9 | with pytest.raises(yapapi.config.MissingConfiguration): 10 | yapapi.config.ApiConfig() 11 | 12 | 13 | def test_api_config_explicit(purge_yagna_os_env): 14 | given_config_kwargs = { 15 | "app_key": "yagna-app-key", 16 | "api_url": "yagna-api_url", 17 | "market_url": "yagna-market_url", 18 | "payment_url": "yagna-payment_url", 19 | "net_url": "yagna-net_url", 20 | "activity_url": "yagna-activity_url", 21 | } 22 | received_config = yapapi.config.ApiConfig(**given_config_kwargs) 23 | for key, value in given_config_kwargs.items(): 24 | assert getattr(received_config, key) == value 25 | 26 | 27 | def test_api_config_only_app_key(purge_yagna_os_env): 28 | os.environ["YAGNA_APPKEY"] = "yagna-app-key" 29 | config = yapapi.config.ApiConfig() 30 | assert config.app_key == "yagna-app-key" 31 | assert config.api_url is not None 32 | assert config.market_url is None 33 | assert config.payment_url is None 34 | assert config.net_url is None 35 | assert config.activity_url is None 36 | 37 | 38 | def test_api_config_from_env(purge_yagna_os_env): 39 | given_config_vars = { 40 | "app_key": "YAGNA_APPKEY", 41 | "api_url": "YAGNA_API_URL", 42 | "market_url": "YAGNA_MARKET_URL", 43 | "payment_url": "YAGNA_PAYMENT_URL", 44 | "net_url": "YAGNA_NET_URL", 45 | "activity_url": "YAGNA_ACTIVITY_URL", 46 | } 47 | for _, env_var in given_config_vars.items(): 48 | os.environ[env_var] = env_var 49 | config = yapapi.config.ApiConfig() 50 | for key, value in given_config_vars.items(): 51 | assert getattr(config, key) == value 52 | -------------------------------------------------------------------------------- /tests/test_log.py: -------------------------------------------------------------------------------- 1 | """Unit tests for the `yapapi.log` module.""" 2 | 3 | import logging 4 | import os 5 | import sys 6 | import tempfile 7 | 8 | from yapapi.events import JobFinished 9 | from yapapi.log import enable_default_logger, get_logger, log_event, log_event_repr 10 | 11 | 12 | def test_log_file_encoding(capsys): 13 | """Test that logging some fancy Unicode to file does not cause encoding errors. 14 | 15 | `capsys` is a `pytest` fixture for capturing and accessing stdout/stderr, see 16 | https://docs.pytest.org/en/stable/capture.html#accessing-captured-output-from-a-test-function. 17 | """ 18 | 19 | # We have to close the temporary file before it can be re-opened by the logging handler 20 | # in Windows, hence we set `delete=False`. 21 | tmp_file = tempfile.NamedTemporaryFile(delete=False) 22 | try: 23 | tmp_file.close() 24 | 25 | enable_default_logger(log_file=tmp_file.name) 26 | logger = logging.getLogger("yapapi") 27 | logger.debug("| (• ◡•)| It's Adventure Time! (❍ᴥ❍ʋ)") 28 | for handler in logger.handlers: 29 | if isinstance(handler, logging.FileHandler): 30 | if handler.baseFilename == tmp_file.name: 31 | handler.close() 32 | 33 | err = capsys.readouterr().err 34 | assert "UnicodeEncodeError" not in err 35 | finally: 36 | os.unlink(tmp_file.name) 37 | 38 | 39 | def test_log_event_emit_traceback(): 40 | """Test that `log.log_event()` can emit logs for events containing tracebacks arguments.""" 41 | 42 | try: 43 | raise Exception("Hello!") 44 | except Exception: 45 | log_event(JobFinished(exc_info=sys.exc_info(), job="42")) 46 | 47 | 48 | def test_log_event_repr_emit_traceback(): 49 | """Test that `log.log_event_repr()` can emit logs for events containing traceback arguments.""" 50 | 51 | try: 52 | raise Exception("Hello!") 53 | except Exception: 54 | log_event_repr(JobFinished(exc_info=sys.exc_info(), job="42")) 55 | 56 | 57 | def test_get_logger_job_id(capsys): 58 | """Test that loggers created by `yapapi.log.get_logger` include job_id in messages.""" 59 | 60 | job_id = "some-unique-job-id" 61 | logger = get_logger("yapapi.test") 62 | logger.info("Hello!", job_id=job_id) 63 | logs = capsys.readouterr().err 64 | assert job_id in logs 65 | 66 | 67 | def test_get_logger_caches(): 68 | """Test `yapapi.log.get_logger` caches the results (just like `logging.getLogger`).""" 69 | 70 | logger_1 = get_logger("yapapi.test") 71 | logger_2 = get_logger("yapapi.test") 72 | assert logger_1 is logger_2 73 | -------------------------------------------------------------------------------- /tests/test_payment_platforms.py: -------------------------------------------------------------------------------- 1 | """Unit tests for code that selects payment platforms based on driver/network specification.""" 2 | 3 | from unittest import mock 4 | 5 | import pytest 6 | 7 | from ya_payment import RequestorApi 8 | 9 | from tests.factories.golem import GolemFactory 10 | from yapapi.engine import DEFAULT_DRIVER, DEFAULT_NETWORK, MAINNET_TOKEN_NAME, TESTNET_TOKEN_NAME 11 | from yapapi.golem import Golem, _Engine 12 | 13 | 14 | @pytest.fixture(autouse=True) 15 | def _set_app_key(monkeypatch): 16 | monkeypatch.setenv("YAGNA_APPKEY", "mock-appkey") 17 | 18 | 19 | class _StopExecutor(Exception): 20 | """An exception raised to stop the test when reaching an expected checkpoint in executor.""" 21 | 22 | 23 | @pytest.fixture() 24 | def _mock_engine_id(monkeypatch): 25 | """Mock Engine `id`.""" 26 | 27 | async def _id(_): 28 | return 29 | 30 | monkeypatch.setattr( 31 | _Engine, 32 | "_id", 33 | _id, 34 | ) 35 | 36 | 37 | @pytest.fixture() 38 | def _mock_create_allocation(monkeypatch): 39 | """Make `RequestorApi.create_allocation()` stop the test.""" 40 | 41 | create_allocation_mock = mock.Mock(side_effect=_StopExecutor("create_allocation() called")) 42 | 43 | monkeypatch.setattr(RequestorApi, "create_allocation", create_allocation_mock) 44 | 45 | return create_allocation_mock 46 | 47 | 48 | @pytest.mark.asyncio 49 | async def test_default(_mock_engine_id, _mock_create_allocation): 50 | """Test the allocation defaults.""" 51 | 52 | with pytest.raises(_StopExecutor): 53 | async with GolemFactory(): 54 | pass 55 | 56 | assert _mock_create_allocation.called 57 | assert ( 58 | _mock_create_allocation.mock_calls[0][1][0].payment_platform 59 | == f"{DEFAULT_DRIVER}-{DEFAULT_NETWORK}-{TESTNET_TOKEN_NAME}" 60 | ) 61 | 62 | 63 | @pytest.mark.asyncio 64 | async def test_mainnet(_mock_engine_id, _mock_create_allocation): 65 | """Test the allocation for a mainnet account.""" 66 | 67 | with pytest.raises(_StopExecutor): 68 | async with Golem(budget=10.0, payment_driver="somedriver", payment_network="mainnet"): 69 | pass 70 | 71 | assert _mock_create_allocation.called 72 | assert ( 73 | _mock_create_allocation.mock_calls[0][1][0].payment_platform 74 | == f"somedriver-mainnet-{MAINNET_TOKEN_NAME}" 75 | ) 76 | 77 | 78 | @pytest.mark.asyncio 79 | async def test_testnet(_mock_engine_id, _mock_create_allocation): 80 | """Test the allocation for a mainnet account.""" 81 | 82 | with pytest.raises(_StopExecutor): 83 | async with Golem(budget=10.0, payment_driver="somedriver", payment_network="othernet"): 84 | pass 85 | 86 | assert _mock_create_allocation.called 87 | assert ( 88 | _mock_create_allocation.mock_calls[0][1][0].payment_platform 89 | == f"somedriver-othernet-{TESTNET_TOKEN_NAME}" 90 | ) 91 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from yapapi.utils import explode_dict 2 | 3 | 4 | def test_explode_dict(): 5 | assert explode_dict( 6 | { 7 | "root_field": 1, 8 | "nested.field": 2, 9 | "nested.obj": { 10 | "works": "fine", 11 | }, 12 | "nested.obj.with_array": [ 13 | "okay!", 14 | ], 15 | "even.more.nested.field": 3, 16 | "arrays.0.are": { 17 | "supported": "too", 18 | }, 19 | "arrays.1": "works fine", 20 | } 21 | ) == { 22 | "root_field": 1, 23 | "nested": { 24 | "field": 2, 25 | "obj": { 26 | "works": "fine", 27 | "with_array": [ 28 | "okay!", 29 | ], 30 | }, 31 | }, 32 | "even": { 33 | "more": { 34 | "nested": { 35 | "field": 3, 36 | }, 37 | }, 38 | }, 39 | "arrays": [ 40 | { 41 | "are": { 42 | "supported": "too", 43 | }, 44 | }, 45 | "works fine", 46 | ], 47 | } 48 | -------------------------------------------------------------------------------- /tests/test_yapapi.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from pathlib import Path 3 | 4 | import toml 5 | 6 | import yapapi 7 | 8 | 9 | def test_version(): 10 | with open(Path(yapapi.__file__).parents[1] / "pyproject.toml") as f: 11 | pyproject = toml.loads(f.read()) 12 | 13 | assert yapapi.__version__ == pyproject["tool"]["poetry"]["version"] 14 | 15 | 16 | def test_windows_event_loop_fix(): 17 | async def _asyncio_test(): 18 | await asyncio.create_subprocess_shell("") 19 | 20 | yapapi.windows_event_loop_fix() 21 | 22 | loop = asyncio.get_event_loop() 23 | task = loop.create_task(_asyncio_test()) 24 | loop.run_until_complete(task) 25 | -------------------------------------------------------------------------------- /yapapi/__init__.py: -------------------------------------------------------------------------------- 1 | """Golem Python API.""" 2 | 3 | import asyncio 4 | import sys 5 | from pathlib import Path 6 | 7 | import toml 8 | from pkg_resources import get_distribution 9 | 10 | from yapapi.ctx import ExecOptions, WorkContext 11 | from yapapi.engine import NoPaymentAccountError 12 | from yapapi.executor import Executor, Task 13 | from yapapi.golem import Golem 14 | 15 | 16 | def get_version() -> str: 17 | """Return the version of the yapapi library package.""" 18 | pyproject_path = Path(__file__).parents[1] / "pyproject.toml" 19 | if pyproject_path.exists(): 20 | with open(pyproject_path) as f: 21 | pyproject = toml.loads(f.read()) 22 | 23 | return pyproject["tool"]["poetry"]["version"] 24 | 25 | return get_distribution("yapapi").version 26 | 27 | 28 | def windows_event_loop_fix(): 29 | """Set up asyncio to use ProactorEventLoop implementation for new event loops on Windows. 30 | 31 | This work-around is only needed for Python 3.6 and 3.7. 32 | With Python 3.8, `ProactorEventLoop` is already the default on Windows. 33 | """ 34 | 35 | if sys.platform == "win32" and sys.version_info < (3, 8): 36 | 37 | class _WindowsEventPolicy(asyncio.events.BaseDefaultEventLoopPolicy): 38 | _loop_factory = asyncio.windows_events.ProactorEventLoop 39 | 40 | asyncio.set_event_loop_policy(_WindowsEventPolicy()) 41 | 42 | 43 | __version__: str = get_version() 44 | __all__ = ( 45 | "Executor", 46 | "props", 47 | "rest", 48 | "executor", 49 | "storage", 50 | "Task", 51 | "WorkContext", 52 | "ExecOptions", 53 | "Golem", 54 | "NoPaymentAccountError", 55 | ) 56 | -------------------------------------------------------------------------------- /yapapi/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from functools import partial 3 | from typing import Optional 4 | 5 | from dataclasses import dataclass, field 6 | 7 | 8 | class MissingConfiguration(Exception): 9 | def __init__(self, key: str, description: str): 10 | self._key = key 11 | self._description = description 12 | 13 | def __str__(self): 14 | return f"Missing configuration for {self._description}. Please set env var {self._key}." 15 | 16 | 17 | @dataclass 18 | class ApiConfig: 19 | """Yagna low level API configuration. 20 | 21 | Attributes: 22 | app_key: Yagna application key. 23 | If not provided, the default is to get the value from `YAGNA_APPKEY` environment 24 | variable. 25 | If no value will be found MissingConfiguration error will be thrown 26 | api_url: base URL or all REST API URLs. Example value: http://127.0.10.10:7500 27 | (no trailing slash). 28 | Uses YAGNA_API_URL environment variable 29 | market_url: If not provided `api_url` will be used to construct it. 30 | Uses YAGNA_MARKET_URL environment variable 31 | payment_url: If not provided `api_url` will be used to construct it. 32 | Uses YAGNA_PAYMENT_URL environment variable 33 | net_url: Uses If not provided `api_url` will be used to construct it. 34 | YAGNA_NET_URL environment variable 35 | activity_url: If not provided `api_url` will be used to construct it. 36 | Uses YAGNA_ACTIVITY_URL environment variable 37 | """ 38 | 39 | app_key: str = field( # type: ignore[assignment] 40 | default_factory=partial(os.getenv, "YAGNA_APPKEY"), 41 | ) 42 | 43 | api_url: str = field( # type: ignore[assignment] 44 | default_factory=partial(os.getenv, "YAGNA_API_URL", "http://127.0.0.1:7465"), 45 | ) 46 | market_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_MARKET_URL")) 47 | payment_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_PAYMENT_URL")) 48 | net_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_NET_URL")) 49 | activity_url: Optional[str] = field(default_factory=partial(os.getenv, "YAGNA_ACTIVITY_URL")) 50 | 51 | def __post_init__(self): 52 | if self.app_key is None: 53 | raise MissingConfiguration(key="YAGNA_APPKEY", description="API authentication token") 54 | -------------------------------------------------------------------------------- /yapapi/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Useful, reusable pieces of code built on top of `yapapi`. 3 | 4 | This part of `yapapi` repository serves to provide requestor agent application authors with reusable 5 | components that we or third-party developers found useful while working on examples and apps that 6 | use our Python API. 7 | 8 | They're not strictly part of the high level API library itself and while we do intend to keep them 9 | in sync with the subsequent releases, they should be treated as experimental and their interface 10 | may change without a proper deprecation process. 11 | """ 12 | -------------------------------------------------------------------------------- /yapapi/contrib/service/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Final 2 | 3 | WEBSOCKET_CHUNK_LIMIT: Final[int] = 2**16 4 | DEFAULT_TIMEOUT: Final[float] = 300.0 5 | -------------------------------------------------------------------------------- /yapapi/contrib/service/chunk.py: -------------------------------------------------------------------------------- 1 | from typing import Generator, TypeVar 2 | 3 | BufferType = TypeVar("BufferType", bytes, memoryview) 4 | 5 | 6 | def chunks(data: BufferType, chunk_limit: int) -> Generator[BufferType, None, None]: 7 | """Split the input buffer into chunks of `chunk_limit` bytes.""" 8 | max_chunk, remainder = divmod(len(data), chunk_limit) 9 | for chunk in range(0, max_chunk + (1 if remainder else 0)): 10 | yield data[chunk * chunk_limit : (chunk + 1) * chunk_limit] 11 | -------------------------------------------------------------------------------- /yapapi/contrib/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | from .provider_filter import ProviderFilter 2 | 3 | __all__ = ("ProviderFilter",) 4 | -------------------------------------------------------------------------------- /yapapi/contrib/strategy/provider_filter.py: -------------------------------------------------------------------------------- 1 | """Provider Filter. 2 | 3 | Market strategy wrapper that enables easy exclusion of offers from certain providers using 4 | a simple boolean condition, while preserving correct scoring of the remaining offers by the 5 | base strategy. 6 | """ 7 | 8 | import inspect 9 | from typing import Awaitable, Callable, Union 10 | 11 | from yapapi.rest.market import OfferProposal 12 | from yapapi.strategy import SCORE_REJECTED, BaseMarketStrategy, WrappingMarketStrategy 13 | 14 | IsAllowedType = Union[ 15 | Callable[[str], bool], 16 | Callable[[str], Awaitable[bool]], 17 | ] 18 | 19 | 20 | class ProviderFilter(WrappingMarketStrategy): 21 | """ProviderFilter - extend a market strategy with a layer that excludes offers from certain \ 22 | issuers. 23 | 24 | :param base_strategy: a market strategy that will be used to score offers from allowed providers 25 | :param is_allowed: a callable that accepts provider_id as an argument and returns either a 26 | boolean, or a boolean-returning awaitable, determining if offers from this provider should 27 | be considered (that is: scored by the `base_strategy`) 28 | 29 | Example 1. Block selected providers:: 30 | 31 | bad_providers = ['bad_provider_1', 'bad_provider_2'] 32 | base_strategy = SomeStrategy(...) 33 | strategy = ProviderFilter(base_strategy, lambda provider_id: provider_id not in bad_providers) 34 | 35 | Example 2. Select providers using a database table:: 36 | 37 | # create an async database connection 38 | # (sync would also work, but could hurt `yapapi` overall performance) 39 | async_conn = ... 40 | 41 | async def is_allowed(provider_id): 42 | result = await async_conn.execute("SELECT 1 FROM allowed_providers WHERE provider_id = ?", provider_id) 43 | return bool(result.fetchall()) 44 | 45 | base_strategy = SomeStrategy() 46 | strategy = ProviderFilter(base_strategy, is_allowed) 47 | 48 | Example 3. Use the default strategy, but disable every provider that fails to create an activity:: 49 | 50 | from yapapi import events 51 | 52 | bad_providers = set() 53 | 54 | def denying_event_consumer(event: events.Event): 55 | if isinstance(event, events.ActivityCreateFailed): 56 | bad_providers.add(event.provider_id) 57 | 58 | golem = Golem(...) 59 | golem.strategy = ProviderFilter(golem.strategy, lambda provider_id: provider_id not in bad_providers) 60 | await golem.add_event_consumer(denying_event_consumer) 61 | 62 | async with golem: 63 | ... 64 | 65 | # NOTE: this will currently work only for **new** offers from the provider, because old offers are already 66 | # scored, this should improve in https://github.com/golemfactory/yapapi/issues/820 67 | """ # noqa: 501 68 | 69 | def __init__(self, base_strategy: BaseMarketStrategy, is_allowed: IsAllowedType): 70 | super().__init__(base_strategy) 71 | self._is_allowed = is_allowed 72 | 73 | async def score_offer(self, offer: OfferProposal) -> float: 74 | if inspect.iscoroutinefunction(self._is_allowed): 75 | allowed = await self._is_allowed(offer.issuer) 76 | else: 77 | allowed = self._is_allowed(offer.issuer) 78 | 79 | if not allowed: 80 | return SCORE_REJECTED 81 | 82 | return await self.base_strategy.score_offer(offer) 83 | -------------------------------------------------------------------------------- /yapapi/event_dispatcher.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Callable, Dict, Set, Type 3 | 4 | from yapapi import events 5 | from yapapi.utils import AsyncWrapper 6 | 7 | 8 | class AsyncEventDispatcher: 9 | def __init__(self): 10 | self._consumers: Dict[AsyncWrapper, Set[Type[events.Event]]] = {} 11 | 12 | def add_event_consumer( 13 | self, 14 | event_consumer: Callable[[events.Event], None], 15 | event_classes: Set[Type[events.Event]], 16 | start_consumer: bool, 17 | ): 18 | consumer = AsyncWrapper(event_consumer) 19 | if start_consumer: 20 | consumer.start() 21 | self._consumers[consumer] = event_classes 22 | 23 | def emit(self, event: events.Event): 24 | for consumer, event_classes in self._consumers.items(): 25 | if any(isinstance(event, event_cls) for event_cls in event_classes): 26 | consumer.async_call(event) 27 | 28 | def start(self): 29 | for consumer in self._consumers: 30 | if consumer.closed: 31 | consumer.start() 32 | 33 | async def stop(self): 34 | if self._consumers: 35 | await asyncio.gather(*(aw.stop() for aw in self._consumers)) 36 | -------------------------------------------------------------------------------- /yapapi/payload/__init__.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | from yapapi.props.builder import AutodecoratingModel 4 | 5 | 6 | class Payload(AutodecoratingModel, abc.ABC): 7 | r"""Base class for descriptions of the payload required by the requestor. 8 | 9 | example usage:: 10 | 11 | import asyncio 12 | 13 | from dataclasses import dataclass 14 | from yapapi.props.builder import DemandBuilder 15 | from yapapi.props.base import prop, constraint 16 | from yapapi.props import inf 17 | 18 | from yapapi.payload import Payload 19 | 20 | CUSTOM_RUNTIME_NAME = "my-runtime" 21 | CUSTOM_PROPERTY = "golem.srv.app.myprop" 22 | 23 | 24 | @dataclass 25 | class MyPayload(Payload): 26 | myprop: str = prop(CUSTOM_PROPERTY, default="myvalue") 27 | runtime: str = constraint(inf.INF_RUNTIME_NAME, default=CUSTOM_RUNTIME_NAME) 28 | min_mem_gib: float = constraint(inf.INF_MEM, operator=">=", default=16) 29 | min_storage_gib: float = constraint(inf.INF_STORAGE, operator=">=", default=1024) 30 | 31 | 32 | async def main(): 33 | builder = DemandBuilder() 34 | payload = MyPayload(myprop="othervalue", min_mem_gib=32) 35 | await builder.decorate(payload) 36 | print(builder) 37 | 38 | asyncio.run(main()) 39 | 40 | output:: 41 | 42 | {'properties': {'golem.srv.app.myprop': 'othervalue'}, 'constraints': ['(&(golem.runtime.name=my-runtime)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024))']} 43 | """ # noqa: E501 44 | -------------------------------------------------------------------------------- /yapapi/payload/package.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | from typing import Optional 4 | 5 | import aiohttp 6 | from dataclasses import dataclass 7 | 8 | from yapapi.payload import Payload 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class PackageException(Exception): 14 | """Exception raised on any problems related to the package repository.""" 15 | 16 | 17 | @dataclass 18 | class Package(Payload): 19 | """Description of a task package (e.g. a VM image) deployed on the provider nodes.""" 20 | 21 | @abc.abstractmethod 22 | async def resolve_url(self) -> str: 23 | """Return package URL.""" 24 | 25 | 26 | async def check_package_url(image_url: str, image_hash: str) -> str: 27 | async with aiohttp.ClientSession() as client: 28 | resp = await client.head(image_url, allow_redirects=True) 29 | if resp.status != 200: 30 | resp.raise_for_status() 31 | 32 | return f"hash:sha3:{image_hash}:{image_url}" 33 | 34 | 35 | def _sizeof_fmt(num, suffix="B"): 36 | for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: 37 | if abs(num) < 1024.0: 38 | return f"{num:3.2f}{unit}{suffix}" 39 | num /= 1024.0 40 | return f"{num:.1f}Yi{suffix}" 41 | 42 | 43 | async def resolve_package_url( 44 | repo_url: str, 45 | image_tag: Optional[str] = None, 46 | image_hash: Optional[str] = None, 47 | image_use_https: bool = False, 48 | dev_mode: bool = False, 49 | ) -> str: 50 | params = {} 51 | 52 | if image_tag: 53 | params["tag"] = image_tag 54 | 55 | if image_hash: 56 | params["hash"] = image_hash 57 | 58 | if not params: 59 | raise PackageException( 60 | "Either an image_hash or an image_tag is required " 61 | "to resolve an image URL from the Golem Registry." 62 | ) 63 | 64 | if "tag" in params and "hash" in params: 65 | raise PackageException( 66 | "Golem Registry images can be resolved by " 67 | "either an image_hash or by an image_tag but not both." 68 | ) 69 | 70 | if dev_mode: 71 | # if dev, skip usage statistics, pass dev option for statistics 72 | params["dev"] = "true" 73 | else: 74 | params["count"] = "true" 75 | 76 | async with aiohttp.ClientSession() as client: 77 | url = f"{repo_url}/v1/image/info" 78 | logger.debug(f"Querying registry portal: url={url}, params={params}") 79 | resp = await client.get(url, params=params) 80 | if resp.status != 200: 81 | try: 82 | text = await resp.text() 83 | except Exception as ex: 84 | logger.error(f"Failed to get body of response: {ex}") 85 | text = "N/A" 86 | 87 | logger.error(f"Failed to resolve image URL: {resp.status} {text}") 88 | raise Exception(f"Failed to resolve image URL: {resp.status} {text}") 89 | 90 | json_resp = await resp.json() 91 | 92 | if image_use_https: 93 | image_url = json_resp["https"] 94 | else: 95 | image_url = json_resp["http"] 96 | image_hash = json_resp["sha3"] 97 | image_size = json_resp["size"] 98 | logger.debug( 99 | f"Resolved image: " 100 | f"url={image_url}, " 101 | f"size={_sizeof_fmt(image_size)}, " 102 | f"hash={image_hash}" 103 | ) 104 | 105 | return f"hash:sha3:{image_hash}:{image_url}" 106 | -------------------------------------------------------------------------------- /yapapi/props/__init__.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from decimal import Decimal 3 | from typing import Optional 4 | 5 | from dataclasses import dataclass, field 6 | 7 | from .base import InvalidPropertiesError, Model, constraint, prop 8 | 9 | 10 | @dataclass 11 | class NodeInfo(Model): 12 | """Properties describing the information regarding the node.""" 13 | 14 | name: Optional[str] = field(default=None, metadata={"key": "golem.node.id.name"}) 15 | """human-readable name of the Golem node""" 16 | 17 | subnet_tag: Optional[str] = field(default=None, metadata={"key": "golem.node.debug.subnet"}) 18 | """the name of the subnet within which the Demands and Offers are matched""" 19 | 20 | 21 | NodeInfoKeys = NodeInfo.property_keys() 22 | 23 | 24 | @dataclass() 25 | class Activity(Model): 26 | """Activity-related Properties.""" 27 | 28 | cost_cap: Optional[Decimal] = field(default=None, metadata={"key": "golem.activity.cost_cap"}) 29 | """Sets a Hard cap on total cost of the Activity (regardless of the usage vector or 30 | pricing function). The Provider is entitled to 'kill' an Activity which exceeds the 31 | capped cost amount indicated by Requestor. 32 | """ 33 | 34 | cost_warning: Optional[Decimal] = field( 35 | default=None, metadata={"key": "golem.activity.cost_warning"} 36 | ) 37 | """Sets a Soft cap on total cost of the Activity (regardless of the usage vector or 38 | pricing function). When the cost_warning amount is reached for the Activity, 39 | the Provider is expected to send a Debit Note to the Requestor, indicating 40 | the current amount due 41 | """ 42 | 43 | timeout_secs: Optional[float] = field( 44 | default=None, metadata={"key": "golem.activity.timeout_secs"} 45 | ) 46 | """A timeout value for batch computation (eg. used for container-based batch 47 | processes). This property allows to set the timeout to be applied by the Provider 48 | when running a batch computation: the Requestor expects the Activity to take 49 | no longer than the specified timeout value - which implies that 50 | eg. the golem.usage.duration_sec counter shall not exceed the specified 51 | timeout value. 52 | """ 53 | 54 | expiration: Optional[datetime] = field( 55 | default=None, metadata={"key": "golem.srv.comp.expiration"} 56 | ) 57 | multi_activity: Optional[bool] = field( 58 | default=None, 59 | metadata={"key": "golem.srv.caps.multi-activity"}, 60 | ) 61 | """Whether client supports multi_activity (executing more than one activity per agreement). 62 | """ 63 | 64 | 65 | ActivityKeys = Activity.property_keys() 66 | 67 | __all__ = ( 68 | "InvalidPropertiesError", 69 | "Model", 70 | "constraint", 71 | "prop", 72 | "NodeInfo", 73 | "NodeInfoKeys", 74 | "Activity", 75 | "ActivityKeys", 76 | ) 77 | -------------------------------------------------------------------------------- /yapapi/props/com.py: -------------------------------------------------------------------------------- 1 | """Payment-related properties.""" 2 | 3 | import abc 4 | import enum 5 | from typing import Any, Dict, List 6 | 7 | from dataclasses import dataclass, field 8 | 9 | from .base import Model, Props 10 | 11 | SCHEME: str = "golem.com.scheme" 12 | PRICE_MODEL: str = "golem.com.pricing.model" 13 | 14 | LINEAR_COEFFS: str = "golem.com.pricing.model.linear.coeffs" 15 | DEFINED_USAGES: str = "golem.com.usage.vector" 16 | 17 | 18 | class BillingScheme(enum.Enum): 19 | PAYU = "payu" 20 | 21 | 22 | class PriceModel(enum.Enum): 23 | LINEAR = "linear" 24 | 25 | 26 | class Counter(enum.Enum): 27 | TIME = "golem.usage.duration_sec" 28 | CPU = "golem.usage.cpu_sec" 29 | STORAGE = "golem.usage.storage_gib" 30 | MAXMEM = "golem.usage.gib" 31 | UNKNOWN = "" 32 | 33 | 34 | @dataclass(frozen=True) 35 | class Com(Model): 36 | """Base model representing the payment model used.""" 37 | 38 | scheme: BillingScheme = field(metadata={"key": SCHEME}) 39 | price_model: PriceModel = field(metadata={"key": PRICE_MODEL}) 40 | 41 | @abc.abstractmethod 42 | def calculate_cost(self, usage: List) -> float: 43 | """Calculate the cost by applying the provided usage vector to the underlying pricing \ 44 | model.""" 45 | 46 | @abc.abstractmethod 47 | def usage_as_dict(self, usage: List) -> Dict: 48 | """Return usage as a dictionary where keys are the appropriate usage counters.""" 49 | 50 | 51 | @dataclass(frozen=True) 52 | class ComLinear(Com): 53 | """Linear payment model.""" 54 | 55 | linear_coeffs: List[float] = field(metadata={"key": LINEAR_COEFFS}) 56 | usage_vector: List[str] = field(metadata={"key": DEFINED_USAGES}) 57 | 58 | @classmethod 59 | def _custom_mapping(cls, props: Props, data: Dict[str, Any]): 60 | # we don't need mapping per-se but we'll do some validation instead 61 | assert data["price_model"] == PriceModel.LINEAR, "expected linear pricing model" 62 | assert ( 63 | len(data["linear_coeffs"]) == len(data["usage_vector"]) + 1 64 | ), "expecting the number of linear_coeffs to correspond to usage_vector + 1 (fixed price)" 65 | assert all( 66 | [isinstance(lc, float) for lc in data["linear_coeffs"]] 67 | ), "linear_coeffs values must be `float`" 68 | assert all( 69 | [isinstance(u, str) for u in data["usage_vector"]] 70 | ), "usage_vector values must be `str`" 71 | 72 | @property 73 | def fixed_price(self) -> float: 74 | return self.linear_coeffs[-1] 75 | 76 | @property 77 | def price_for(self) -> Dict[str, float]: 78 | return {u: self.linear_coeffs[i] for (i, u) in enumerate(self.usage_vector)} 79 | 80 | def calculate_cost(self, usage: List): 81 | usage = usage + [1.0] # append the "usage" of the fixed component 82 | return sum([c * usage[i] for (i, c) in enumerate(self.linear_coeffs)]) 83 | 84 | def usage_as_dict(self, usage: List) -> Dict[str, float]: 85 | return {self.usage_vector[i]: u for (i, u) in enumerate(usage)} 86 | -------------------------------------------------------------------------------- /yapapi/props/inf.py: -------------------------------------------------------------------------------- 1 | """Infrastructural properties.""" 2 | 3 | from enum import Enum 4 | from typing import Any, Dict, List, Optional 5 | 6 | from dataclasses import dataclass 7 | from deprecated import deprecated 8 | 9 | from .base import Model, prop 10 | 11 | INF_MEM: str = "golem.inf.mem.gib" 12 | INF_STORAGE: str = "golem.inf.storage.gib" 13 | INF_CORES: str = "golem.inf.cpu.cores" 14 | INF_THREADS: str = "golem.inf.cpu.threads" 15 | TRANSFER_CAPS: str = "golem.activity.caps.transfer.protocol" 16 | INF_RUNTIME_NAME = "golem.runtime.name" 17 | 18 | RUNTIME_VM = "vm" 19 | 20 | 21 | @dataclass 22 | class WasmInterface(Enum): 23 | WASI_0 = "0" 24 | WASI_0preview1 = "0preview1" 25 | 26 | 27 | @dataclass 28 | class InfBase(Model): 29 | runtime: str = prop(INF_RUNTIME_NAME) 30 | 31 | mem: float = prop(INF_MEM) 32 | storage: Optional[float] = prop(INF_STORAGE, default=None) 33 | transfers: Optional[List[str]] = prop(TRANSFER_CAPS, default=None) 34 | 35 | 36 | @dataclass 37 | class ExeUnitRequest(Model): 38 | package_url: str = prop("golem.srv.comp.task_package") 39 | 40 | 41 | @deprecated(version="0.6.0", reason="this is part of yapapi.payload.vm now") 42 | class VmPackageFormat(Enum): 43 | UNKNOWN = None 44 | GVMKIT_SQUASH = "gvmkit-squash" 45 | 46 | 47 | @dataclass 48 | class ExeUnitManifestRequest(Model): 49 | manifest: str = prop("golem.srv.comp.payload") 50 | manifest_sig: Optional[str] = prop("golem.srv.comp.payload.sig", default=None) 51 | manifest_sig_algorithm: Optional[str] = prop( 52 | "golem.srv.comp.payload.sig.algorithm", default=None 53 | ) 54 | manifest_cert: Optional[str] = prop("golem.srv.comp.payload.cert", default=None) 55 | package_format: VmPackageFormat = prop( 56 | "golem.srv.comp.vm.package_format", default=VmPackageFormat.GVMKIT_SQUASH 57 | ) 58 | node_descriptor: Optional[Dict[str, Any]] = prop( 59 | "golem.!exp.gap-31.v0.node.descriptor", default=None 60 | ) 61 | 62 | 63 | @deprecated(version="0.6.0", reason="this is part of yapapi.payload.vm now") 64 | @dataclass 65 | class VmRequest(ExeUnitRequest): 66 | package_format: VmPackageFormat = prop("golem.srv.comp.vm.package_format") 67 | -------------------------------------------------------------------------------- /yapapi/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golemfactory/yapapi/eced2b852f560b3a7d35d5894d4f6b7268790b68/yapapi/py.typed -------------------------------------------------------------------------------- /yapapi/rest/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Mid-level binding for Golem REST API. 3 | 4 | Serves as a more convenient interface between the agent code and the REST API. 5 | """ 6 | 7 | from . import activity, market 8 | from .activity import ActivityService as Activity 9 | from .configuration import Configuration 10 | from .market import Market 11 | from .net import Net 12 | from .payment import Payment 13 | 14 | __all__ = ( 15 | "activity", 16 | "market", 17 | "Activity", 18 | "Configuration", 19 | "Market", 20 | "Net", 21 | "Payment", 22 | ) 23 | -------------------------------------------------------------------------------- /yapapi/rest/common.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | import logging 4 | from typing import Callable, Optional 5 | 6 | import aiohttp 7 | 8 | import ya_activity 9 | import ya_market 10 | import ya_payment 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def is_intermittent_error(e: Exception) -> bool: 16 | """Check if `e` indicates an intermittent communication failure such as network timeout.""" 17 | 18 | is_timeout_exception = isinstance(e, asyncio.TimeoutError) or ( 19 | isinstance(e, (ya_activity.ApiException, ya_market.ApiException, ya_payment.ApiException)) 20 | and e.status in (408, 504) 21 | ) 22 | 23 | return ( 24 | is_timeout_exception 25 | or isinstance(e, aiohttp.ServerDisconnectedError) 26 | # OS error with errno 32 is "Broken pipe" 27 | or (isinstance(e, aiohttp.ClientOSError) and e.errno == 32) 28 | ) 29 | 30 | 31 | class SuppressedExceptions: 32 | """An async context manager for suppressing exceptions satisfying given condition.""" 33 | 34 | exception: Optional[Exception] 35 | 36 | def __init__(self, condition: Callable[[Exception], bool], report_exceptions: bool = True): 37 | self._condition = condition 38 | self._report_exceptions = report_exceptions 39 | self.exception = None 40 | 41 | async def __aenter__(self) -> "SuppressedExceptions": 42 | return self 43 | 44 | async def __aexit__(self, exc_type, exc_value, traceback): 45 | if exc_value and self._condition(exc_value): 46 | self.exception = exc_value 47 | if self._report_exceptions: 48 | logger.debug( 49 | "Exception suppressed: %r", exc_value, exc_info=(exc_type, exc_value, traceback) 50 | ) 51 | return True 52 | return False 53 | 54 | 55 | def repeat_on_error( 56 | max_tries: int, 57 | condition: Callable[[Exception], bool] = is_intermittent_error, 58 | interval: float = 1.0, 59 | ): 60 | """Decorate a function to repeat calls up to `max_tries` times when errors occur. 61 | 62 | Only exceptions satisfying the given `condition` will cause the decorated function 63 | to be retried. All remaining exceptions will fall through. 64 | """ 65 | 66 | def decorator(func): 67 | @functools.wraps(func) 68 | async def wrapper(*args, **kwargs): 69 | """Make at most `max_tries` attempts to call `func`.""" 70 | 71 | for try_num in range(1, max_tries + 1): 72 | if try_num > 1: 73 | await asyncio.sleep(interval) 74 | 75 | async with SuppressedExceptions(condition, False) as se: 76 | return await func(*args, **kwargs) 77 | 78 | assert se.exception # noqa (unreachable) 79 | repeat = try_num < max_tries 80 | msg = f"API call timed out (attempt {try_num}/{max_tries}), " 81 | msg += f"retrying in {interval} s" if repeat else "giving up" 82 | # Don't print traceback if this was the last attempt, let the caller do it. 83 | logger.debug(msg, exc_info=repeat) 84 | if not repeat: 85 | raise se.exception 86 | 87 | return wrapper 88 | 89 | return decorator 90 | -------------------------------------------------------------------------------- /yapapi/rest/net.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from ya_net import ApiClient, RequestorApi 4 | from ya_net import models as yan 5 | 6 | 7 | class Net(object): 8 | """Mid-level interface to the Net REST API.""" 9 | 10 | def __init__(self, api_client: ApiClient): 11 | self._api = RequestorApi(api_client) 12 | 13 | @property 14 | def api_url(self): 15 | return self._api.api_client.configuration.host 16 | 17 | async def create_network( 18 | self, network_address: str, netmask: Optional[str], gateway: Optional[str] 19 | ) -> str: 20 | yan_network = await self._api.create_network( 21 | yan.Network( 22 | ip=network_address, 23 | mask=netmask, 24 | gateway=gateway, 25 | ) 26 | ) 27 | return yan_network.id 28 | 29 | async def remove_network(self, network_id: str) -> None: 30 | await self._api.remove_network(network_id) 31 | 32 | async def add_address(self, network_id: str, ip: str): 33 | address = yan.Address(ip) 34 | await self._api.add_address(network_id, address) 35 | 36 | async def add_node(self, network_id: str, node_id: str, ip: str): 37 | await self._api.add_node(network_id, yan.Node(node_id, ip)) 38 | 39 | async def remove_node(self, network_id: str, node_id: str): 40 | await self._api.remove_node(network_id, node_id) 41 | -------------------------------------------------------------------------------- /yapapi/rest/resource.py: -------------------------------------------------------------------------------- 1 | from typing import AsyncContextManager, TypeVar 2 | 3 | _T = TypeVar("_T") 4 | 5 | 6 | class ResourceCtx(AsyncContextManager[_T]): 7 | async def detach(self) -> _T: 8 | resource = await self.__aenter__() 9 | return resource 10 | -------------------------------------------------------------------------------- /yapapi/script/capture.py: -------------------------------------------------------------------------------- 1 | import enum 2 | from typing import Any, Dict, Optional 3 | 4 | from dataclasses import dataclass 5 | 6 | 7 | class CaptureMode(enum.Enum): 8 | HEAD = "head" 9 | TAIL = "tail" 10 | HEAD_TAIL = "headTail" 11 | STREAM = "stream" 12 | 13 | 14 | class CaptureFormat(enum.Enum): 15 | BIN = "bin" 16 | STR = "str" 17 | 18 | 19 | @dataclass 20 | class CaptureContext: 21 | mode: CaptureMode 22 | limit: Optional[int] 23 | fmt: Optional[CaptureFormat] 24 | 25 | @classmethod 26 | def build(cls, mode=None, limit=None, fmt=None) -> "CaptureContext": 27 | if mode in (None, "all"): 28 | return cls._build(CaptureMode.HEAD, fmt=fmt) 29 | elif mode == "stream": 30 | return cls._build(CaptureMode.STREAM, limit=limit, fmt=fmt) 31 | elif mode == "head": 32 | return cls._build(CaptureMode.HEAD, limit=limit, fmt=fmt) 33 | elif mode == "tail": 34 | return cls._build(CaptureMode.TAIL, limit=limit, fmt=fmt) 35 | elif mode == "headTail": 36 | return cls._build(CaptureMode.HEAD_TAIL, limit=limit, fmt=fmt) 37 | raise RuntimeError(f"Invalid output capture mode: {mode}") 38 | 39 | @classmethod 40 | def _build( 41 | cls, 42 | mode: CaptureMode, 43 | limit: Optional[int] = None, 44 | fmt: Optional[str] = None, 45 | ) -> "CaptureContext": 46 | cap_fmt: Optional[CaptureFormat] = CaptureFormat(fmt) if fmt else None 47 | return cls(mode=mode, fmt=cap_fmt, limit=limit) 48 | 49 | def to_dict(self) -> Dict: 50 | inner: Dict[str, Any] = dict() 51 | 52 | if self.limit: 53 | inner[self.mode.value] = self.limit 54 | if self.fmt: 55 | inner["format"] = self.fmt.value 56 | 57 | return {"stream" if self.mode == CaptureMode.STREAM else "atEnd": inner} 58 | 59 | def is_streaming(self) -> bool: 60 | return self.mode == CaptureMode.STREAM 61 | -------------------------------------------------------------------------------- /yapapi/services/__init__.py: -------------------------------------------------------------------------------- 1 | from .cluster import Cluster 2 | from .service import Service, ServiceInstance, ServiceSerialization, ServiceType 3 | from .service_runner import ServiceRunner 4 | from .service_state import ServiceState 5 | 6 | __all__ = ( 7 | "Cluster", 8 | "Service", 9 | "ServiceInstance", 10 | "ServiceType", 11 | "ServiceRunner", 12 | "ServiceSerialization", 13 | "ServiceState", 14 | ) 15 | -------------------------------------------------------------------------------- /yapapi/services/service_state.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | import statemachine 4 | 5 | if TYPE_CHECKING: 6 | from .service import ServiceInstance 7 | 8 | 9 | class ServiceState(statemachine.StateMachine): 10 | """State machine describing the state and lifecycle of a :class:`Service` instance.""" 11 | 12 | # states 13 | pending = statemachine.State("pending", initial=True) 14 | """The service instance has not yet been assigned to a provider.""" 15 | 16 | starting = statemachine.State("starting") 17 | """The service instance is starting on a provider. 18 | 19 | The activity within which the service is running has been created on a provider 20 | node and now the service instance's :meth:`~yapapi.services.Service.start` 21 | handler is active and has not yet finished. 22 | """ 23 | 24 | running = statemachine.State("running") 25 | """The service instance is running on a provider. 26 | 27 | The instance's :meth:`~yapapi.services.Service.start` handler has finished and 28 | the :meth:`~yapapi.services.Service.run` handler is active. 29 | """ 30 | 31 | stopping = statemachine.State("stopping") 32 | """The service instance is stopping on a provider. 33 | 34 | The instance's :meth:`~yapapi.services.Service.run` handler has finished and 35 | the :meth:`~yapapi.services.Service.shutdown` handler is active. 36 | """ 37 | 38 | terminated = statemachine.State("terminated") 39 | """The service instance has been terminated and is no longer bound to an activity. 40 | 41 | This means that either the service has been explicitly stopped by the requestor, 42 | or the activity that the service had been attached-to has been terminated - e.g. 43 | by a failure on the provider's end or as a result of termination of the agreement 44 | between the requestor and the provider. 45 | """ 46 | unresponsive = statemachine.State("unresponsive") 47 | 48 | suspended = statemachine.State("suspended") 49 | """This service instance has been suspended. 50 | 51 | Its handlers should not be processed by the ServiceRunner anymore but no resultant changes 52 | to the activity itself should be made. 53 | """ 54 | 55 | # transitions 56 | start: statemachine.Transition = pending.to(starting) 57 | ready: statemachine.Transition = starting.to(running) 58 | stop: statemachine.Transition = running.to(stopping) 59 | terminate: statemachine.Transition = terminated.from_(starting, running, stopping, terminated) 60 | mark_unresponsive: statemachine.Transition = unresponsive.from_( 61 | starting, running, stopping, terminated 62 | ) 63 | restart: statemachine.Transition = pending.from_( 64 | pending, starting, running, stopping, terminated, unresponsive 65 | ) 66 | 67 | lifecycle = start | ready | stop | terminate 68 | """Transition performed when handler for the current state terminates normally. 69 | That is, not due to an error or `ControlSignal.stop` 70 | """ 71 | 72 | error_or_stop = stop | terminate 73 | """transition performed on error or `ControlSignal.stop`""" 74 | 75 | suspend: statemachine.Transition = suspended.from_(starting, running, stopping) 76 | """transition performed on `ControlSignal.suspend`""" 77 | 78 | AVAILABLE = (starting, running, stopping) 79 | """A helper set of states in which the service instance is bound to an activity 80 | and can be interacted with.""" 81 | 82 | instance: "ServiceInstance" 83 | """The ServiceRunner's service instance.""" 84 | 85 | def on_enter_state(self, state: statemachine.State): 86 | """Register `state` in the instance's list of visited states.""" 87 | self.instance.visited_states.append(state) 88 | -------------------------------------------------------------------------------- /yapapi/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import ( 2 | DEBIT_NOTE_INTERVAL_GRACE_PERIOD, 3 | PROP_DEBIT_NOTE_ACCEPTANCE_TIMEOUT, 4 | PROP_DEBIT_NOTE_INTERVAL_SEC, 5 | PROP_PAYMENT_TIMEOUT_SEC, 6 | SCORE_NEUTRAL, 7 | SCORE_REJECTED, 8 | SCORE_TRUSTED, 9 | BaseMarketStrategy, 10 | MarketStrategy, 11 | PropValueRange, 12 | ) 13 | from .decrease_score_unconfirmed import DecreaseScoreForUnconfirmedAgreement 14 | from .dummy import DummyMS 15 | from .least_expensive import LeastExpensiveLinearPayuMS 16 | from .wrapping_strategy import WrappingMarketStrategy 17 | 18 | __all__ = ( 19 | "DEBIT_NOTE_INTERVAL_GRACE_PERIOD", 20 | "PROP_DEBIT_NOTE_ACCEPTANCE_TIMEOUT", 21 | "PROP_DEBIT_NOTE_INTERVAL_SEC", 22 | "PROP_PAYMENT_TIMEOUT_SEC", 23 | "SCORE_NEUTRAL", 24 | "SCORE_REJECTED", 25 | "SCORE_TRUSTED", 26 | "BaseMarketStrategy", 27 | "MarketStrategy", 28 | "PropValueRange", 29 | "DecreaseScoreForUnconfirmedAgreement", 30 | "DummyMS", 31 | "LeastExpensiveLinearPayuMS", 32 | "WrappingMarketStrategy", 33 | ) 34 | -------------------------------------------------------------------------------- /yapapi/strategy/decrease_score_unconfirmed.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Set 3 | 4 | from yapapi import events, rest 5 | from yapapi.props.builder import DemandBuilder 6 | 7 | from .wrapping_strategy import WrappingMarketStrategy 8 | 9 | 10 | class DecreaseScoreForUnconfirmedAgreement(WrappingMarketStrategy): 11 | """A market strategy wrapper that modifies scoring of providers based on history of agreements. 12 | 13 | It decreases the scores for providers that rejected our agreements. 14 | 15 | The strategy keeps an internal list of providers who have rejected proposed agreements and 16 | removes them from this list when they accept one. 17 | """ 18 | 19 | factor: float 20 | 21 | def __init__(self, base_strategy, factor): 22 | """Initialize instance. 23 | 24 | :param base_strategy: the base strategy around which this strategy is wrapped 25 | :param factor: the factor by which the score of an offer for a provider which 26 | failed to confirm the last agreement proposed to them will be multiplied 27 | """ 28 | super().__init__(base_strategy) 29 | self.factor = factor 30 | self._logger = logging.getLogger(f"{__name__}.{type(self).__name__}") 31 | self._rejecting_providers: Set[str] = set() 32 | 33 | def on_event(self, event: events.Event) -> None: 34 | """Modify the internal `_rejecting_providers` list on `AgreementConfirmed/Rejected`. 35 | 36 | This method needs to be added as an event consumer in 37 | :func:`yapapi.Golem.add_event_consumer`. 38 | """ 39 | if isinstance(event, events.AgreementConfirmed): 40 | self._rejecting_providers.discard(event.provider_id) 41 | elif isinstance(event, events.AgreementRejected): 42 | self._rejecting_providers.add(event.provider_id) 43 | 44 | async def decorate_demand(self, demand: DemandBuilder) -> None: 45 | """Decorate `demand` using the base strategy.""" 46 | await self.base_strategy.decorate_demand(demand) 47 | 48 | async def score_offer(self, offer: rest.market.OfferProposal) -> float: 49 | """Score `offer` using the base strategy and apply penalty if needed. 50 | 51 | If the offer issuer failed to approve the previous agreement (if any) 52 | and the base score is positive, then the base score is multiplied by `self.factor`. 53 | """ 54 | score = await self.base_strategy.score_offer(offer) 55 | if offer.issuer in self._rejecting_providers and score > 0: 56 | self._logger.debug("Decreasing score for offer %s from '%s'", offer.id, offer.issuer) 57 | score *= self.factor 58 | return score 59 | -------------------------------------------------------------------------------- /yapapi/strategy/dummy.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from decimal import Decimal 3 | from types import MappingProxyType 4 | from typing import Dict, Mapping, Optional, Union 5 | 6 | from dataclasses import dataclass 7 | from deprecated import deprecated 8 | 9 | from yapapi import rest 10 | from yapapi.props import Activity, com 11 | from yapapi.props.builder import DemandBuilder 12 | from yapapi.props.com import Counter 13 | 14 | from .base import SCORE_NEUTRAL, SCORE_REJECTED, MarketStrategy 15 | 16 | 17 | @deprecated(version="0.9.0", reason="Use `LeastExpensiveLinearPayuMS` instead.") 18 | @dataclass 19 | class DummyMS(MarketStrategy, object): 20 | """A default market strategy implementation. 21 | 22 | [ DEPRECATED, use `LeastExpensiveLinearPayuMS` instead ] 23 | 24 | Its :func:`score_offer()` method returns :const:`SCORE_NEUTRAL` for every offer with prices 25 | that do not exceed maximum prices specified for each counter. 26 | For other offers, returns :const:`SCORE_REJECTED`. 27 | """ 28 | 29 | def __init__( 30 | self, 31 | max_fixed_price: Decimal = Decimal("0.05"), 32 | max_price_for: Mapping[Union[Counter, str], Decimal] = MappingProxyType({}), 33 | activity: Optional[Activity] = None, 34 | ): 35 | self._max_fixed_price = max_fixed_price 36 | self._max_price_for: Dict[str, Decimal] = defaultdict(lambda: Decimal("inf")) 37 | self._max_price_for.update( 38 | {com.Counter.TIME.value: Decimal("0.002"), com.Counter.CPU.value: Decimal("0.002") * 10} 39 | ) 40 | self._max_price_for.update( 41 | {(c.value if isinstance(c, Counter) else c): v for (c, v) in max_price_for.items()} 42 | ) 43 | self._activity = activity 44 | 45 | async def decorate_demand(self, demand: DemandBuilder) -> None: 46 | await super().decorate_demand(demand) 47 | 48 | # Ensure that the offer uses `PriceModel.LINEAR` price model 49 | demand.ensure(f"({com.PRICE_MODEL}={com.PriceModel.LINEAR.value})") 50 | self._activity = Activity.from_properties(demand.properties) 51 | 52 | async def score_offer(self, offer: rest.market.OfferProposal) -> float: 53 | """Score `offer`. Returns either `SCORE_REJECTED` or `SCORE_NEUTRAL`.""" 54 | 55 | linear: com.ComLinear = com.ComLinear.from_properties(offer.props) 56 | 57 | if linear.scheme != com.BillingScheme.PAYU: 58 | return SCORE_REJECTED 59 | 60 | if linear.fixed_price > self._max_fixed_price: 61 | return SCORE_REJECTED 62 | for counter, price in linear.price_for.items(): 63 | if counter not in self._max_price_for: 64 | return SCORE_REJECTED 65 | if price > self._max_price_for[counter]: 66 | return SCORE_REJECTED 67 | 68 | return SCORE_NEUTRAL 69 | -------------------------------------------------------------------------------- /yapapi/strategy/least_expensive.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import defaultdict 3 | from decimal import Decimal 4 | from types import MappingProxyType 5 | from typing import Dict, Mapping, Union 6 | 7 | from dataclasses import dataclass 8 | 9 | from yapapi import rest 10 | from yapapi.props import com 11 | from yapapi.props.builder import DemandBuilder 12 | from yapapi.props.com import Counter 13 | 14 | from .base import SCORE_REJECTED, SCORE_TRUSTED, MarketStrategy 15 | 16 | 17 | @dataclass 18 | class LeastExpensiveLinearPayuMS(MarketStrategy, object): 19 | """A strategy that scores offers according to cost for given computation time.""" 20 | 21 | def __init__( 22 | self, 23 | expected_time_secs: int = 60, 24 | max_fixed_price: Decimal = Decimal("inf"), 25 | max_price_for: Mapping[Union[Counter, str], Decimal] = MappingProxyType({}), 26 | ): 27 | self._expected_time_secs = expected_time_secs 28 | self._logger = logging.getLogger(f"{__name__}.{type(self).__name__}") 29 | self._max_fixed_price = max_fixed_price 30 | self._max_price_for: Dict[str, Decimal] = defaultdict(lambda: Decimal("inf")) 31 | self._max_price_for.update( 32 | {(c.value if isinstance(c, Counter) else c): v for (c, v) in max_price_for.items()} 33 | ) 34 | 35 | async def decorate_demand(self, demand: DemandBuilder) -> None: 36 | await super().decorate_demand(demand) 37 | 38 | # Ensure that the offer uses `PriceModel.LINEAR` price model. 39 | demand.ensure(f"({com.PRICE_MODEL}={com.PriceModel.LINEAR.value})") 40 | 41 | async def score_offer(self, offer: rest.market.OfferProposal) -> float: 42 | """Score `offer` according to cost for expected computation time.""" 43 | 44 | linear: com.ComLinear = com.ComLinear.from_properties(offer.props) 45 | self._logger.debug("Scoring offer %s, parameters: %s", offer.id, linear) 46 | if linear.scheme != com.BillingScheme.PAYU: 47 | self._logger.debug( 48 | "Rejected offer %s: unsupported scheme '%s'", offer.id, linear.scheme 49 | ) 50 | return SCORE_REJECTED 51 | 52 | if linear.fixed_price > self._max_fixed_price: 53 | self._logger.debug( 54 | "Rejected offer %s: fixed price higher than fixed price cap %f.", 55 | offer.id, 56 | self._max_fixed_price, 57 | ) 58 | return SCORE_REJECTED 59 | 60 | if linear.fixed_price < 0: 61 | self._logger.debug("Rejected offer %s: negative fixed price", offer.id) 62 | return SCORE_REJECTED 63 | 64 | expected_usage = [] 65 | 66 | for resource in linear.usage_vector: 67 | if linear.price_for[resource] > self._max_price_for[resource]: 68 | self._logger.debug( 69 | "Rejected offer %s: price for '%s' higher than price cap %f.", 70 | offer.id, 71 | resource, 72 | self._max_price_for[resource], 73 | ) 74 | return SCORE_REJECTED 75 | 76 | if linear.price_for[resource] < 0: 77 | self._logger.debug("Rejected offer %s: negative price for '%s'", offer.id, resource) 78 | return SCORE_REJECTED 79 | 80 | expected_usage.append(self._expected_time_secs) 81 | 82 | # The higher the expected price value, the lower the score. 83 | # The score is always lower than SCORE_TRUSTED and is always higher than 0. 84 | score = SCORE_TRUSTED * 1.0 / (linear.calculate_cost(expected_usage) + 1.01) 85 | return score 86 | -------------------------------------------------------------------------------- /yapapi/strategy/wrapping_strategy.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from decimal import Decimal 3 | 4 | from yapapi import rest 5 | from yapapi.props.builder import DemandBuilder 6 | 7 | from .base import BaseMarketStrategy 8 | 9 | 10 | class WrappingMarketStrategy(BaseMarketStrategy, abc.ABC): 11 | """Helper abstract class which allows other/user defined strategies to wrap some other \ 12 | strategies, without overriding the attributes (e.g. defaults) defined on the derived-from \ 13 | strategy. 14 | 15 | WrappingMarketStrategy classes are unusable on their own and always have to wrap some base 16 | strategy. 17 | 18 | By default, all attributes and method calls are forwarded to the `base_strategy`. 19 | """ 20 | 21 | base_strategy: BaseMarketStrategy 22 | """base strategy wrapped by this wrapper.""" 23 | 24 | def __init__(self, base_strategy: BaseMarketStrategy): 25 | """Initialize instance. 26 | 27 | :param base_strategy: the base strategy around which this strategy is wrapped. 28 | """ 29 | self.base_strategy = base_strategy 30 | 31 | async def decorate_demand(self, demand: DemandBuilder) -> None: 32 | await self.base_strategy.decorate_demand(demand) 33 | 34 | async def score_offer(self, offer: rest.market.OfferProposal) -> float: 35 | return await self.base_strategy.score_offer(offer) 36 | 37 | async def invoice_accepted_amount(self, invoice: rest.payment.Invoice) -> Decimal: 38 | return await self.base_strategy.invoice_accepted_amount(invoice) 39 | 40 | async def debit_note_accepted_amount(self, debit_note: rest.payment.DebitNote) -> Decimal: 41 | return await self.base_strategy.debit_note_accepted_amount(debit_note) 42 | 43 | async def respond_to_provider_offer( 44 | self, 45 | our_demand: DemandBuilder, 46 | provider_offer: rest.market.OfferProposal, 47 | ) -> DemandBuilder: 48 | return await self.base_strategy.respond_to_provider_offer(our_demand, provider_offer) 49 | 50 | def __getattr__(self, item): 51 | """Forward all calls for undefined properties and variables to the base class.""" 52 | return getattr(self.base_strategy, item) 53 | --------------------------------------------------------------------------------