├── .bandit ├── .github ├── auto_assign-issues.yml └── workflows │ ├── coverity.yml │ ├── integration.yml │ ├── scorecard.yml │ ├── security.yml │ └── tests.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .pylintrc ├── .vulture_allowlist ├── CODEOWNERS ├── LICENSE ├── LICENSES └── third_party │ └── third-party-licenses.txt ├── MANIFEST.in ├── Makefile ├── README.md ├── SECURITY.md ├── ci └── tutorial │ ├── model.py │ └── sigopt_config.exp ├── examples ├── README.md ├── basic.py ├── other_languages.py └── parallel.py ├── integration_test ├── __init__.py ├── test_experiment.py ├── test_sigopt.py ├── test_xgboost_experiment.py └── test_xgboost_run.py ├── lint ├── publish ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── sigopt ├── __init__.py ├── aiexperiment_context.py ├── cli │ ├── __init__.py │ ├── __main__.py │ ├── arguments │ │ ├── __init__.py │ │ ├── commands.py │ │ ├── experiment_file.py │ │ ├── experiment_id.py │ │ ├── load_yaml.py │ │ ├── project.py │ │ ├── run_file.py │ │ ├── source_file.py │ │ └── validate.py │ ├── commands │ │ ├── __init__.py │ │ ├── base.py │ │ ├── config.py │ │ ├── experiment │ │ │ ├── __init__.py │ │ │ ├── archive.py │ │ │ ├── create.py │ │ │ └── unarchive.py │ │ ├── init.py │ │ ├── local │ │ │ ├── __init__.py │ │ │ ├── optimize.py │ │ │ ├── run.py │ │ │ └── start_worker.py │ │ ├── optimize_base.py │ │ ├── project │ │ │ ├── __init__.py │ │ │ └── create.py │ │ ├── run_base.py │ │ ├── training_run │ │ │ ├── __init__.py │ │ │ ├── archive.py │ │ │ └── unarchive.py │ │ └── version.py │ ├── resources │ │ ├── __init__.py │ │ ├── init_dockerfile.txt │ │ ├── init_dockerignore.txt │ │ ├── init_experiment.txt │ │ └── init_run.txt │ └── utils.py ├── compat.py ├── config.py ├── decorators.py ├── defaults.py ├── endpoint.py ├── examples │ ├── __init__.py │ └── franke.py ├── exception.py ├── factory.py ├── file_utils.py ├── interface.py ├── lib.py ├── local_run_context.py ├── log_capture.py ├── magics.py ├── model_aware_run.py ├── objects.py ├── paths.py ├── ratelimit.py ├── request_driver.py ├── resource.py ├── run_context.py ├── run_factory.py ├── run_params.py ├── sigopt_logging.py ├── urllib3_patch.py ├── utils.py ├── validate │ ├── __init__.py │ ├── aiexperiment_input.py │ ├── common.py │ ├── exceptions.py │ ├── keys.py │ └── run_input.py ├── version.py └── xgboost │ ├── __init__.py │ ├── checkpoint_callback.py │ ├── compat.py │ ├── compute_metrics.py │ ├── constants.py │ ├── experiment.py │ ├── run.py │ └── utils.py ├── test ├── __init__.py ├── cli │ ├── test_cli_config.py │ ├── test_files │ │ ├── import_hello.py │ │ ├── invalid_sigopt.yml │ │ ├── print_args.py │ │ ├── print_hello.py │ │ ├── valid_sigopt.yml │ │ └── warning_sigopt.yml │ ├── test_init.py │ ├── test_optimize.py │ ├── test_run.py │ ├── test_start_worker.py │ ├── test_truncate.py │ └── test_version.py ├── client │ ├── __init__.py │ ├── json_data │ │ ├── client.json │ │ ├── experiment.json │ │ ├── importances.json │ │ ├── metric.json │ │ ├── metric_importances.json │ │ ├── organization.json │ │ ├── pagination.json │ │ ├── plan.json │ │ ├── queued_suggestion.json │ │ ├── suggestion.json │ │ ├── task.json │ │ ├── token.json │ │ └── training_run.json │ ├── test_endpoint.py │ ├── test_interface.py │ ├── test_lib.py │ ├── test_object.py │ ├── test_pagination.py │ ├── test_request_driver.py │ ├── test_requestor.py │ ├── test_resource.py │ └── test_urllib3_patch.py ├── runs │ ├── test_config.py │ ├── test_context.py │ ├── test_defaults.py │ ├── test_factory.py │ ├── test_files │ │ ├── test.bmp │ │ ├── test.png │ │ └── test.svg │ ├── test_local_run_context.py │ ├── test_set_project.py │ ├── test_utils.py │ └── utils.py ├── utils.py ├── validate │ ├── test_validate_experiment.py │ └── test_validate_run.py └── xgboost │ ├── __init__.py │ ├── test_compute_metric.py │ ├── test_experiment_config.py │ └── test_run_options_parsing.py ├── tools ├── generate_feature_importances.py ├── generate_vulture_allowlist └── run_vulture.sh └── trivy.yaml /.github/auto_assign-issues.yml: -------------------------------------------------------------------------------- 1 | assignees: 2 | - sigopt/Dev 3 | -------------------------------------------------------------------------------- /.github/workflows/coverity.yml: -------------------------------------------------------------------------------- 1 | name: Coverity Scan 2 | permissions: read-all 3 | on: 4 | push: 5 | branches: [main] 6 | jobs: 7 | coverity: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | - run: git ls-files > git-ls-files.lst 12 | - uses: vapier/coverity-scan-action@v1 13 | with: 14 | project: sigopt-python 15 | email: ${{ secrets.COVERITY_SCAN_EMAIL }} 16 | token: ${{ secrets.COVERITY_SCAN_TOKEN }} 17 | build_language: other 18 | command: "--no-command --fs-capture-list git-ls-files.lst" 19 | -------------------------------------------------------------------------------- /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: integration tests 2 | permissions: read-all 3 | run-name: Integration tests for ${{ github.repository }}@${{ github.ref }} 4 | on: 5 | push: {} 6 | schedule: 7 | - cron: "0 8,16 * * *" 8 | jobs: 9 | integration-test: 10 | runs-on: ubuntu-latest 11 | env: 12 | SIGOPT_PROJECT: hyperopt-integration-test 13 | SIGOPT_API_URL: https://sigopt.ninja:4443/api 14 | SIGOPT_API_VERIFY_SSL_CERTS: /home/runner/work/sigopt-python/sigopt-server/artifacts/tls/root-ca.crt 15 | steps: 16 | - name: Check out repository code 17 | uses: actions/checkout@v4 18 | - name: Set up Python 3.10 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: "3.10" 22 | - run: echo '127.0.0.1 sigopt.ninja' | sudo tee -a /etc/hosts 23 | - run: git clone https://github.com/sigopt/sigopt-server ../sigopt-server 24 | - run: | 25 | cd ../sigopt-server 26 | git checkout main 27 | EDITOR=true ./setup.sh | tee setup_output.log 28 | ./start.sh & 29 | - run: | 30 | cd ../sigopt-server 31 | ./ci/wait_for.sh url "$SIGOPT_API_URL/health" 32 | - run: | 33 | EMAIL="$(tail -n 2 ../sigopt-server/setup_output.log | grep -E '^ email:' | awk -F': ' '{ print $2 }')" 34 | PASSWORD="$(tail -n 2 ../sigopt-server/setup_output.log | grep -E '^ password:' | awk -F': ' '{ print $2 }')" 35 | curl -fsk -XPOST "$SIGOPT_API_URL/v1/sessions" -d '{"email": "'"$EMAIL"'", "password": "'"$PASSWORD"'"}' | \ 36 | jq -r .api_token.token >sigopt_user_token.txt 37 | curl -fsk --user "$(cat sigopt_user_token.txt):" "$SIGOPT_API_URL/v1/clients/1/tokens" | \ 38 | jq -r '.data[] | select(.token_type == "client-api") | .token' >sigopt_api_token.txt 39 | - run: pip install '.[dev]' 40 | - run: sudo apt-get update && sudo apt-get -y install default-jre 41 | - run: docker run -p 27017:27017 -d mongo:5.0.6 42 | - run: hyperopt-mongo-worker --mongo=localhost:27017/foo_db --poll-interval=0.1 --max-consecutive-failures=100000 &>/dev/null & 43 | - run: env SIGOPT_API_TOKEN="$(cat sigopt_api_token.txt)" sigopt create project 44 | - name: Integration tests 45 | run: env SIGOPT_API_TOKEN="$(cat sigopt_api_token.txt)" pytest -rw -v integration_test/ 46 | tutorial: 47 | runs-on: ubuntu-latest 48 | env: 49 | SIGOPT_PROJECT: tutorial 50 | SIGOPT_API_URL: https://sigopt.ninja:4443/api 51 | SIGOPT_API_VERIFY_SSL_CERTS: /home/runner/work/sigopt-python/sigopt-server/artifacts/tls/root-ca.crt 52 | steps: 53 | - name: Check out repository code 54 | uses: actions/checkout@v4 55 | - name: Set up Python 3.10 56 | uses: actions/setup-python@v5 57 | with: 58 | python-version: "3.10" 59 | - run: sudo apt-get install -yqq expect 60 | - run: echo '127.0.0.1 sigopt.ninja' | sudo tee -a /etc/hosts 61 | - run: git clone https://github.com/sigopt/sigopt-server ../sigopt-server 62 | - run: | 63 | cd ../sigopt-server 64 | git checkout main 65 | EDITOR=true ./setup.sh | tee setup_output.log 66 | ./start.sh & 67 | - run: | 68 | cd ../sigopt-server 69 | ./ci/wait_for.sh url "$SIGOPT_API_URL/health" 70 | - run: | 71 | EMAIL="$(tail -n 2 ../sigopt-server/setup_output.log | grep -E '^ email:' | awk -F': ' '{ print $2 }')" 72 | PASSWORD="$(tail -n 2 ../sigopt-server/setup_output.log | grep -E '^ password:' | awk -F': ' '{ print $2 }')" 73 | curl -fsk -XPOST "$SIGOPT_API_URL/v1/sessions" -d '{"email": "'"$EMAIL"'", "password": "'"$PASSWORD"'"}' | \ 74 | jq -r .api_token.token >sigopt_user_token.txt 75 | curl -fsk --user "$(cat sigopt_user_token.txt):" "$SIGOPT_API_URL/v1/clients/1/tokens" | \ 76 | jq -r '.data[] | select(.token_type == "client-api") | .token' >sigopt_api_token.txt 77 | - run: pip install '.[xgboost]' scikit-learn 78 | - run: env SIGOPT_API_TOKEN="$(cat sigopt_api_token.txt)" sigopt create project 79 | - run: env TEST_ACCOUNT_API_TOKEN="$(cat sigopt_api_token.txt)" ./ci/tutorial/sigopt_config.exp 80 | - run: sigopt run python ./ci/tutorial/model.py 81 | -------------------------------------------------------------------------------- /.github/workflows/scorecard.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. They are provided 2 | # by a third-party and are governed by separate terms of service, privacy 3 | # policy, and support documentation. 4 | 5 | name: Scorecard supply-chain security 6 | on: 7 | # For Branch-Protection check. Only the default branch is supported. See 8 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection 9 | branch_protection_rule: 10 | # To guarantee Maintained check is occasionally updated. See 11 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained 12 | schedule: 13 | - cron: '37 2 * * 5' 14 | push: 15 | branches: [ "main" ] 16 | 17 | # Declare default permissions as read only. 18 | permissions: read-all 19 | 20 | jobs: 21 | analysis: 22 | name: Scorecard analysis 23 | runs-on: ubuntu-latest 24 | permissions: 25 | # Needed to upload the results to code-scanning dashboard. 26 | security-events: write 27 | # Needed to publish results and get a badge (see publish_results below). 28 | id-token: write 29 | # Uncomment the permissions below if installing in a private repository. 30 | # contents: read 31 | # actions: read 32 | 33 | steps: 34 | - name: "Checkout code" 35 | uses: actions/checkout@v4 36 | with: 37 | persist-credentials: false 38 | 39 | - name: "Run analysis" 40 | uses: ossf/scorecard-action@v2.3.1 41 | with: 42 | results_file: results.sarif 43 | results_format: sarif 44 | # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: 45 | # - you want to enable the Branch-Protection check on a *public* repository, or 46 | # - you are installing Scorecard on a *private* repository 47 | # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. 48 | # repo_token: ${{ secrets.SCORECARD_TOKEN }} 49 | 50 | # Public repositories: 51 | # - Publish results to OpenSSF REST API for easy access by consumers 52 | # - Allows the repository to include the Scorecard badge. 53 | # - See https://github.com/ossf/scorecard-action#publishing-results. 54 | # For private repositories: 55 | # - `publish_results` will always be set to `false`, regardless 56 | # of the value entered here. 57 | publish_results: true 58 | 59 | # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF 60 | # format to the repository Actions tab. 61 | - name: "Upload artifact" 62 | uses: actions/upload-artifact@v4 63 | with: 64 | name: SARIF file 65 | path: results.sarif 66 | retention-days: 5 67 | 68 | # Upload the results to GitHub's code scanning dashboard. 69 | - name: "Upload to code-scanning" 70 | uses: github/codeql-action/upload-sarif@v3 71 | with: 72 | sarif_file: results.sarif 73 | -------------------------------------------------------------------------------- /.github/workflows/security.yml: -------------------------------------------------------------------------------- 1 | name: security checks 2 | permissions: read-all 3 | run-name: Security checks for ${{ github.repository }}@${{ github.ref }} 4 | on: 5 | push: {} 6 | schedule: 7 | - cron: "0 8,16 * * *" 8 | jobs: 9 | trivy-scan-fs: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out repository code 13 | uses: actions/checkout@v4 14 | - name: Run Trivy 15 | uses: aquasecurity/trivy-action@master 16 | with: 17 | scan-type: fs 18 | scan-ref: . 19 | trivy-config: trivy.yaml 20 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: SigOpt Python tests 2 | permissions: read-all 3 | run-name: ${{ github.actor }} is testing ${{ github.repository }}@${{ github.ref }} 4 | on: 5 | push: {} 6 | schedule: 7 | - cron: "0 8,16 * * *" 8 | jobs: 9 | pytest: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | python-version: 14 | - "3.9" 15 | - "3.10" 16 | - "3.11" 17 | - "3.12" 18 | - "3.13" 19 | test-suite: 20 | - cli 21 | - client 22 | - runs 23 | - validate 24 | - xgboost 25 | env: 26 | AWS_DEFAULT_REGION: us-east-1 27 | steps: 28 | - name: Check out repository code 29 | uses: actions/checkout@v4 30 | - name: Set up Python ${{ matrix.python-version }} 31 | uses: actions/setup-python@v5 32 | with: 33 | python-version: ${{ matrix.python-version }} 34 | - run: pip install '.[xgboost]' -r requirements-dev.txt 35 | - run: pytest -rw -v test/${{ matrix.test-suite }} 36 | pylint: 37 | runs-on: ubuntu-latest 38 | steps: 39 | - name: Check out repository code 40 | uses: actions/checkout@v4 41 | - name: Set up Python 3.10 42 | with: 43 | python-version: "3.10" 44 | uses: actions/setup-python@v5 45 | - name: Set up Python 3.10 46 | uses: actions/setup-python@v5 47 | with: 48 | python-version: "3.10" 49 | - run: pip install '.[dev]' 50 | - run: pre-commit run pylint --all-files 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .pytest_cache/ 3 | build/ 4 | dist/ 5 | *.egg-info/ 6 | *~ 7 | **/venv 8 | 9 | #ctags 10 | tags 11 | 12 | #vim related 13 | *.swp 14 | 15 | # PyCharm related stuff 16 | .idea* 17 | .cache/ 18 | .ipynb_checkpoints 19 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | skip: [detect-aws-credentials, pylint] 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks.git 5 | rev: v4.6.0 6 | hooks: 7 | - id: no-commit-to-branch 8 | args: [--branch=main] 9 | - id: check-merge-conflict 10 | - id: detect-private-key 11 | - id: detect-aws-credentials 12 | - id: check-added-large-files 13 | args: [--maxkb=100] 14 | exclude: "^LICENSES/third_party/third-party-licenses.txt$" 15 | - id: check-executables-have-shebangs 16 | - id: check-shebang-scripts-are-executable 17 | - id: check-toml 18 | - id: check-yaml 19 | - id: check-ast 20 | - id: debug-statements 21 | - id: end-of-file-fixer 22 | - id: mixed-line-ending 23 | args: [--fix=lf] 24 | - id: requirements-txt-fixer 25 | - id: trailing-whitespace 26 | - repo: https://github.com/sigopt/sigopt-tools.git 27 | rev: "v0.0.2" 28 | hooks: 29 | - id: copyright-license-disclaimer 30 | args: ["--license=MIT", "--owner=Intel Corporation"] 31 | - id: sigoptlint-python 32 | - id: sigoptlint-shell 33 | - repo: https://github.com/sigopt/black.git 34 | rev: sigopt-22.10.0 35 | hooks: 36 | - id: black 37 | args: [--preview] 38 | - repo: https://github.com/pycqa/flake8 39 | rev: 7.1.1 40 | hooks: 41 | - id: flake8 42 | - repo: https://github.com/PyCQA/isort.git 43 | rev: "5.13.2" 44 | hooks: 45 | - id: isort 46 | - repo: local 47 | hooks: 48 | - id: pylint 49 | name: pylint 50 | entry: env PYTHONPATH=./test pylint 51 | language: system 52 | types: [python] 53 | args: ["-rn", "-sn"] 54 | - repo: https://github.com/jendrikseipp/vulture.git 55 | rev: "v2.12" 56 | hooks: 57 | - id: vulture 58 | entry: tools/run_vulture.sh 59 | args: [.vulture_allowlist] 60 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [FORMAT] 2 | indent-string=' ' 3 | max-line-length=120 4 | 5 | [MESSAGES CONTROL] 6 | disable= 7 | abstract-method, 8 | broad-except, 9 | consider-using-with, 10 | duplicate-code, 11 | fixme, 12 | global-statement, 13 | import-outside-toplevel, 14 | invalid-name, 15 | missing-docstring, 16 | no-else-return, 17 | no-member, 18 | no-self-use, 19 | no-value-for-parameter, 20 | protected-access, 21 | redefined-outer-name, 22 | superfluous-parens, 23 | too-few-public-methods, 24 | too-many-arguments, 25 | too-many-branches, 26 | too-many-instance-attributes, 27 | too-many-locals, 28 | too-many-public-methods, 29 | too-many-return-statements, 30 | too-many-statements, 31 | unsubscriptable-object, 32 | unused-argument, 33 | unused-wildcard-import, 34 | useless-object-inheritance, 35 | wildcard-import, 36 | wrong-import-order, 37 | wrong-import-position 38 | -------------------------------------------------------------------------------- /.vulture_allowlist: -------------------------------------------------------------------------------- 1 | load_ipython_extension # unused function (sigopt/__init__.py:43) 2 | franke_function # unused function (sigopt/examples/franke.py:8) 3 | FRANKE_EXPERIMENT_DEFINITION # unused variable (sigopt/examples/franke.py:19) 4 | _.optimize # unused method (sigopt/magics.py:108) 5 | scale # unused variable (sigopt/objects.py:383) 6 | shape_a # unused variable (sigopt/objects.py:384) 7 | shape_b # unused variable (sigopt/objects.py:385) 8 | grid # unused variable (sigopt/objects.py:393) 9 | prior # unused variable (sigopt/objects.py:396) 10 | active_run_count # unused variable (sigopt/objects.py:410) 11 | finished_run_count # unused variable (sigopt/objects.py:411) 12 | total_run_count # unused variable (sigopt/objects.py:412) 13 | active_run_count # unused variable (sigopt/objects.py:417) 14 | finished_run_count # unused variable (sigopt/objects.py:418) 15 | total_run_count # unused variable (sigopt/objects.py:419) 16 | lookback_checkpoints # unused variable (sigopt/objects.py:453) 17 | min_checkpoints # unused variable (sigopt/objects.py:456) 18 | max_checkpoints # unused variable (sigopt/objects.py:461) 19 | early_stopping_criteria # unused variable (sigopt/objects.py:462) 20 | num_solutions # unused variable (sigopt/objects.py:483) 21 | observation_budget # unused variable (sigopt/objects.py:484) 22 | training_monitor # unused variable (sigopt/objects.py:491) 23 | num_solutions # unused variable (sigopt/objects.py:507) 24 | should_stop # unused variable (sigopt/objects.py:538) 25 | reasons # unused variable (sigopt/objects.py:539) 26 | should_stop # unused variable (sigopt/objects.py:600) 27 | stopping_reasons # unused variable (sigopt/objects.py:601) 28 | training_run # unused variable (sigopt/objects.py:602) 29 | email # unused variable (sigopt/objects.py:609) 30 | _.pool_classes_by_scheme # unused attribute (sigopt/request_driver.py:26) 31 | ConnectionCls # unused variable (sigopt/urllib3_patch.py:48) 32 | ConnectionCls # unused variable (sigopt/urllib3_patch.py:63) 33 | _.p1 # unused attribute (test/runs/test_factory.py:48) 34 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # default fallback for all files: 2 | * @sigopt/Dev 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2015 SigOpt Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE 3 | include setup.py 4 | include requirements.txt 5 | include requirements-dev.txt 6 | recursive-include sigopt *.py 7 | recursive-include test *.py 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test lint integration_test vulture vulture-allowlist 2 | 3 | test: 4 | @PYTHONPATH=. python -m pytest -rw -v test 5 | 6 | integration_test: 7 | @PYTHONPATH=. python -m pytest -rw -v integration_test 8 | 9 | vulture: 10 | @./tools/run_vulture.sh . .vulture_allowlist 11 | 12 | vulture-allowlist: 13 | @./tools/generate_vulture_allowlist > .vulture_allowlist 14 | 15 | update: 16 | @python setup.py clean --all 17 | @pip install -e '.[all]' 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/sigopt/sigopt-python/badge)](https://securityscorecards.dev/viewer/?uri=github.com/sigopt/sigopt-python) 8 | [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/sigopt/sigopt-python/main.svg)](https://results.pre-commit.ci/latest/github/sigopt/sigopt-python/main) 9 | ![integration](https://github.com/sigopt/sigopt-python/actions/workflows/integration.yml/badge.svg) 10 | ![tests](https://github.com/sigopt/sigopt-python/actions/workflows/tests.yml/badge.svg) 11 | 12 | # SigOpt Python API 13 | 14 | This is the [SigOpt](https://sigopt.com) Python API client. 15 | Use this to natively call SigOpt API endpoints to create experiments and report data. 16 | 17 | For more help getting started with SigOpt and Python, check out the [docs](https://docs.sigopt.com/core-module-api-references/get_started). 18 | 19 | Take a look in `examples` for example usage. 20 | 21 | ## Getting Started 22 | 23 | Install the sigopt python modules with `pip install sigopt`. 24 | 25 | Sign up for an account at [https://sigopt.com](https://sigopt.com). 26 | In order to use the API, you'll need your API token from the [API tokens page](https://sigopt.com/tokens). 27 | 28 | To call the API, instantiate a connection with your token. 29 | 30 | ### Authentication 31 | Authenticate each connection with your API token directly (will override any token set via environment variable): 32 | ```python 33 | from sigopt import Connection 34 | conn = Connection(client_token=api_token) 35 | ``` 36 | 37 | ### Authentication with Environment Variable 38 | Insert your API token into the environment variable `SIGOPT_API_TOKEN`, and instantiate a connection: 39 | 40 | ```python 41 | from sigopt import Connection 42 | conn = Connection() 43 | ``` 44 | 45 | 46 | ## Issuing Requests 47 | Then, you can use the connection to issue API requests. An example creating an experiment and running the 48 | optimization loop: 49 | 50 | ```python 51 | from sigopt import Connection 52 | from sigopt.examples import franke_function 53 | conn = Connection(client_token=SIGOPT_API_TOKEN) 54 | 55 | experiment = conn.experiments().create( 56 | name='Franke Optimization', 57 | parameters=[ 58 | dict(name='x', type='double', bounds=dict(min=0.0, max=1.0)), 59 | dict(name='y', type='double', bounds=dict(min=0.0, max=1.0)), 60 | ], 61 | metrics=[dict(name='f', objective='maximize')], 62 | ) 63 | print("Created experiment: https://sigopt.com/experiment/" + experiment.id); 64 | 65 | # Evaluate your model with the suggested parameter assignments 66 | # Franke function - http://www.sfu.ca/~ssurjano/franke2d.html 67 | def evaluate_model(assignments): 68 | return franke_function(assignments['x'], assignments['y']) 69 | 70 | # Run the Optimization Loop between 10x - 20x the number of parameters 71 | for _ in range(20): 72 | suggestion = conn.experiments(experiment.id).suggestions().create() 73 | value = evaluate_model(suggestion.assignments) 74 | conn.experiments(experiment.id).observations().create( 75 | suggestion=suggestion.id, 76 | values=[dict(name='f', value=value)], 77 | ) 78 | ``` 79 | 80 | ## API Token 81 | 82 | Your API token does not have permission to view or modify information about individual user accounts, 83 | so it is safe to include when running SigOpt in production. 84 | 85 | ## Endpoints 86 | 87 | Endpoints are grouped by objects on the `Connection`. 88 | For example, endpoints that interact with experiments are under `conn.experiments`. 89 | `ENDPOINT_GROUP(ID)` operates on a single object, while `ENDPOINT_GROUP()` will operate on multiple objects. 90 | 91 | `POST`, `GET`, `PUT` and `DELETE` translate to the method calls `create`, `fetch`, `update` and `delete`. 92 | To retrieve an experiment, call `conn.experiments(ID).fetch()`. To create an experiment call 93 | `conn.experiments(ID).create()`. Parameters are passed to the API as named arguments. 94 | 95 | Just like in the resource urls, `suggestions` and `observations` are under `experiments`. 96 | Access these objects with `conn.experiments(ID).suggestions` and `conn.experiments(ID).observations`. 97 | The REST endpoint `POST /v1/experiments/1/suggestions` then translates to `conn.experiments(ID).suggestions().create()`. 98 | 99 | ## Testing 100 | 101 | To run the included tests, just run 102 | 103 | ```bash 104 | pip install -r requirements-dev.txt 105 | make test 106 | ``` 107 | 108 | To lint, install requirements (included in the previous step) and run 109 | ```bash 110 | make lint 111 | ``` 112 | 113 | Use `vulture` to check no use code/paramters 114 | ```bash 115 | make vulture 116 | ``` 117 | 118 | Generate `vulture` allowlist 119 | ```bash 120 | make vulture-allowlist 121 | ``` 122 | 123 | The `vulture` allowlist file `.vulture_allowlist` can be edit to add/remove allowed no use code/parameters. 124 | 125 | ## Earlier versions 126 | Earlier versions supported a hyperopt integration. Since hyperopt now seems to be a retired project, we are removing this integration. If you need that integration please select version 8.8.3 127 | 128 | ## Acknowledgments 129 | 130 | We would like to thank the following people for their contributions: 131 | 132 | - [@aadamson](https://github.com/aadamson) for their contributions in supporting custom `requests.Session` objects [#170](https://github.com/sigopt/sigopt-python/pull/170) 133 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Security Policy 8 | 9 | Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 10 | 11 | # Reporting a Vulnerability 12 | 13 | Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). 14 | -------------------------------------------------------------------------------- /ci/tutorial/model.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | # model.py 5 | import sklearn.datasets 6 | import sklearn.metrics 7 | from xgboost import XGBClassifier # pylint: disable=no-name-in-module 8 | 9 | import sigopt 10 | 11 | 12 | # Data preparation required to run and evaluate the sample model 13 | X, y = sklearn.datasets.load_iris(return_X_y=True) 14 | Xtrain, ytrain = X[:100], y[:100] 15 | 16 | # Track the name of the dataset used for your Run 17 | sigopt.log_dataset("iris 2/3 training, full test") 18 | # Set n_estimators as the hyperparameter to explore for your Experiment 19 | sigopt.params.setdefault("n_estimators", 100) 20 | # Track the name of the model used for your Run 21 | sigopt.log_model("xgboost") 22 | 23 | # Instantiate and train your sample model 24 | model = XGBClassifier( 25 | n_estimators=sigopt.params.n_estimators, 26 | use_label_encoder=False, 27 | eval_metric="logloss", 28 | ) 29 | model.fit(Xtrain, ytrain) 30 | pred = model.predict(X) 31 | 32 | # Track the metric value and metric name for each Run 33 | sigopt.log_metric("accuracy", sklearn.metrics.accuracy_score(pred, y)) 34 | -------------------------------------------------------------------------------- /ci/tutorial/sigopt_config.exp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/expect 2 | 3 | spawn sigopt config 4 | 5 | expect "SigOpt API token (find at https://app.sigopt.com/tokens/info):" 6 | 7 | send "$env(TEST_ACCOUNT_API_TOKEN)\r" 8 | 9 | send "y\r" 10 | 11 | send "y\r" 12 | 13 | expect eof 14 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # SigOpt API Examples 8 | For the complete API documentation, visit [https://docs.sigopt.com](https://docs.sigopt.com). 9 | ## Python 10 | Check out our `basic` python example and implement your own function to evaluate. Dive deeper with our `parallel` example, which runs multiple optimization loops at the same time. 11 | ## Other Languages 12 | If your metric evaluation is in an executable file written in another language you can use the example `other_languages` as a starting point to perform metric evaluation in a sub process. 13 | 14 | This script will pass the suggested assignments as command line arguments, and will expect the script's output to be a float representing a fuction evaluated on these assignments. 15 | 16 | To start, you'll need to create an experiment with parameter names matching the command line arguments of your program. 17 | 18 | ### Example Usage 19 | 20 | For example, your filename is `test` and it expects an argument `x` on the command line that is a double. Say this script normally spews setup info, but if you run `./test --quiet`, the only information sent to stdout is the value of your function evalauted at `x`. Set up an experiment with one parameter named `x` that has type `double`, and run the following command: 21 | ``` 22 | python other_languages.py --command='./test --quiet' --experiment_id=EXPERIMENT_ID --client_token=$CLIENT_TOKEN 23 | ``` 24 | The above command will run the following sub process to evaluate your metric, automatially requesting the suggsetion beforehand and reporting the observation afterwards: 25 | ``` 26 | ./test --quiet --x=SUGGESTION_FOR_X 27 | ``` 28 | 29 | Feel free to use, or modify for your own needs! 30 | -------------------------------------------------------------------------------- /examples/basic.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import argparse 5 | 6 | import sigopt 7 | 8 | 9 | # Take a suggestion from sigopt and evaluate your function 10 | def execute_model(run): 11 | # train a model 12 | # evaluate a model 13 | # return the accuracy 14 | raise NotImplementedError("Return a number, which represents your metric for this run") 15 | 16 | 17 | if __name__ == "__main__": 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument("--budget", type=int, default=20) 20 | parser.add_argument( 21 | "--client_token", 22 | required=True, 23 | help="Find your CLIENT_TOKEN at https://sigopt.com/tokens", 24 | ) 25 | the_args = parser.parse_args() 26 | 27 | # Descriptor of what kind of dataset you are modeling 28 | sigopt.log_dataset("Example dataset") 29 | # Useful for keeping track of where you got the data 30 | sigopt.log_metadata(key="Dataset Source", value="Example Source") 31 | # e.g. Sklern, xgboost, etc. 32 | sigopt.log_metadata(key="Feature Pipeline Name", value="Example Pipeline") 33 | # What kind of learning you are attemping 34 | sigopt.log_model("Example Model Technique") 35 | # Create an experiment with one paramter, x 36 | experiment = sigopt.create_experiment( 37 | name="Basic Test experiment", 38 | parameters=[{"name": "x", "bounds": {"max": 50.0, "min": 0.0}, "type": "double"}], 39 | metrics=[{"name": "holdout_accuracy", "objective": "maximize"}], 40 | parallel_bandwidth=1, 41 | budget=the_args.budget, 42 | ) 43 | print("Created experiment id {0}".format(experiment.id)) 44 | 45 | # In a loop: receive a suggestion, evaluate the metric, report an observation 46 | for run in experiment.loop(): 47 | with run: 48 | holdout_accuracy = execute_model(run) 49 | run.log_metric("holdout_accuracy", holdout_accuracy) 50 | 51 | best_runs = experiment.get_best_runs() 52 | print(best_runs) 53 | -------------------------------------------------------------------------------- /examples/other_languages.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import argparse 5 | import sys 6 | from subprocess import PIPE, Popen 7 | 8 | from sigopt import Connection 9 | 10 | 11 | class SubProcessEvaluator(object): 12 | def __init__(self, command): 13 | self.command = command 14 | 15 | # Take a suggestion from sigopt and evaluate your function 16 | # Sends command line arguments to your executable file with the same names as the 17 | # parameters of your experiment. Expected output is one line containing a float that 18 | # is your function evaluated at the suggested assignments. 19 | # For example, if your command is './test' and you have one double parameter with suggested 20 | # value 11.05, this script will run 21 | # ./test --x=11.05 22 | def evaluate_metric(self, assignments): 23 | arguments = [ 24 | "--{}={}".format(param_name, assignment) for param_name, assignment in assignments.to_json().iteritems() 25 | ] 26 | process = Popen(self.command.split() + arguments, stdout=PIPE, stderr=PIPE) 27 | (stdoutdata, stderrdata) = process.communicate() 28 | sys.stderr.write(stderrdata) 29 | return float(stdoutdata.strip()) 30 | 31 | 32 | if __name__ == "__main__": 33 | parser = argparse.ArgumentParser() 34 | parser.add_argument( 35 | "--command", 36 | required=True, 37 | help=( 38 | "The command to run the function whose parameters you would like to" 39 | " optimize. Should accept parameters as command line argument and output" 40 | " only the evaluated metric at the suggested point." 41 | ), 42 | ) 43 | parser.add_argument( 44 | "--experiment_id", 45 | required=True, 46 | help=( 47 | "The parameters of this experiment should be the " 48 | "same type and name of the command line arguments to your executable file." 49 | ), 50 | ) 51 | parser.add_argument( 52 | "--client_token", 53 | required=True, 54 | help="Find your CLIENT_TOKEN at https://sigopt.com/tokens", 55 | ) 56 | the_args = parser.parse_args() 57 | 58 | connection = Connection(client_token=the_args.client_token) 59 | experiment = connection.experiments(the_args.experiment_id).fetch() 60 | connection.experiments(the_args.experiment_id).suggestions().delete(state="open") 61 | evaluator = SubProcessEvaluator(the_args.command) 62 | 63 | # In a loop: receive a suggestion, evaluate the metric, report an observation 64 | while True: 65 | suggestion = connection.experiments(experiment.id).suggestions().create() 66 | print("Evaluating at suggested assignments: {0}".format(suggestion.assignments)) 67 | value = evaluator.evaluate_metric(suggestion.assignments) 68 | print("Reporting observation of value: {0}".format(value)) 69 | connection.experiments(experiment.id).observations().create( 70 | suggestion=suggestion.id, 71 | value=value, 72 | ) 73 | -------------------------------------------------------------------------------- /examples/parallel.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import argparse 5 | 6 | import sigopt 7 | 8 | 9 | def run_command_on_machine(machine_number, command): 10 | # log into machine 11 | # execute command 12 | raise NotImplementedError("Log into the specified machines, execute the included command") 13 | 14 | 15 | if __name__ == "__main__": 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument("--budget", type=int, default=20) 18 | parser.add_argument( 19 | "--parallel_bandwidth", 20 | type=int, 21 | default=3, 22 | help="Number of machines you are running learning on", 23 | ) 24 | parser.add_argument( 25 | "--client_token", 26 | required=True, 27 | help="Find your CLIENT_TOKEN at https://sigopt.com/tokens", 28 | ) 29 | the_args = parser.parse_args() 30 | 31 | # Descriptor of what kind of dataset you are modeling 32 | sigopt.log_dataset("Example dataset") 33 | # Useful for keeping track of where you got the data 34 | sigopt.log_metadata(key="Dataset Source", value="Example Source") 35 | # e.g. Sklern, xgboost, etc. 36 | sigopt.log_metadata(key="Feature Pipeline Name", value="Example Pipeline") 37 | # What kind of learning you are attemping 38 | sigopt.log_model("Example Model Technique") 39 | # Create an experiment with one paramter, x 40 | experiment = sigopt.create_experiment( 41 | name="Basic Test experiment", 42 | type="offline", 43 | parameters=[{"name": "x", "bounds": {"max": 50.0, "min": 0.0}, "type": "double"}], 44 | metrics=[{"name": "holdout_accuracy", "objective": "maximize"}], 45 | parallel_bandwidth=the_args.parallel_bandwidth, 46 | budget=the_args.budget, 47 | ) 48 | print("Created experiment id {0}".format(experiment.id)) 49 | 50 | # In a loop: on each machine, start off a learning process, then each reports separately 51 | # to Sigopt the results 52 | for machine_number in range(experiment.parallel_bandwidth): 53 | run_command_on_machine( 54 | machine_number, 55 | f"sigopt start-worker {experiment.id} python run-model.py", 56 | ) 57 | 58 | best_runs = experiment.get_best_runs() 59 | print(best_runs) 60 | -------------------------------------------------------------------------------- /integration_test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-python/855b9a7a590162627ea2ef934f793a567a31b8be/integration_test/__init__.py -------------------------------------------------------------------------------- /integration_test/test_experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import sigopt 5 | 6 | 7 | class TestExperiment(object): 8 | def test_update(self): 9 | parameters = [ 10 | {"name": "max_depth", "type": "int", "bounds": {"min": 2, "max": 5}}, 11 | {"name": "num_boost_round", "type": "int", "bounds": {"min": 2, "max": 5}}, 12 | ] 13 | config = { 14 | "name": "experiment-integration-test", 15 | "type": "offline", 16 | "parameters": parameters, 17 | "metrics": [{"name": "f1", "strategy": "optimize", "objective": "maximize"}], 18 | "parallel_bandwidth": 1, 19 | "budget": 3, 20 | } 21 | experiment = sigopt.create_aiexperiment(**config) 22 | parameters = experiment.parameters 23 | parameters[0].bounds.max = 100 24 | parameters[1].bounds.min = 1 25 | new_config = { 26 | "name": "experiment-integration-test-1", 27 | "parameters": parameters, 28 | "parallel_bandwidth": 2, 29 | "budget": 4, 30 | } 31 | experiment.update(**new_config) 32 | updated_experiment = sigopt.get_aiexperiment(experiment.id) 33 | assert updated_experiment.name == "experiment-integration-test-1" 34 | assert updated_experiment.budget == 4 35 | assert updated_experiment.parallel_bandwidth == 2 36 | assert updated_experiment.parameters[0].bounds.min == 2 37 | assert updated_experiment.parameters[0].bounds.max == 100 38 | assert updated_experiment.parameters[1].bounds.min == 1 39 | assert updated_experiment.parameters[1].bounds.max == 5 40 | -------------------------------------------------------------------------------- /integration_test/test_sigopt.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import pytest 5 | 6 | import sigopt 7 | 8 | 9 | class TestSigOpt(object): 10 | def create_run(self, n): 11 | return { 12 | "name": f"batch-{n}", 13 | "assignments": { 14 | "x": n, 15 | "y": n * n, 16 | }, 17 | "values": { 18 | "r0": dict(value=n * n + n), 19 | "r1": dict(value=n**2), 20 | }, 21 | "state": "completed" if (n % 2 == 0) else "failed", 22 | } 23 | 24 | @pytest.mark.parametrize("n", [10, 11]) 25 | @pytest.mark.parametrize("max_batch_size", [2, 3, 10, 20]) 26 | def test_upload_runs(self, n, max_batch_size): 27 | runs = [self.create_run(i) for i in range(n)] 28 | training_runs = sigopt.upload_runs(runs, max_batch_size) 29 | assert len(training_runs) == n 30 | -------------------------------------------------------------------------------- /lint: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | set -o pipefail 4 | 5 | 6 | PYTHON_VERSION=$(python --version 2>&1) 7 | if [[ $PYTHON_VERSION =~ "Python 2.6" || $PYTHON_VERSION =~ "Python 3.2" ]]; then 8 | echo 'Skipping lint for unsupported python version' 9 | exit 0 10 | fi 11 | 12 | pylint sigopt controller/controller test integration_test -r n --rcfile=.pylintrc 13 | 14 | ./tools/check_copyright_and_license_disclaimers.py . 15 | -------------------------------------------------------------------------------- /publish: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -o pipefail 5 | 6 | VERSION=$1 7 | if [ "x" = "x$VERSION" ]; then 8 | echo 'Must provide version to deploy.' 9 | exit 1 10 | fi 11 | 12 | if [ "x-h" = "x$VERSION" ]; then 13 | echo 'usage: publish [-h] version' 14 | exit 0 15 | fi 16 | 17 | python setup.py sdist 18 | python setup.py bdist_wheel --universal 19 | 20 | echo 21 | echo 22 | 23 | echo 'Publishing the following files:' 24 | for FILE in `ls dist/*$VERSION*`; do 25 | echo " $FILE" 26 | done 27 | 28 | twine upload "dist/*$VERSION*" 29 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | title = 'SigOpt Python' 2 | 3 | [tool.black] 4 | line-length = 120 5 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # For continuous integration and development 2 | 3 | matplotlib>=3.3.4 4 | mock>=3.0.5 5 | nose==1.3.7 6 | notebook 7 | numpy>=1.15.0,<2.0.0 8 | Pillow 9 | pre-commit>=2.5.2,<3 10 | pylint==2.9.6 11 | pyspark 12 | pytest>=7.2.1 13 | scikit-learn>=1.5.0,<2 14 | setuptools>=47.3.1 15 | twine>=3.2.0,<4.0.0 16 | vulture==2.7 17 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | backoff>=1.10.0,<2.0.0 2 | click>=8.0.0 3 | GitPython>=2.0.0 4 | packaging>=21.3 5 | pypng>=0.0.20 6 | PyYAML>=5,<7 7 | requests>=2.25.0,<3.0.0 8 | urllib3>=1.26.5,<2.0.0 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore=E129,E127,E203,E302,E131,E111,E114,E121,E501,E126,E123,E305,E402,I101,I100,N806,F403,E241,E731,F999,F401,F405,W503,E741,W504,E124,E231 3 | exclude=*_pb2.py 4 | 5 | [isort] 6 | combine_star=True 7 | force_grid_wrap=0 8 | include_trailing_comma=True 9 | indent=' ' 10 | line_length=120 11 | lines_after_imports=2 12 | multi_line_output=3 13 | use_parentheses=True 14 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | import sys 6 | import warnings 7 | 8 | from setuptools import find_packages, setup 9 | 10 | 11 | if sys.version_info[0] == 3 and sys.version_info[1] < 7: 12 | warnings.warn( 13 | ( 14 | "Python versions lower than 3.7 are no longer supported. Please upgrade to" 15 | " Python 3.7 or newer or use an older version of the sigopt-python client." 16 | ), 17 | DeprecationWarning, 18 | ) 19 | 20 | # NOTE: We can't `import sigopt.version` directly, because that 21 | # will cause us to execute `sigopt/__init__.py`, which may transitively import 22 | # packages that may not have been installed yet. So jump straight to sigopt/version.py 23 | # and execute that directly, which should be simple enough that it doesn't import anything 24 | # Learned from https://github.com/stripe/stripe-python (MIT licensed) 25 | version_contents = {} 26 | here = os.path.abspath(os.path.dirname(__file__)) 27 | with open(os.path.join(here, "sigopt", "version.py"), encoding="utf-8") as f: 28 | exec(f.read(), version_contents) # pylint: disable=exec-used 29 | VERSION = version_contents["VERSION"] 30 | 31 | with open(os.path.join(here, "requirements.txt")) as requirements_fp: 32 | install_requires = requirements_fp.read().split("\n") 33 | with open(os.path.join(here, "requirements-dev.txt")) as requirements_dev_fp: 34 | dev_install_requires = requirements_dev_fp.read().split("\n") 35 | 36 | xgboost_install_requires = ["xgboost>=1.3.1", "numpy>=1.15.0"] 37 | lite_install_requires = ["sigoptlite>=0.1.1"] 38 | 39 | setup( 40 | name="sigopt", 41 | version=VERSION, 42 | description="SigOpt Python API Client", 43 | author="SigOpt", 44 | author_email="support@sigopt.com", 45 | url="https://sigopt.com/", 46 | packages=find_packages(exclude=["tests*"]), 47 | package_data={ 48 | "": ["*.ms", "*.txt", "*.yml", "*.yaml"], 49 | }, 50 | install_requires=install_requires, 51 | extras_require={ 52 | "dev": dev_install_requires + xgboost_install_requires + lite_install_requires, 53 | "xgboost": xgboost_install_requires, 54 | "lite": lite_install_requires, 55 | }, 56 | entry_points={ 57 | "console_scripts": [ 58 | "sigopt=sigopt.cli.__main__:sigopt_cli", 59 | ], 60 | }, 61 | classifiers=[ 62 | "Development Status :: 5 - Production/Stable", 63 | "Intended Audience :: Developers", 64 | "License :: OSI Approved :: MIT License", 65 | "Operating System :: OS Independent", 66 | "Programming Language :: Python", 67 | "Topic :: Software Development :: Libraries :: Python Modules", 68 | ], 69 | ) 70 | -------------------------------------------------------------------------------- /sigopt/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import warnings 5 | 6 | from .config import config 7 | from .defaults import get_default_project 8 | from .factory import SigOptFactory 9 | from .interface import Connection 10 | from .run_context import global_run_context as _global_run_context 11 | from .sigopt_logging import enable_print_logging 12 | from .version import VERSION 13 | 14 | 15 | params = _global_run_context.params 16 | log_checkpoint = _global_run_context.log_checkpoint 17 | log_dataset = _global_run_context.log_dataset 18 | log_failure = _global_run_context.log_failure 19 | log_image = _global_run_context.log_image 20 | log_metadata = _global_run_context.log_metadata 21 | log_metric = _global_run_context.log_metric 22 | log_metrics = _global_run_context.log_metrics 23 | log_model = _global_run_context.log_model 24 | config.set_context_entry(_global_run_context) 25 | 26 | _global_factory = SigOptFactory(get_default_project()) 27 | create_run = _global_factory.create_run 28 | create_aiexperiment = _global_factory.create_aiexperiment 29 | create_experiment = _global_factory.create_experiment 30 | create_project = _global_factory.create_project 31 | get_aiexperiment = _global_factory.get_aiexperiment 32 | get_experiment = _global_factory.get_experiment 33 | archive_aiexperiment = _global_factory.archive_aiexperiment 34 | archive_experiment = _global_factory.archive_experiment 35 | unarchive_aiexperiment = _global_factory.unarchive_aiexperiment 36 | unarchive_experiment = _global_factory.unarchive_experiment 37 | archive_run = _global_factory.archive_run 38 | unarchive_run = _global_factory.unarchive_run 39 | get_run = _global_factory.get_run 40 | upload_runs = _global_factory.upload_runs 41 | 42 | 43 | def load_ipython_extension(ipython): 44 | from .magics import SigOptMagics as _Magics 45 | 46 | ipython.register_magics(_Magics) 47 | enable_print_logging() 48 | 49 | 50 | def get_run_id(): 51 | return _global_run_context.id 52 | 53 | 54 | def set_project(project): 55 | if get_run_id() is not None: 56 | warnings.warn( 57 | ( 58 | "set_project does nothing when your code is executed with the SigOpt" 59 | " CLI. Set the SIGOPT_PROJECT environment variable or use the --project" 60 | " CLI option instead." 61 | ), 62 | UserWarning, 63 | ) 64 | return _global_factory.set_project(project) 65 | -------------------------------------------------------------------------------- /sigopt/aiexperiment_context.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import json 5 | import threading 6 | 7 | from .objects import Parameter 8 | from .run_context import global_run_context 9 | from .run_factory import BaseRunFactory 10 | from .validate.aiexperiment_input import validate_aiexperiment_update_input 11 | 12 | 13 | class AIExperimentContext(BaseRunFactory): 14 | """Wraps the AIExperiment object and provides extra utility methods.""" 15 | 16 | def __init__(self, aiexperiment, connection): 17 | self._aiexperiment = aiexperiment 18 | self._refresh_lock = threading.Lock() 19 | self._connection = connection 20 | 21 | def refresh(self): 22 | """Refresh the state of the AIExperiment from the SigOpt API.""" 23 | connection = self._connection 24 | with self._refresh_lock: 25 | self._aiexperiment = connection.aiexperiments(self.id).fetch() 26 | 27 | def is_finished(self): 28 | """Check if the AIExperiment has consumed its entire budget.""" 29 | self.refresh() 30 | return self.progress.remaining_budget is not None and self.progress.remaining_budget <= 0 31 | 32 | def loop(self, name=None): 33 | """Create runs until the AIExperiment has finished.""" 34 | while not self.is_finished(): 35 | yield self.create_run(name=name) 36 | 37 | def archive(self): 38 | connection = self._connection 39 | connection.aiexperiments(self.id).delete() 40 | self.refresh() 41 | 42 | @property 43 | def project(self): 44 | # delegate to __getattr__ 45 | raise AttributeError 46 | 47 | def __getattr__(self, attr): 48 | return getattr(self._aiexperiment, attr) 49 | 50 | def _create_run(self, name, metadata): 51 | aiexperiment = self._aiexperiment 52 | connection = self._connection 53 | run = ( 54 | connection.aiexperiments(aiexperiment.id) 55 | .training_runs() 56 | .create( 57 | name=name, 58 | metadata=metadata, 59 | ) 60 | ) 61 | run_context = self.run_context_class(connection, run, global_run_context.params) 62 | return run_context 63 | 64 | def get_runs(self): 65 | return ( 66 | self._connection.clients(self.client) 67 | .projects(self.project) 68 | .training_runs() 69 | .fetch( 70 | filters=json.dumps( 71 | [ 72 | { 73 | "field": "experiment", 74 | "operator": "==", 75 | "value": self.id, 76 | } 77 | ] 78 | ) 79 | ) 80 | .iterate_pages() 81 | ) 82 | 83 | def get_best_runs(self): 84 | return self._connection.aiexperiments(self.id).best_training_runs().fetch().iterate_pages() 85 | 86 | def _parse_parameter(self, parameter): 87 | if isinstance(parameter, Parameter): 88 | parameter = parameter.as_json(parameter) 89 | for attr in ["constraints", "conditions"]: 90 | if not parameter.get(attr): 91 | parameter.pop(attr, None) 92 | return parameter 93 | 94 | def update(self, **kwargs): 95 | if "parameters" in kwargs: 96 | parameters = [self._parse_parameter(p) for p in kwargs["parameters"]] 97 | kwargs["parameters"] = parameters 98 | kwargs = validate_aiexperiment_update_input(kwargs) 99 | return self._connection.aiexperiments(self.id).update(**kwargs) 100 | -------------------------------------------------------------------------------- /sigopt/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from sigopt.cli.commands import sigopt_cli as cli 5 | -------------------------------------------------------------------------------- /sigopt/cli/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .commands import sigopt_cli 5 | 6 | 7 | if __name__ == "__main__": 8 | sigopt_cli() 9 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .commands import commands_argument 5 | from .experiment_file import experiment_file_option 6 | from .experiment_id import experiment_id_argument 7 | from .load_yaml import load_yaml_callback 8 | from .project import project_name_option, project_option 9 | from .run_file import run_file_option 10 | from .source_file import source_file_option 11 | from .validate import validate_id, validate_ids 12 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/commands.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | 7 | commands_argument = click.argument("commands", nargs=-1, type=click.UNPROCESSED) 8 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/experiment_file.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.validate import validate_aiexperiment_input 7 | 8 | from .load_yaml import load_yaml_callback 9 | 10 | 11 | experiment_file_option = click.option( 12 | "-e", 13 | "--experiment-file", 14 | default="experiment.yml", 15 | type=click.Path(exists=True), 16 | callback=load_yaml_callback(validate_aiexperiment_input), 17 | help="A YAML file that defines your AIExperiment.", 18 | ) 19 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/experiment_id.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from .validate import validate_id 7 | 8 | 9 | experiment_id_argument = click.argument("EXPERIMENT_ID", callback=validate_id) 10 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/load_yaml.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import errno 5 | 6 | import click 7 | import yaml 8 | 9 | from sigopt.validate import ValidationError 10 | 11 | 12 | class ValidatedData: 13 | def __init__(self, filename, validated_data): 14 | self.filename = filename 15 | self.data = validated_data 16 | 17 | 18 | def load_yaml(filename, validator, ignore_no_file): 19 | if filename is None: 20 | return None 21 | try: 22 | with open(filename) as yaml_fp: 23 | data = yaml.safe_load(yaml_fp) 24 | except OSError as ose: 25 | if ose.errno == errno.ENOENT and ignore_no_file: 26 | return None 27 | raise click.BadParameter(f"Could not open {filename}: {ose}") from ose 28 | except (yaml.parser.ParserError, yaml.scanner.ScannerError) as pe: 29 | raise click.BadParameter(f"Could not parse {filename}: {pe}") from pe 30 | 31 | try: 32 | validated_data = validator(data) 33 | except ValidationError as ve: 34 | raise click.BadParameter(f"Bad format in {filename}: {ve}") from ve 35 | 36 | return ValidatedData(filename, validated_data) 37 | 38 | 39 | def load_yaml_callback(validator, ignore_no_file=False): 40 | return lambda ctx, p, value: load_yaml(value, validator, ignore_no_file=ignore_no_file) 41 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/project.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.defaults import check_valid_project_id, get_default_project 7 | 8 | 9 | def validate_project_id_callback(ctx, p, value): # pylint: disable=unused-argument 10 | if value is None: 11 | return get_default_project() 12 | try: 13 | check_valid_project_id(value) 14 | except ValueError as ve: 15 | raise click.BadParameter(str(ve)) from ve 16 | return value 17 | 18 | 19 | project_option = click.option( 20 | "-p", 21 | "--project", 22 | callback=validate_project_id_callback, 23 | help=""" 24 | Provide the project ID. 25 | Projects can be created at https://app.sigopt.com/projects or with the command `sigopt create project`. 26 | If a project ID is not provided then the project ID is determined in the following order: 27 | first from the SIGOPT_PROJECT environment variable, then by the name of the current directory. 28 | """, 29 | ) 30 | 31 | 32 | def validate_project_name_callback(ctx, p, value): # pylint: disable=unused-argument 33 | if value is None: 34 | return get_default_project() 35 | return value 36 | 37 | 38 | project_name_option = click.option( 39 | "--project-name", 40 | callback=validate_project_name_callback, 41 | help="The name of the project.", 42 | ) 43 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/run_file.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.validate import validate_run_input 7 | 8 | from .load_yaml import load_yaml_callback 9 | 10 | 11 | run_file_option = click.option( 12 | "-r", 13 | "--run-file", 14 | default="run.yml", 15 | type=click.Path(), 16 | callback=load_yaml_callback(validate_run_input, ignore_no_file=True), 17 | help="A YAML file that defines your run. The contents will be stored as data on your run.", 18 | ) 19 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/source_file.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | 7 | def file_contents(ctx, param, value): # pylint: disable=unused-argument 8 | if value is None: 9 | return None 10 | with open(value, "r") as fp: 11 | return fp.read() 12 | 13 | 14 | source_file_option = click.option( 15 | "-s", 16 | "--source-file", 17 | type=click.Path(exists=True), 18 | callback=file_contents, 19 | help="A file containing the source code for your run. The contents will be stored as data on your run.", 20 | ) 21 | -------------------------------------------------------------------------------- /sigopt/cli/arguments/validate.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | 7 | def validate_id(ctx, param, value): 8 | if value.isdigit(): 9 | return value 10 | raise click.BadParameter("ID must be a string of digits") 11 | 12 | 13 | def validate_ids(ctx, param, value): 14 | return [validate_id(ctx, param, item) for item in value] 15 | -------------------------------------------------------------------------------- /sigopt/cli/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import sigopt.cli.commands.config 5 | import sigopt.cli.commands.experiment 6 | import sigopt.cli.commands.init 7 | import sigopt.cli.commands.local 8 | import sigopt.cli.commands.project 9 | import sigopt.cli.commands.training_run 10 | import sigopt.cli.commands.version 11 | 12 | from .base import sigopt_cli 13 | -------------------------------------------------------------------------------- /sigopt/cli/commands/base.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.config import config 7 | 8 | from ..utils import setup_cli 9 | 10 | 11 | @click.group() 12 | def sigopt_cli(): 13 | setup_cli(config) 14 | 15 | 16 | @sigopt_cli.group("create") 17 | def create_command(): 18 | """Commands for creating SigOpt Objects.""" 19 | 20 | 21 | @sigopt_cli.group("archive") 22 | def archive_command(): 23 | """Commands for archiving SigOpt Objects.""" 24 | 25 | 26 | @sigopt_cli.group("unarchive") 27 | def unarchive_command(): 28 | """Commands for unarchiving SigOpt Objects.""" 29 | -------------------------------------------------------------------------------- /sigopt/cli/commands/config.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from ...config import config as _config 7 | from .base import sigopt_cli 8 | 9 | 10 | API_TOKEN_PROMPT = "SigOpt API token (find at https://app.sigopt.com/tokens/info)" 11 | 12 | LOG_COLLECTION_PROMPT = """Log Collection 13 | \tThis will capture and upload the standard output and standard error of your 14 | \tRuns from the CLI and notebook cells so that you can view them on the SigOpt dashboard. 15 | Enable log collection""" 16 | 17 | CELL_TRACKING_PROMPT = """Notebook Cell Tracking 18 | \tThis will record and upload the content of your notebook cells so that you can view them 19 | \ton the SigOpt dashboard. 20 | Enable cell tracking""" 21 | 22 | 23 | @sigopt_cli.command() 24 | @click.option("--api-token", prompt=API_TOKEN_PROMPT) 25 | @click.option( 26 | "--enable-log-collection/--no-enable-log-collection", 27 | prompt=LOG_COLLECTION_PROMPT, 28 | ) 29 | @click.option( 30 | "--enable-cell-tracking/--no-enable-cell-tracking", 31 | prompt=CELL_TRACKING_PROMPT, 32 | ) 33 | def config(api_token, enable_log_collection, enable_cell_tracking): 34 | """Configure the SigOpt client.""" 35 | _config.persist_configuration_options( 36 | { 37 | _config.API_TOKEN_KEY: api_token, 38 | _config.CELL_TRACKING_ENABLED_KEY: enable_cell_tracking, 39 | _config.LOG_COLLECTION_ENABLED_KEY: enable_log_collection, 40 | } 41 | ) 42 | -------------------------------------------------------------------------------- /sigopt/cli/commands/experiment/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import sigopt.cli.commands.experiment.archive 5 | import sigopt.cli.commands.experiment.create 6 | import sigopt.cli.commands.experiment.unarchive 7 | -------------------------------------------------------------------------------- /sigopt/cli/commands/experiment/archive.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.factory import SigOptFactory 7 | 8 | from ...arguments import project_option, validate_ids 9 | from ..base import archive_command 10 | 11 | 12 | @archive_command.command("experiment") 13 | @project_option 14 | @click.argument("EXPERIMENT_IDS", nargs=-1, callback=validate_ids) 15 | def archive(project, experiment_ids): 16 | """Archive SigOpt Experiments.""" 17 | factory = SigOptFactory(project) 18 | factory.set_up_cli() 19 | for experiment_id in experiment_ids: 20 | try: 21 | factory.connection.experiments(experiment_id).delete() 22 | except Exception as e: 23 | raise click.ClickException(f"experiment_id: {experiment_id}, {e}") from e 24 | -------------------------------------------------------------------------------- /sigopt/cli/commands/experiment/create.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from sigopt.sigopt_logging import print_logger 5 | 6 | from ...arguments import experiment_file_option, project_option 7 | from ...utils import create_aiexperiment_from_validated_data 8 | from ..base import create_command 9 | 10 | 11 | def print_start_worker_help(aiexperiment): 12 | msg = f""" 13 | You can now start workers for this experiment with the following CLI command: 14 | > sigopt start-worker {aiexperiment.id} 15 | 16 | Or use the python client library: 17 | 18 | #/usr/bin/env python3 19 | import sigopt 20 | experiment = sigopt.get_experiment({aiexperiment.id!r}) 21 | for run in experiment.loop(): 22 | with run: 23 | ... 24 | """ 25 | print_logger.info(msg) 26 | 27 | 28 | @create_command.command("experiment") 29 | @experiment_file_option 30 | @project_option 31 | def create(experiment_file, project): 32 | """Create a SigOpt AIExperiment.""" 33 | aiexperiment = create_aiexperiment_from_validated_data(experiment_file, project) 34 | print_start_worker_help(aiexperiment) 35 | -------------------------------------------------------------------------------- /sigopt/cli/commands/experiment/unarchive.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.factory import SigOptFactory 7 | 8 | from ...arguments import project_option, validate_ids 9 | from ..base import unarchive_command 10 | 11 | 12 | @unarchive_command.command("experiment") 13 | @project_option 14 | @click.argument("EXPERIMENT_IDS", nargs=-1, callback=validate_ids) 15 | def unarchive(project, experiment_ids): 16 | """Unarchive SigOpt Experiments.""" 17 | factory = SigOptFactory(project) 18 | factory.set_up_cli() 19 | for experiment_id in experiment_ids: 20 | try: 21 | factory.unarchive_aiexperiment(experiment_id) 22 | except Exception as e: 23 | raise click.ClickException(f"experiment_id: {experiment_id}, {e}") from e 24 | -------------------------------------------------------------------------------- /sigopt/cli/commands/init.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | 6 | import click 7 | import pkg_resources 8 | 9 | from .base import sigopt_cli 10 | 11 | 12 | def write_file(path, resource): 13 | if os.path.exists(path): 14 | if not os.path.isfile(path): 15 | raise click.ClickException(f"{path} already exists and is not a file.") 16 | should_write = click.prompt( 17 | f"{path} already exists. Replace it? (Y/n)", 18 | type=bool, 19 | ) 20 | else: 21 | should_write = True 22 | if should_write: 23 | contents = pkg_resources.resource_string("sigopt.cli.resources", resource) 24 | with open(path, "wb") as fp: 25 | fp.write(contents) 26 | print(f"Wrote file contents for {path}") 27 | else: 28 | print(f"Skipping {path}") 29 | 30 | 31 | @sigopt_cli.command() 32 | def init(): 33 | """Initialize a directory for a SigOpt project.""" 34 | write_file("run.yml", "init_run.txt") 35 | write_file("experiment.yml", "init_experiment.txt") 36 | write_file("Dockerfile", "init_dockerfile.txt") 37 | write_file(".dockerignore", "init_dockerignore.txt") 38 | -------------------------------------------------------------------------------- /sigopt/cli/commands/local/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import sigopt.cli.commands.local.optimize 5 | import sigopt.cli.commands.local.run 6 | import sigopt.cli.commands.local.start_worker 7 | -------------------------------------------------------------------------------- /sigopt/cli/commands/local/optimize.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from sigopt.config import config 5 | 6 | from ...arguments import project_option, source_file_option 7 | from ...utils import cli_experiment_loop, create_aiexperiment_from_validated_data 8 | from ..base import sigopt_cli 9 | from ..optimize_base import optimize_command 10 | 11 | 12 | @sigopt_cli.command( 13 | context_settings=dict( 14 | allow_interspersed_args=False, 15 | ignore_unknown_options=True, 16 | ) 17 | ) 18 | @optimize_command 19 | @source_file_option 20 | @project_option 21 | def optimize(command, run_options, experiment_file, source_file, project): 22 | """Run a SigOpt AIExperiment. Requires a path to an experiment YAML file.""" 23 | experiment = create_aiexperiment_from_validated_data(experiment_file, project) 24 | cli_experiment_loop(config, experiment, command, run_options, source_file) 25 | -------------------------------------------------------------------------------- /sigopt/cli/commands/local/run.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from sigopt.config import config 5 | from sigopt.factory import SigOptFactory 6 | 7 | from ...arguments import project_option, source_file_option 8 | from ...utils import run_user_program 9 | from ..base import sigopt_cli 10 | from ..run_base import run_command 11 | 12 | 13 | @sigopt_cli.command( 14 | context_settings=dict( 15 | allow_interspersed_args=False, 16 | ignore_unknown_options=True, 17 | ) 18 | ) 19 | @run_command 20 | @source_file_option 21 | @project_option 22 | def run(command, run_options, source_file, project): 23 | """Create a SigOpt Run.""" 24 | factory = SigOptFactory(project) 25 | factory.set_up_cli() 26 | with factory.create_run(name=run_options.get("name")) as run_context: 27 | run_user_program(config, run_context, command, source_file) 28 | -------------------------------------------------------------------------------- /sigopt/cli/commands/local/start_worker.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.config import config 7 | from sigopt.factory import SigOptFactory 8 | 9 | from ...arguments import experiment_id_argument, source_file_option 10 | from ...utils import cli_experiment_loop 11 | from ..base import sigopt_cli 12 | from ..run_base import run_command 13 | 14 | 15 | @sigopt_cli.command( 16 | context_settings=dict( 17 | allow_interspersed_args=False, 18 | ignore_unknown_options=True, 19 | ) 20 | ) 21 | @experiment_id_argument 22 | @run_command 23 | @source_file_option 24 | def start_worker(experiment_id, command, run_options, source_file): 25 | """Start a worker for the given AIExperiment.""" 26 | factory = SigOptFactory.from_default_project() 27 | factory.set_up_cli() 28 | try: 29 | experiment = factory.get_aiexperiment(experiment_id) 30 | except ValueError as ve: 31 | raise click.ClickException(str(ve)) 32 | cli_experiment_loop(config, experiment, command, run_options, source_file) 33 | -------------------------------------------------------------------------------- /sigopt/cli/commands/optimize_base.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from ..arguments import experiment_file_option 5 | from .run_base import run_command 6 | 7 | 8 | def optimize_command(f): 9 | f = run_command(f) 10 | f = experiment_file_option(f) 11 | 12 | return f 13 | -------------------------------------------------------------------------------- /sigopt/cli/commands/project/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import sigopt.cli.commands.project.create 5 | -------------------------------------------------------------------------------- /sigopt/cli/commands/project/create.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.exception import ConflictingProjectException 7 | from sigopt.factory import SigOptFactory 8 | from sigopt.sigopt_logging import print_logger 9 | 10 | from ...arguments import project_name_option, project_option 11 | from ..base import create_command 12 | 13 | 14 | @create_command.command("project") 15 | @project_option 16 | @project_name_option 17 | def create(project, project_name): 18 | """Create a SigOpt Project.""" 19 | factory = SigOptFactory(project) 20 | try: 21 | factory.create_project(name=project_name) 22 | except ConflictingProjectException as cpe: 23 | raise click.ClickException(cpe) from cpe 24 | print_logger.info("Project '%s' created.", project) 25 | print_logger.info("To use this project, set the SIGOPT_PROJECT environment variable:") 26 | print_logger.info("> export SIGOPT_PROJECT='%s'", project) 27 | -------------------------------------------------------------------------------- /sigopt/cli/commands/run_base.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import functools 5 | 6 | import click 7 | 8 | from ..arguments import commands_argument, run_file_option 9 | 10 | 11 | def run_command(f): 12 | @commands_argument 13 | @run_file_option 14 | @functools.wraps(f) 15 | def wrapper(*args, commands, run_file, **kwargs): 16 | if run_file: 17 | run_options = run_file.data 18 | else: 19 | run_options = {} 20 | if not commands: 21 | try: 22 | commands = run_options["run"] 23 | except KeyError as ke: 24 | raise click.UsageError( 25 | "Missing command: Please specify your run command via arguments or in the 'run' section of the run file." 26 | ) from ke 27 | return f(*args, command=commands, run_options=run_options, **kwargs) 28 | 29 | return wrapper 30 | -------------------------------------------------------------------------------- /sigopt/cli/commands/training_run/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import sigopt.cli.commands.training_run.archive 5 | import sigopt.cli.commands.training_run.unarchive 6 | -------------------------------------------------------------------------------- /sigopt/cli/commands/training_run/archive.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.factory import SigOptFactory 7 | 8 | from ...arguments import project_option, validate_ids 9 | from ..base import archive_command 10 | 11 | 12 | @archive_command.command("run") 13 | @project_option 14 | @click.argument("RUN_IDS", nargs=-1, callback=validate_ids) 15 | def archive(project, run_ids): 16 | """Archive SigOpt Runs.""" 17 | factory = SigOptFactory(project) 18 | factory.set_up_cli() 19 | for run_id in run_ids: 20 | try: 21 | factory.archive_run(run_id) 22 | except Exception as e: 23 | raise click.ClickException(f"run_id: {run_id}, {e}") from e 24 | -------------------------------------------------------------------------------- /sigopt/cli/commands/training_run/unarchive.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import click 5 | 6 | from sigopt.factory import SigOptFactory 7 | 8 | from ...arguments import project_option, validate_ids 9 | from ..base import unarchive_command 10 | 11 | 12 | @unarchive_command.command("run") 13 | @project_option 14 | @click.argument("RUN_IDS", nargs=-1, callback=validate_ids) 15 | def unarchive(project, run_ids): 16 | """Unarchive SigOpt Runs.""" 17 | factory = SigOptFactory(project) 18 | factory.set_up_cli() 19 | for run_id in run_ids: 20 | try: 21 | factory.unarchive_run(run_id) 22 | except Exception as e: 23 | raise click.ClickException(f"run_id: {run_id}, {e}") from e 24 | -------------------------------------------------------------------------------- /sigopt/cli/commands/version.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from sigopt.version import VERSION 5 | 6 | from .base import sigopt_cli 7 | 8 | 9 | @sigopt_cli.command() 10 | def version(): 11 | """Show the installed SigOpt version.""" 12 | print(VERSION) 13 | -------------------------------------------------------------------------------- /sigopt/cli/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-python/855b9a7a590162627ea2ef934f793a567a31b8be/sigopt/cli/resources/__init__.py -------------------------------------------------------------------------------- /sigopt/cli/resources/init_dockerfile.txt: -------------------------------------------------------------------------------- 1 | # This file defines the environment that your model will run in 2 | # See the Dockerfile reference for more info https://docs.docker.com/engine/reference/builder/#format 3 | 4 | # The FROM line defines the starting image. 5 | FROM python:3.9 6 | 7 | RUN mkdir -p /sigopt 8 | WORKDIR /sigopt 9 | 10 | RUN pip install --no-cache-dir --user sigopt 11 | 12 | # Uncomment/modify these lines to install your system dependencies. 13 | # RUN set -ex; apt-get -y update; apt-get -y install gcc 14 | 15 | # Uncomment/modify these lines to install your python dependencies. 16 | # COPY requirements.txt /sigopt/requirements.txt 17 | # RUN pip install --no-cache-dir --user -r requirements.txt 18 | 19 | # copy your code into the Dockerfile 20 | COPY . /sigopt 21 | -------------------------------------------------------------------------------- /sigopt/cli/resources/init_dockerignore.txt: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .git 3 | .venv 4 | __pycache__ 5 | venv 6 | -------------------------------------------------------------------------------- /sigopt/cli/resources/init_experiment.txt: -------------------------------------------------------------------------------- 1 | name: My Experiment 2 | metrics: 3 | - name: #[METRIC_NAME] 4 | parameters: 5 | - name: #[PARAMETER_NAME] 6 | type: #[PARMETER_TYPE] 7 | bounds: 8 | min: #[MIN_BOUND] 9 | max: #[MAX_BOUND] 10 | parallel_bandwidth: 1 11 | budget: 10 12 | -------------------------------------------------------------------------------- /sigopt/cli/resources/init_run.txt: -------------------------------------------------------------------------------- 1 | name: My Run 2 | # Fill in run command/commands 3 | run: # python model.py 4 | image: #[IMAGE] 5 | -------------------------------------------------------------------------------- /sigopt/cli/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import errno 5 | import io 6 | import os 7 | import shlex 8 | import signal 9 | import subprocess # nosec 10 | import sys 11 | import threading 12 | 13 | import click 14 | 15 | from sigopt.factory import SigOptFactory 16 | from sigopt.run_context import GlobalRunContext 17 | from sigopt.sigopt_logging import enable_print_logging, print_logger 18 | 19 | from .arguments.load_yaml import ValidatedData 20 | 21 | 22 | class StreamThread(threading.Thread): 23 | def __init__(self, input_stream, output_stream): 24 | super().__init__() 25 | self.input_stream = input_stream 26 | self.output_stream = output_stream 27 | self.buffer = io.StringIO() 28 | self.lock = threading.Lock() 29 | 30 | def read_input_line(self): 31 | try: 32 | with self.lock: 33 | return self.input_stream.readline() 34 | except ValueError as ve: 35 | raise StopIteration() from ve 36 | 37 | def run(self): 38 | for line in iter(self.read_input_line, "".encode()): 39 | try: 40 | data = line.decode("utf-8", "strict") 41 | except UnicodeDecodeError: 42 | data = "Failed to decode binary data to utf-8" 43 | finally: 44 | self.buffer.write(data) 45 | self.output_stream.write(data) 46 | 47 | def stop(self): 48 | with self.lock: 49 | self.input_stream.close() 50 | self.join() 51 | return self.buffer.getvalue() 52 | 53 | 54 | def get_git_hexsha(): 55 | try: 56 | import git 57 | from git.exc import InvalidGitRepositoryError 58 | except ImportError: 59 | return None 60 | try: 61 | repo = git.Repo(search_parent_directories=True) 62 | return repo.head.object.hexsha 63 | except InvalidGitRepositoryError: 64 | return None 65 | 66 | 67 | def get_subprocess_environment(config, run_context, env=None): 68 | config.set_context_entry(GlobalRunContext(run_context)) 69 | ret = os.environ.copy() 70 | ret.update(config.get_environment_context()) 71 | ret.update(env or {}) 72 | return ret 73 | 74 | 75 | def run_subprocess(config, run_context, commands, env=None): 76 | return run_subprocess_command(config, run_context, cmd=commands, env=env) 77 | 78 | 79 | def run_subprocess_command(config, run_context, cmd, env=None): 80 | env = get_subprocess_environment(config, run_context, env) 81 | proc_stdout, proc_stderr = subprocess.PIPE, subprocess.PIPE 82 | try: 83 | proc = subprocess.Popen( 84 | cmd, 85 | env=env, 86 | stdout=proc_stdout, 87 | stderr=proc_stderr, 88 | ) 89 | except OSError as ose: 90 | msg = str(ose) 91 | is_fnfe = isinstance(ose, FileNotFoundError) 92 | is_eacces = ose.errno == errno.EACCES 93 | if is_fnfe or is_eacces and os.path.exists(ose.filename): 94 | is_full_path = ose.filename.startswith("/") or ose.filename.startswith("./") 95 | is_executable = os.access(ose.filename, os.X_OK) 96 | is_py = ose.filename.endswith(".py") 97 | is_sh = ose.filename.endswith(".sh") or ose.filename.endswith(".bash") 98 | if is_fnfe and not is_full_path and is_executable: 99 | msg += "\nPlease prefix your script with `./`, ex:" 100 | msg += "\n$ sigopt SUBCOMMAND -- ./{ose.filename} {shlex.join(cmd[1:])}" 101 | elif is_py and not is_executable: 102 | msg += "\nPlease include the python executable when running python files, ex:" 103 | msg += f"\n$ sigopt SUBCOMMAND -- python {shlex.join(cmd)}" 104 | elif is_sh and not is_executable: 105 | msg += f"\nPlease make your shell script executable, ex:\n$ chmod +x {ose.filename}" 106 | raise click.ClickException(msg) from ose 107 | stdout, stderr = StreamThread(proc.stdout, sys.stdout), StreamThread(proc.stderr, sys.stderr) 108 | stdout.start() 109 | stderr.start() 110 | return_code = 0 111 | try: 112 | return_code = proc.wait() 113 | except KeyboardInterrupt: 114 | try: 115 | os.kill(proc.pid, signal.SIGINT) 116 | except ProcessLookupError: 117 | pass 118 | proc.wait() 119 | raise 120 | finally: 121 | stdout_content, stderr_content = stdout.stop(), stderr.stop() 122 | if config.log_collection_enabled: 123 | run_context.set_logs( 124 | { 125 | "stdout": stdout_content, 126 | "stderr": stderr_content, 127 | } 128 | ) 129 | return return_code 130 | 131 | 132 | def run_user_program(config, run_context, commands, source_code_content): 133 | source_code = {} 134 | git_hash = get_git_hexsha() 135 | if git_hash: 136 | source_code["hash"] = git_hash 137 | if source_code_content is not None: 138 | source_code["content"] = source_code_content 139 | run_context.log_source_code(**source_code) 140 | exit_code = run_subprocess(config, run_context, commands) 141 | if exit_code != 0: 142 | print_logger.error("command exited with non-zero status: %s", exit_code) 143 | return exit_code 144 | 145 | 146 | def setup_cli(config): 147 | config.set_user_agent_info(["CLI"]) 148 | enable_print_logging() 149 | 150 | 151 | def create_aiexperiment_from_validated_data(experiment_file, project): 152 | assert isinstance(experiment_file, ValidatedData) 153 | factory = SigOptFactory(project) 154 | return factory.create_prevalidated_aiexperiment(experiment_file.data) 155 | 156 | 157 | def cli_experiment_loop(config, experiment, command, run_options, source_code_content): 158 | for run_context in experiment.loop(name=run_options.get("name")): 159 | with run_context: 160 | run_user_program(config, run_context, command, source_code_content) 161 | -------------------------------------------------------------------------------- /sigopt/compat.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | # pylint: disable=unused-import 5 | 6 | try: 7 | import json 8 | except ImportError: 9 | try: 10 | import simplejson as json 11 | except ImportError as ie: 12 | raise ImportError( 13 | "No json library installed. Try running `pip install simplejson` to install a compatible json library." 14 | ) from ie 15 | -------------------------------------------------------------------------------- /sigopt/config.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from __future__ import print_function 5 | 6 | import base64 7 | import errno 8 | import json 9 | import os 10 | 11 | from .paths import get_root_subdir 12 | 13 | 14 | class UserAgentInfoContext(object): 15 | CONFIG_CONTEXT_KEY = "user_agent_info" 16 | 17 | @classmethod 18 | def from_config(cls, _config): 19 | return cls(_config.get_context_data(cls)) 20 | 21 | def __init__(self, info): 22 | self.info = info 23 | 24 | def to_json(self): 25 | return self.info 26 | 27 | 28 | class Config(object): 29 | API_TOKEN_KEY = "api_token" 30 | CELL_TRACKING_ENABLED_KEY = "code_tracking_enabled" 31 | LOG_COLLECTION_ENABLED_KEY = "log_collection_enabled" 32 | CONTEXT_ENVIRONMENT_KEY = "SIGOPT_CONTEXT" 33 | 34 | def __init__(self): 35 | self._config_json_path = os.path.abspath( 36 | os.path.join( 37 | get_root_subdir("client"), 38 | "config.json", 39 | ) 40 | ) 41 | self._configuration = self._read_config_json() 42 | self._json_context = {} 43 | try: 44 | encoded_context = os.environ[self.CONTEXT_ENVIRONMENT_KEY] 45 | except KeyError: 46 | pass 47 | else: 48 | decoded = base64.b64decode(encoded_context).decode("utf-8") 49 | self._json_context = json.loads(decoded) 50 | self._object_context = {} 51 | 52 | def get_context_data(self, entry_cls): 53 | key = entry_cls.CONFIG_CONTEXT_KEY 54 | instance = self._object_context.get(key) 55 | if instance: 56 | return instance.to_json() 57 | return self._json_context.get(key) 58 | 59 | def set_context_entry(self, entry): 60 | self._object_context[entry.CONFIG_CONTEXT_KEY] = entry 61 | 62 | def get_environment_context(self): 63 | context = dict(self._json_context) 64 | for key, value in self._object_context.items(): 65 | context[key] = value.to_json() 66 | return {self.CONTEXT_ENVIRONMENT_KEY: base64.b64encode(json.dumps(context).encode())} 67 | 68 | @property 69 | def api_token(self): 70 | return self._configuration.get(self.API_TOKEN_KEY) 71 | 72 | @property 73 | def cell_tracking_enabled(self): 74 | return self._configuration.get(self.CELL_TRACKING_ENABLED_KEY, False) 75 | 76 | @property 77 | def log_collection_enabled(self): 78 | return self._configuration.get(self.LOG_COLLECTION_ENABLED_KEY, False) 79 | 80 | def _ensure_config_json_path(self): 81 | config_path = self._config_json_path 82 | try: 83 | os.makedirs(os.path.dirname(config_path)) 84 | except OSError as e: 85 | if e.errno != errno.EEXIST: 86 | raise 87 | return config_path 88 | 89 | def _read_config_json(self): 90 | try: 91 | with open(self._config_json_path) as config_json_fp: 92 | return json.load(config_json_fp) 93 | except (IOError, OSError) as e: 94 | if e.errno == errno.ENOENT: 95 | return {} 96 | raise 97 | 98 | def _write_config_json(self, configuration): 99 | config_path = self._ensure_config_json_path() 100 | with open(config_path, "w") as config_json_fp: 101 | json.dump(configuration, config_json_fp, indent=2, sort_keys=True) 102 | print("", file=config_json_fp) 103 | 104 | def persist_configuration_options(self, options): 105 | self._configuration.update(options) 106 | self._write_config_json(self._configuration) 107 | 108 | def set_user_agent_info(self, info): 109 | self.set_context_entry(UserAgentInfoContext(info)) 110 | 111 | def get_user_agent_info(self): 112 | return UserAgentInfoContext.from_config(self).info 113 | 114 | 115 | config = Config() 116 | -------------------------------------------------------------------------------- /sigopt/decorators.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2024 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | 6 | def public(f): 7 | """ 8 | Indicates that the function or method is meant to be part of the public interface. 9 | Ie. intended to be used outside sigopt-python. 10 | """ 11 | return f 12 | -------------------------------------------------------------------------------- /sigopt/defaults.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import datetime 5 | import http 6 | import os 7 | import re 8 | 9 | from .exception import ApiException, ProjectNotFoundException 10 | 11 | 12 | INVALID_PROJECT_ID_STRING_CHARACTERS = re.compile(r"[^a-z0-9\-_\.]") 13 | VALID_PROJECT_ID = re.compile(r"[a-z0-9\-_\.]+\Z") 14 | 15 | 16 | def normalize_project_id(project_id): 17 | project_id = project_id.lower() 18 | return re.sub(INVALID_PROJECT_ID_STRING_CHARACTERS, "", project_id) 19 | 20 | 21 | def check_valid_project_id(project_id): 22 | if not VALID_PROJECT_ID.match(project_id): 23 | raise ValueError( 24 | f"Project ID is invalid: '{project_id}'\nProject IDs can only consist of" 25 | " lowercase letters, digits, hyphens (-), underscores (_) and periods (.)." 26 | ) 27 | 28 | 29 | def get_default_project(): 30 | project_id = os.environ.get("SIGOPT_PROJECT") 31 | if project_id: 32 | check_valid_project_id(project_id) 33 | return project_id 34 | cwd_project_id = os.path.basename(os.getcwd()) 35 | project_id = normalize_project_id(cwd_project_id) 36 | try: 37 | check_valid_project_id(project_id) 38 | except ValueError as ve: 39 | raise ValueError( 40 | f"The current directory '{cwd_project_id}' could not be converted into a" 41 | " valid project id. Please rename the directory or use the SIGOPT_PROJECT" 42 | " environment variable instead." 43 | ) from ve 44 | return project_id 45 | 46 | 47 | def get_default_name(project): 48 | datetime_string = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") 49 | return f"{project} {datetime_string}" 50 | 51 | 52 | def get_client_id(connection): 53 | return connection.tokens("self").fetch().client 54 | 55 | 56 | def ensure_project_exists(connection, project_id): 57 | client_id = get_client_id(connection) 58 | try: 59 | connection.clients(client_id).projects(project_id).fetch() 60 | except ApiException as e: 61 | if e.status_code == http.HTTPStatus.NOT_FOUND: 62 | raise ProjectNotFoundException(project_id) from e 63 | raise 64 | return client_id 65 | -------------------------------------------------------------------------------- /sigopt/endpoint.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .compat import json as simplejson 5 | 6 | 7 | class BoundApiEndpoint(object): 8 | def __init__(self, bound_resource, endpoint): 9 | self._bound_resource = bound_resource 10 | self._endpoint = endpoint 11 | 12 | def call_with_json(self, json): 13 | return self.call_with_params(simplejson.loads(json)) 14 | 15 | def call_with_params(self, params): 16 | name = self._endpoint._name 17 | path = list(self._bound_resource._base_path) 18 | if name: 19 | path.append(name) 20 | conn = self._bound_resource._resource._conn 21 | raw_response = None 22 | 23 | raw_response = conn.request(self._endpoint._method, path, params, None) 24 | 25 | if raw_response is not None and self._endpoint._response_cls is not None: 26 | return self._endpoint._response_cls(raw_response, self, params) 27 | return None 28 | 29 | def __call__(self, **kwargs): 30 | return self.call_with_params(kwargs) 31 | 32 | 33 | class ApiEndpoint(object): 34 | def __init__(self, name, response_cls, method, attribute_name=None): 35 | self._name = name 36 | self._response_cls = response_cls 37 | self._method = method 38 | self._attribute_name = attribute_name or name 39 | -------------------------------------------------------------------------------- /sigopt/examples/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .franke import * 5 | -------------------------------------------------------------------------------- /sigopt/examples/franke.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import math 5 | 6 | 7 | # Franke function - http://www.sfu.ca/~ssurjano/franke2d.html 8 | def franke_function(x1, x2): 9 | return ( 10 | 0.75 * math.exp(-((9 * x1 - 2) ** 2) / 4.0 - (9 * x2 - 2) ** 2 / 4.0) 11 | + 0.75 * math.exp(-((9 * x1 + 1) ** 2) / 49.0 - (9 * x2 + 1) / 10.0) 12 | + 0.5 * math.exp(-((9 * x1 - 7) ** 2) / 4.0 - (9 * x2 - 3) ** 2 / 4.0) 13 | - 0.2 * math.exp(-((9 * x1 - 4) ** 2) - (9 * x2 - 7) ** 2) 14 | ) 15 | 16 | 17 | # Create a SigOpt experiment that optimized the Franke function with 18 | # connection.experiments().create(**FRANKE_EXPERIMENT_DEFINITION) 19 | FRANKE_EXPERIMENT_DEFINITION = { 20 | "name": "Franke Optimization", 21 | "parameters": [ 22 | {"name": "x", "bounds": {"max": 1, "min": 0}, "type": "double", "precision": 4}, 23 | {"name": "y", "bounds": {"max": 1, "min": 0}, "type": "double", "precision": 4}, 24 | ], 25 | } 26 | -------------------------------------------------------------------------------- /sigopt/exception.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import copy 5 | 6 | 7 | class SigOptException(Exception): 8 | pass 9 | 10 | 11 | class ConnectionException(SigOptException): 12 | """ 13 | An exception that occurs when the SigOpt API was unavailable. 14 | """ 15 | 16 | def __init__(self, message): 17 | super().__init__(message) 18 | self.message = message 19 | 20 | def __str__(self): 21 | return "{0}: {1}".format( 22 | "ConnectionException", 23 | self.message if self.message is not None else "", 24 | ) 25 | 26 | 27 | class ApiException(SigOptException): 28 | """ 29 | An exception that occurs when the SigOpt API was contacted successfully, but 30 | it responded with an error. 31 | """ 32 | 33 | def __init__(self, body, status_code): 34 | self.message = body.get("message", None) if body is not None else None 35 | self._body = body 36 | if self.message is not None: 37 | super().__init__(self.message) 38 | else: 39 | super().__init__() 40 | self.status_code = status_code 41 | 42 | def __str__(self): 43 | return "{0} ({1}): {2}".format( 44 | "ApiException", 45 | self.status_code, 46 | self.message if self.message is not None else "", 47 | ) 48 | 49 | def to_json(self): 50 | return copy.deepcopy(self._body) 51 | 52 | 53 | class ConflictingProjectException(SigOptException): 54 | def __init__(self, project_id): 55 | super().__init__(f"The project with id '{project_id}' already exists.") 56 | 57 | 58 | class ProjectNotFoundException(SigOptException): 59 | def __init__(self, project_id): 60 | super().__init__( 61 | f"The project '{project_id}' does not exist.\nTry any of the following" 62 | f" steps to resolve this:\n * create a project with the ID '{project_id}'" 63 | f" with the command\n `sigopt create project --project '{project_id}'`" 64 | " or by visiting\n https://app.sigopt.com/projects\n * change the" 65 | " project ID by setting the SIGOPT_PROJECT environment variable or\n by" 66 | " renaming the current directory\n * (advanced) if the project you want" 67 | " to use is in a different team,\n change your API token by switching" 68 | " to that team and then going to\n https://app.sigopt.com/tokens/info" 69 | ) 70 | -------------------------------------------------------------------------------- /sigopt/file_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import base64 5 | import hashlib 6 | import io 7 | import mimetypes 8 | import warnings 9 | 10 | import png 11 | 12 | 13 | def try_load_pil_image(image): 14 | try: 15 | from PIL.Image import Image as PILImage 16 | except ImportError: 17 | return None 18 | if isinstance(image, PILImage): 19 | image_data = io.BytesIO() 20 | image.save(image_data, "PNG") 21 | image_data.seek(0) 22 | return getattr(image, "filename", None), image_data, "image/png" 23 | return None 24 | 25 | 26 | def try_load_matplotlib_image(image): 27 | try: 28 | from matplotlib.figure import Figure as MatplotlibFigure 29 | except ImportError: 30 | return None 31 | if isinstance(image, MatplotlibFigure): 32 | image_data = io.BytesIO() 33 | image.savefig(image_data, format="svg") 34 | image_data.seek(0) 35 | return None, image_data, "image/svg+xml" 36 | return None 37 | 38 | 39 | def try_load_numpy_image(image): 40 | try: 41 | from numpy import ndarray 42 | from numpy import uint8 as numpy_uint8 43 | except ImportError: 44 | return None 45 | if isinstance(image, ndarray): 46 | channels = 0 47 | if len(image.shape) == 2: 48 | channels = 1 49 | elif len(image.shape) == 3: 50 | channels = image.shape[2] 51 | if not channels: 52 | raise Exception(f"images provided as numpy arrays must have 2 or 3 dimensions, provided shape: {image.shape}") 53 | channels_to_mode = { 54 | 1: "L", 55 | 3: "RGB", 56 | 4: "RGBA", 57 | } 58 | if channels not in channels_to_mode: 59 | raise Exception(f"images provided as numpy arrays must have 1, 3 or 4 channels, provided channels: {channels}") 60 | mode = channels_to_mode[channels] 61 | clipped_image = image.clip(0, 255) 62 | byte_image = clipped_image.astype(numpy_uint8) 63 | height, width = image.shape[:2] 64 | pypng_compatible = byte_image.reshape(height, width * channels) 65 | writer = png.Writer(width, height, greyscale=(mode == "L"), alpha=(mode == "RGBA")) 66 | image_data = io.BytesIO() 67 | writer.write(image_data, pypng_compatible) 68 | return None, image_data, "image/png" 69 | return None 70 | 71 | 72 | MIME_TYPE_REMAP = { 73 | # the mime type image/x-ms-bmp is returned in some environments 74 | # it is not officially supported by Chrome 75 | # it is still used in some cases for legacy IE7 support 76 | # prefer Chrome support over IE7 support 77 | "image/x-ms-bmp": "image/bmp", 78 | } 79 | 80 | SUPPORTED_IMAGE_MIME_TYPES = { 81 | "image/apng", 82 | "image/bmp", 83 | "image/gif", 84 | "image/jpeg", 85 | "image/png", 86 | "image/svg+xml", 87 | "image/webp", 88 | "image/x-icon", 89 | } 90 | 91 | 92 | def create_api_image_payload(image): 93 | if isinstance(image, str): 94 | content_type = mimetypes.guess_type(image) 95 | if content_type is None: 96 | warnings.warn( 97 | f"Could not guess image type from provided filename, skipping upload: {image}", 98 | RuntimeWarning, 99 | ) 100 | return None 101 | content_type, _ = content_type 102 | content_type = MIME_TYPE_REMAP.get(content_type, content_type) 103 | if content_type not in SUPPORTED_IMAGE_MIME_TYPES: 104 | friendly_supported_types = ", ".join(sorted(SUPPORTED_IMAGE_MIME_TYPES)) 105 | warnings.warn( 106 | ( 107 | f"File type `{content_type}` is not supported, please use one of" 108 | f" the supported types: {friendly_supported_types}" 109 | ), 110 | RuntimeWarning, 111 | ) 112 | return None 113 | return image, open(image, "rb"), content_type 114 | payload = try_load_pil_image(image) 115 | if payload is not None: 116 | return payload 117 | payload = try_load_matplotlib_image(image) 118 | if payload is not None: 119 | return payload 120 | payload = try_load_numpy_image(image) 121 | if payload is not None: 122 | return payload 123 | warnings.warn( 124 | ( 125 | f"Image type not supported: {type(image)}. Supported types: str," 126 | " PIL.Image.Image, matplotlib.figure.Figure, numpy.ndarray" 127 | ), 128 | RuntimeWarning, 129 | ) 130 | return None 131 | 132 | 133 | def get_blob_properties(image_data): 134 | md5 = hashlib.md5() # nosec 135 | image_data.seek(0) 136 | while True: 137 | chunk = image_data.read(2**20) 138 | if not chunk: 139 | break 140 | md5.update(chunk) 141 | length = image_data.tell() 142 | b64_md5 = base64.b64encode(md5.digest()).decode() 143 | return length, b64_md5 144 | -------------------------------------------------------------------------------- /sigopt/lib.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import math as _math 5 | import numbers as _numbers 6 | from collections.abc import Mapping as _Mapping 7 | from collections.abc import Sequence as _Sequence 8 | 9 | 10 | def is_numpy_array(val): 11 | return val.__class__.__name__ == "ndarray" 12 | 13 | 14 | def is_sequence(val): 15 | """ 16 | Returns True iff this is a "list-like" type. 17 | Avoids the common error that strings are iterable 18 | """ 19 | if is_numpy_array(val): 20 | return True 21 | return isinstance(val, _Sequence) and not isinstance(val, str) and not isinstance(val, bytes) 22 | 23 | 24 | def is_mapping(val): 25 | """ 26 | Returns True iff this is a "dict-like" type 27 | """ 28 | return isinstance(val, _Mapping) 29 | 30 | 31 | def is_integer(num): 32 | """ 33 | Returns True iff this is an integer type. Avoids the common error that bools 34 | are instances of int, and handles numpy correctly 35 | """ 36 | if isinstance(num, bool): 37 | return False 38 | if isinstance(num, _numbers.Integral): 39 | return True 40 | return False 41 | 42 | 43 | def is_number(x): 44 | if isinstance(x, bool): 45 | return False 46 | if isinstance(x, float) and _math.isnan(x): 47 | return False 48 | return isinstance(x, _numbers.Number) or is_integer(x) 49 | 50 | 51 | def is_string(s): 52 | return isinstance(s, str) 53 | 54 | 55 | def remove_nones(mapping): 56 | return {key: value for key, value in mapping.items() if value is not None} 57 | 58 | 59 | def validate_name(warn, name): 60 | if not is_string(name): 61 | raise ValueError(f"The {warn} must be a string, not {type(name).__name__}") 62 | if not name: 63 | raise ValueError(f"The {warn} cannot be an empty string") 64 | 65 | 66 | def sanitize_number(warn, name, value): 67 | if is_integer(value): 68 | return value 69 | try: 70 | value = float(value) 71 | if _math.isinf(value) or _math.isnan(value): 72 | raise ValueError 73 | return value 74 | except (ValueError, TypeError) as e: 75 | raise ValueError(f"The {warn} logged for `{name}` could not be converted to a number: {value!r}") from e 76 | -------------------------------------------------------------------------------- /sigopt/local_run_context.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import copy 5 | 6 | from .run_context import BaseRunContext 7 | 8 | 9 | class LocalRunContext(BaseRunContext): 10 | def __init__(self, **kwargs): 11 | self.run = copy.deepcopy(kwargs) if kwargs else {} 12 | 13 | def get(self, name=None, default_type=dict): 14 | if name is None: 15 | return self.run 16 | if name not in self.run: 17 | self.run.setdefault(name, default_type()) 18 | return self.run[name] 19 | 20 | def log_state(self, state): 21 | self.run["state"] = state 22 | 23 | def _set_parameters(self, parameters): 24 | self.get("assignments").update(parameters) 25 | 26 | def _log_failure(self): 27 | self.run["state"] = "failed" 28 | 29 | def _log_metadata(self, metadata): 30 | self.get("metadata").update(metadata) 31 | 32 | def _log_metrics(self, metrics): 33 | self.get("values").update(metrics) 34 | 35 | def _set_parameters_meta(self, parameters_meta): 36 | self.get("assignments_meta").update(parameters_meta) 37 | 38 | def _set_parameters_sources(self, assignments_sources): 39 | self.get("assignments_sources").update(assignments_sources) 40 | 41 | def log_parameters(self, params, source=None, source_meta=None): 42 | self.set_parameters(params) 43 | if source is not None: 44 | self.set_parameters_source(params, source) 45 | if source_meta is not None: 46 | self.set_parameters_sources_meta(source, **source_meta) 47 | -------------------------------------------------------------------------------- /sigopt/log_capture.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import io 5 | import sys 6 | import threading 7 | 8 | from .decorators import public 9 | 10 | 11 | class MonitorStream(io.IOBase): 12 | def __init__(self, original_stream): 13 | super().__init__() 14 | self.buffer_lock = threading.Lock() 15 | self.original_stream = original_stream 16 | self.buffer_stream = None 17 | self._replace_buffer_stream() 18 | 19 | def _replace_buffer_stream(self): 20 | self.buffer_stream = io.StringIO() 21 | 22 | @public 23 | def close(self): 24 | raise IOError("MonitorStream cannot be closed") 25 | 26 | @public 27 | @property 28 | def closed(self): 29 | return self.original_stream.closed 30 | 31 | @public 32 | def fileno(self): 33 | raise IOError("MonitorStream has no fileno") 34 | 35 | @public 36 | def flush(self): 37 | return self.original_stream.flush() 38 | 39 | @public 40 | def isatty(self): 41 | return False 42 | 43 | @public 44 | def readable(self): 45 | return False 46 | 47 | @public 48 | def readline(self, *args, **kwargs): 49 | return self.original_stream.readline(*args, **kwargs) 50 | 51 | @public 52 | def readlines(self, *args, **kwargs): 53 | return self.original_stream.readlines(*args, **kwargs) 54 | 55 | @public 56 | def seek(self, *args, **kwargs): 57 | raise IOError("MonitorStream is not seekable") 58 | 59 | @public 60 | def seekable(self): 61 | return False 62 | 63 | @public 64 | def tell(self, *args, **kwargs): 65 | raise IOError("MonitorStream is not seekable") 66 | 67 | @public 68 | def writable(self): 69 | return True 70 | 71 | @public 72 | def write(self, content): 73 | rval = self.original_stream.write(content) 74 | with self.buffer_lock: 75 | self.buffer_stream.write(content) 76 | return rval 77 | 78 | @public 79 | def writelines(self, lines): 80 | for line in lines: 81 | self.write(line) 82 | 83 | def get_buffer_contents(self): 84 | with self.buffer_lock: 85 | content = self.buffer_stream.getvalue() 86 | self._replace_buffer_stream() 87 | return content 88 | 89 | 90 | class BaseStreamMonitor(object): 91 | def get_stream_data(self): 92 | raise NotImplementedError() 93 | 94 | def __enter__(self): 95 | raise NotImplementedError() 96 | 97 | def __exit__(self, typ, value, trace): 98 | del trace 99 | raise NotImplementedError() 100 | 101 | 102 | class NullStreamMonitor(BaseStreamMonitor): 103 | def get_stream_data(self): 104 | return None 105 | 106 | def __enter__(self): 107 | return self 108 | 109 | def __exit__(self, typ, value, trace): 110 | del trace 111 | 112 | 113 | class SystemOutputStreamMonitor(BaseStreamMonitor): 114 | def __init__(self): 115 | super().__init__() 116 | self.monitor_streams = None 117 | 118 | def get_stream_data(self): 119 | if self.monitor_streams is None: 120 | return None 121 | stdout_content, stderr_content = (monitor_stream.get_buffer_contents() for monitor_stream in self.monitor_streams) 122 | return stdout_content, stderr_content 123 | 124 | def __enter__(self): 125 | if self.monitor_streams is not None: 126 | raise Exception("Already monitoring") 127 | self.monitor_streams = MonitorStream(sys.stdout), MonitorStream(sys.stderr) 128 | sys.stdout, sys.stderr = self.monitor_streams 129 | return self 130 | 131 | def __exit__(self, typ, value, trace): 132 | del trace 133 | sys.stdout, sys.stderr = (monitor_stream.original_stream for monitor_stream in self.monitor_streams) 134 | -------------------------------------------------------------------------------- /sigopt/magics.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import http 5 | import io 6 | import sys 7 | 8 | import click 9 | import IPython 10 | import yaml 11 | from IPython.core.magic import Magics, cell_magic, line_magic, magics_class 12 | 13 | from .cli.commands.config import API_TOKEN_PROMPT, CELL_TRACKING_PROMPT, LOG_COLLECTION_PROMPT 14 | from .config import config 15 | from .defaults import get_default_project 16 | from .exception import ApiException 17 | from .factory import SigOptFactory 18 | from .log_capture import NullStreamMonitor, SystemOutputStreamMonitor 19 | from .run_context import global_run_context 20 | from .sigopt_logging import print_logger 21 | from .validate import ValidationError, validate_aiexperiment_input 22 | 23 | 24 | def get_ns(): 25 | # NOTE: inspired by 26 | # https://github.com/ipython/ipython/blob/5577a476295146641fbd6f8c992d374b905dc1bc/IPython/core/interactiveshell.py 27 | # Walk up the stack trace until we find the 'exit' command 28 | stack_depth = 1 29 | while True: 30 | frame = sys._getframe(stack_depth) 31 | f_locals = frame.f_locals 32 | try: 33 | if isinstance(f_locals["exit"], IPython.core.autocall.ExitAutocall): 34 | return f_locals 35 | except KeyError: 36 | pass 37 | stack_depth += 1 38 | 39 | 40 | @magics_class 41 | class SigOptMagics(Magics): 42 | def __init__(self, shell): 43 | super().__init__(shell) 44 | self._experiment = None 45 | self._factory = SigOptFactory(get_default_project()) 46 | 47 | def setup(self): 48 | config.set_user_agent_info( 49 | [ 50 | "Notebook", 51 | "/".join(["IPython", IPython.__version__]), 52 | ] 53 | ) 54 | 55 | @cell_magic 56 | def experiment(self, _, cell): 57 | ns = get_ns() 58 | 59 | # pylint: disable=eval-used 60 | cell_value = eval(cell, ns) 61 | # pylint: enable=eval-used 62 | if isinstance(cell_value, dict): 63 | experiment_body = dict(cell_value) 64 | else: 65 | experiment_body = yaml.safe_load(io.StringIO(cell_value)) 66 | self.setup() 67 | try: 68 | validated = validate_aiexperiment_input(experiment_body) 69 | except ValidationError as validation_error: 70 | print_logger.error("ValidationError: %s", str(validation_error)) 71 | return 72 | try: 73 | self._experiment = self._factory.create_prevalidated_aiexperiment(validated) 74 | except ApiException as api_exception: 75 | if api_exception.status_code == http.HTTPStatus.BAD_REQUEST: 76 | print_logger.error("ApiException: %s", str(api_exception)) 77 | 78 | def exec_cell(self, run_context, cell, ns): 79 | global_run_context.set_run_context(run_context) 80 | try: 81 | if config.cell_tracking_enabled: 82 | run_context.log_source_code(content=cell) 83 | stream_monitor = SystemOutputStreamMonitor() if config.log_collection_enabled else NullStreamMonitor() 84 | with stream_monitor: 85 | # pylint: disable=exec-used 86 | exec(cell, ns) 87 | # pylint: enable=exec-used 88 | stream_data = stream_monitor.get_stream_data() 89 | if stream_data: 90 | stdout, stderr = stream_data 91 | run_context.set_logs({"stdout": stdout, "stderr": stderr}) 92 | finally: 93 | global_run_context.clear_run_context() 94 | 95 | @cell_magic 96 | def run(self, line, cell): 97 | ns = get_ns() 98 | 99 | name = None 100 | if line: 101 | name = line 102 | 103 | self.setup() 104 | run_context = self._factory.create_run(name=name) 105 | with run_context: 106 | self.exec_cell(run_context, cell, ns) 107 | 108 | @cell_magic 109 | def optimize(self, line, cell): 110 | ns = get_ns() 111 | 112 | if self._experiment is None: 113 | raise Exception("Please create an experiment first with the %%experiment magic command") 114 | 115 | name = None 116 | if line: 117 | name = line 118 | 119 | self.setup() 120 | 121 | for run_context in self._experiment.loop(name=name): 122 | with run_context: 123 | self.exec_cell(run_context, cell, ns) 124 | 125 | @line_magic 126 | def sigopt(self, line): 127 | command = line.strip() 128 | if command == "config": 129 | api_token = click.prompt(API_TOKEN_PROMPT, hide_input=True) 130 | enable_log_collection = click.confirm(LOG_COLLECTION_PROMPT, default=False) 131 | enable_code_tracking = click.confirm(CELL_TRACKING_PROMPT, default=False) 132 | config.persist_configuration_options( 133 | { 134 | config.API_TOKEN_KEY: api_token, 135 | config.CELL_TRACKING_ENABLED_KEY: enable_code_tracking, 136 | config.LOG_COLLECTION_ENABLED_KEY: enable_log_collection, 137 | } 138 | ) 139 | self._factory.connection.set_client_token(api_token) 140 | else: 141 | raise ValueError(f"Unknown sigopt command: {command}") 142 | -------------------------------------------------------------------------------- /sigopt/model_aware_run.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .run_context import RunContext 5 | 6 | 7 | class ModelAwareRun: 8 | def __init__(self, run, model): 9 | assert isinstance(run, RunContext) 10 | self.run = run 11 | self.model = model 12 | -------------------------------------------------------------------------------- /sigopt/paths.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | 6 | 7 | def get_root_dir(): 8 | root_dir = os.environ.get("SIGOPT_HOME", os.path.join("~", ".sigopt")) 9 | return os.path.expanduser(root_dir) 10 | 11 | 12 | def get_root_subdir(dirname): 13 | return os.path.join(get_root_dir(), dirname) 14 | -------------------------------------------------------------------------------- /sigopt/ratelimit.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import secrets 5 | import threading 6 | import time 7 | 8 | 9 | class _FailedStatusRateLimit(object): 10 | def __init__(self, limit): 11 | self.limit = limit 12 | self.thread_lock = threading.Lock() 13 | self.count = 0 14 | 15 | def increment_and_check(self): 16 | with self.thread_lock: 17 | self.count += 1 18 | multiples_over = self.count // self.limit 19 | if multiples_over: 20 | quadratic_backoff = multiples_over**2 21 | jitter = secrets.SystemRandom().random() * 2 22 | time.sleep(quadratic_backoff + jitter) 23 | 24 | def clear(self): 25 | with self.thread_lock: 26 | self.count = 0 27 | 28 | 29 | failed_status_rate_limit = _FailedStatusRateLimit(5) 30 | -------------------------------------------------------------------------------- /sigopt/request_driver.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | from http import HTTPStatus 6 | 7 | import backoff 8 | import requests 9 | from requests.adapters import HTTPAdapter 10 | 11 | from .compat import json as simplejson 12 | from .config import config 13 | from .exception import ApiException, ConnectionException 14 | from .objects import ApiObject 15 | from .ratelimit import failed_status_rate_limit 16 | from .urllib3_patch import ExpiringHTTPConnectionPool, ExpiringHTTPSConnectionPool 17 | from .version import VERSION 18 | 19 | 20 | DEFAULT_API_URL = "https://api.sigopt.com" 21 | DEFAULT_HTTP_TIMEOUT = 150 22 | 23 | 24 | def get_expiring_session(): 25 | adapter = HTTPAdapter() 26 | adapter.poolmanager.pool_classes_by_scheme = { 27 | "http": ExpiringHTTPConnectionPool, 28 | "https": ExpiringHTTPSConnectionPool, 29 | } 30 | session = requests.Session() 31 | session.mount("http://", adapter) 32 | session.mount("https://", adapter) 33 | return session 34 | 35 | 36 | class RequestDriver(object): 37 | api_version = "v1" 38 | 39 | def __init__( 40 | self, 41 | client_token=None, 42 | headers=None, 43 | proxies=None, 44 | timeout=DEFAULT_HTTP_TIMEOUT, 45 | client_ssl_certs=None, 46 | session=None, 47 | api_url=None, 48 | ): 49 | if client_token is None: 50 | client_token = os.environ.get("SIGOPT_API_TOKEN", config.api_token) 51 | if not client_token: 52 | raise ValueError("Must provide client_token or set environment variable SIGOPT_API_TOKEN") 53 | self.auth = None 54 | self.set_client_token(client_token) 55 | # no-verify overrides a passed in path 56 | no_verify_ssl_certs = os.environ.get("SIGOPT_API_NO_VERIFY_SSL_CERTS") 57 | if no_verify_ssl_certs: 58 | self.verify_ssl_certs = False 59 | else: 60 | self.verify_ssl_certs = os.environ.get("SIGOPT_API_VERIFY_SSL_CERTS") 61 | self.proxies = proxies 62 | self.timeout = timeout 63 | self.client_ssl_certs = client_ssl_certs 64 | self.session = session or get_expiring_session() 65 | self.api_url = api_url or os.environ.get("SIGOPT_API_URL") or DEFAULT_API_URL 66 | self.default_headers = { 67 | "Content-Type": "application/json", 68 | "X-SigOpt-Python-Version": VERSION, 69 | } 70 | if headers: 71 | self.default_headers.update(headers) 72 | 73 | def _set_auth(self, username, password): 74 | if username is not None: 75 | self.auth = requests.auth.HTTPBasicAuth(username, password) 76 | else: 77 | self.auth = None 78 | 79 | def _request_params(self, params): 80 | req_params = params or {} 81 | 82 | def serialize(value): 83 | if isinstance(value, (dict, list)): 84 | return simplejson.dumps(value, indent=None, separators=(",", ":")) 85 | return str(value) 86 | 87 | return dict(((key, serialize(ApiObject.as_json(value))) for key, value in req_params.items() if value is not None)) 88 | 89 | def set_client_token(self, client_token): 90 | self._set_auth(client_token, "") 91 | 92 | def set_api_url(self, api_url): 93 | self.api_url = api_url 94 | 95 | def _request(self, method, url, params, json, headers): 96 | headers = self._with_default_headers(headers) 97 | try: 98 | caller = self.session or requests 99 | response = caller.request( 100 | method=method, 101 | url=url, 102 | params=params, 103 | json=json, 104 | auth=self.auth, 105 | headers=headers, 106 | verify=self.verify_ssl_certs, 107 | proxies=self.proxies, 108 | timeout=self.timeout, 109 | cert=self.client_ssl_certs, 110 | ) 111 | except requests.exceptions.RequestException as rqe: 112 | message = ["An error occurred connecting to SigOpt."] 113 | if not url or not url.startswith(DEFAULT_API_URL): 114 | message.append("The host may be misconfigured or unavailable.") 115 | message.append("Contact support@sigopt.com for assistance.") 116 | message.append("") 117 | message.append(str(rqe)) 118 | raise ConnectionException("\n".join(message)) from rqe 119 | return response 120 | 121 | def request(self, method, path, data, headers): 122 | method = method.upper() 123 | url = "/".join(str(v) for v in (self.api_url, self.api_version, *path)) 124 | if method in ("GET", "DELETE"): 125 | json, params = None, self._request_params(data) 126 | else: 127 | json, params = ApiObject.as_json(data), None 128 | retry = backoff.on_predicate( 129 | backoff.expo, 130 | lambda response: response.status_code == HTTPStatus.TOO_MANY_REQUESTS, 131 | max_time=self.timeout, 132 | jitter=backoff.full_jitter, 133 | ) 134 | response = retry(self._request)(method, url, params, json, headers) 135 | return self._handle_response(response) 136 | 137 | def _with_default_headers(self, headers): 138 | user_agent_str = f"sigopt-python/{VERSION}" 139 | user_agent_info = config.get_user_agent_info() 140 | if user_agent_info: 141 | user_agent_info_str = "".join( 142 | [ 143 | "(", 144 | "; ".join(user_agent_info), 145 | ")", 146 | ] 147 | ) 148 | user_agent_str = " ".join([user_agent_str, user_agent_info_str]) 149 | 150 | request_headers = {"User-Agent": user_agent_str} 151 | 152 | if headers: 153 | request_headers.update(headers) 154 | 155 | request_headers.update(self.default_headers) 156 | return request_headers 157 | 158 | def _handle_response(self, response): 159 | status_code = response.status_code 160 | is_success = 200 <= status_code <= 299 161 | 162 | if status_code == 204: 163 | response_json = None 164 | else: 165 | try: 166 | response_json = simplejson.loads(response.text) 167 | except ValueError: 168 | response_json = {"message": response.text} 169 | status_code = 500 if is_success else status_code 170 | 171 | if is_success: 172 | failed_status_rate_limit.clear() 173 | return response_json 174 | failed_status_rate_limit.increment_and_check() 175 | raise ApiException(response_json, status_code) 176 | -------------------------------------------------------------------------------- /sigopt/resource.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .endpoint import BoundApiEndpoint 5 | 6 | 7 | _NO_ARG = object() 8 | 9 | 10 | class BoundApiResource(object): 11 | def __init__(self, resource, id_, path): 12 | self._resource = resource 13 | 14 | self._base_path = list(path) 15 | if id_ is not _NO_ARG: 16 | self._base_path.append(id_) 17 | 18 | def get_bound_entity(self, name): 19 | endpoint = self._resource._endpoints.get(name) 20 | if endpoint: 21 | return BoundApiEndpoint(self, endpoint) 22 | sub_resource = self._resource._sub_resources.get(name) 23 | if sub_resource: 24 | return PartiallyBoundApiResource(sub_resource, self) 25 | return None 26 | 27 | def __getattr__(self, attr): 28 | bound_entity = self.get_bound_entity(attr) 29 | if bound_entity: 30 | return bound_entity 31 | raise AttributeError( 32 | "Cannot find attribute `{attribute}` on resource `{resource}`, likely no" 33 | " endpoint exists for: {path}/{attribute}, or `{resource}` does not support" 34 | " `{attribute}`.".format( 35 | attribute=attr, 36 | resource=self._resource._name, 37 | path="/".join(self._base_path), 38 | ) 39 | ) 40 | 41 | 42 | class PartiallyBoundApiResource(object): 43 | def __init__(self, resource, bound_parent_resource): 44 | self._resource = resource 45 | self._bound_parent_resource = bound_parent_resource 46 | 47 | def __call__(self, id_=_NO_ARG): 48 | path = self._bound_parent_resource._base_path + [self._resource._name] 49 | return BoundApiResource(self._resource, id_, path) 50 | 51 | 52 | class BaseApiResource(object): 53 | def __init__(self, conn, name, endpoints=None, resources=None): 54 | self._conn = conn 55 | self._name = name 56 | 57 | self._endpoints = dict(((endpoint._attribute_name, endpoint) for endpoint in endpoints)) if endpoints else {} 58 | 59 | self._sub_resources = dict(((resource._name, resource) for resource in resources)) if resources else {} 60 | 61 | def __call__(self, id_=_NO_ARG): 62 | return BoundApiResource(self, id_, [self._name]) 63 | 64 | 65 | class ApiResource(BaseApiResource): 66 | def __init__(self, conn, name, endpoints=None, resources=None): 67 | super().__init__( 68 | conn=conn, 69 | name=name, 70 | endpoints=endpoints, 71 | resources=resources, 72 | ) 73 | -------------------------------------------------------------------------------- /sigopt/run_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .defaults import get_default_name 5 | from .run_context import RunContext 6 | from .sigopt_logging import print_logger 7 | 8 | 9 | class BaseRunFactory(object): 10 | run_context_class = RunContext 11 | 12 | def _on_run_created(self, run): 13 | print_logger.info( 14 | "Run started, view it on the SigOpt dashboard at https://app.sigopt.com/run/%s", 15 | run.id, 16 | ) 17 | 18 | @property 19 | def project(self): 20 | raise NotImplementedError 21 | 22 | def _create_run(self, name, metadata): 23 | raise NotImplementedError 24 | 25 | def create_run(self, name=None, metadata=None): 26 | if name is None: 27 | name = get_default_name(self.project) 28 | run = self._create_run(name, metadata) 29 | self._on_run_created(run) 30 | return run 31 | -------------------------------------------------------------------------------- /sigopt/run_params.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from collections.abc import MutableMapping 5 | 6 | from .decorators import public 7 | from .lib import is_string 8 | 9 | 10 | _get = object.__getattribute__ 11 | _set = object.__setattr__ 12 | 13 | 14 | class RunParameters(MutableMapping): 15 | def __init__(self, run_context, fixed_items, default_items=None): 16 | if default_items: 17 | items = dict(default_items) 18 | items.update(fixed_items) 19 | else: 20 | items = dict(fixed_items) 21 | _set(self, "__items", items) 22 | _set(self, "__run_context", run_context) 23 | _set(self, "__fixed_keys", set(fixed_items.keys())) 24 | 25 | @public 26 | def update(self, *args, **kwds): # pylint: disable=arguments-differ 27 | # this update is atomic, which reduces the number of calls to set_parameter(s) 28 | # the default implementation of update would result in a partial update if any of the setters failed 29 | # ex. (x := {}).update([(1, 2), ({}, 4)]) => raises TypeError and x == {1: 2} 30 | tmp = dict() 31 | tmp.update(*args, **kwds) 32 | self.__check_dict_for_update(tmp, check_fixed=True) 33 | _get(self, "__items").update(tmp) 34 | _get(self, "__run_context").set_parameters(tmp) 35 | 36 | @public 37 | def setdefaults(self, *args, **kwds): 38 | tmp = dict() 39 | tmp.update(*args, **kwds) 40 | self.__check_dict_for_update(tmp, check_fixed=False) 41 | items = _get(self, "__items") 42 | unset_keys = set(tmp) - set(items) 43 | update = {key: tmp[key] for key in unset_keys} 44 | items.update(update) 45 | _get(self, "__run_context").set_parameters(update) 46 | return self 47 | 48 | def __getattr__(self, attr): 49 | try: 50 | return self[attr] 51 | except KeyError as ke: 52 | raise AttributeError(f"no parameter with name {attr!r}") from ke 53 | 54 | def __setattr__(self, attr, value): 55 | try: 56 | self[attr] = value 57 | except KeyError as ke: 58 | raise AttributeError(str(ke)) from ke 59 | 60 | def __check_key_type(self, key): 61 | if not is_string(key): 62 | raise TypeError(f"parameter names must be strings, got {type(key).__name__!r}") 63 | 64 | def __check_key_is_not_fixed(self, key): 65 | if key in _get(self, "__fixed_keys"): 66 | raise ValueError(f"value of {key!r} cannot be changed") 67 | 68 | def __check_dict_for_update(self, update_dict, check_fixed): 69 | for key in update_dict: 70 | self.__check_key_type(key) 71 | if check_fixed: 72 | for key in update_dict: 73 | self.__check_key_is_not_fixed(key) 74 | 75 | def __getitem__(self, key): 76 | return _get(self, "__items")[key] 77 | 78 | def __setitem__(self, key, value): 79 | self.__check_key_type(key) 80 | self.__check_key_is_not_fixed(key) 81 | _get(self, "__items")[key] = value 82 | _get(self, "__run_context").set_parameter(key, value) 83 | 84 | def __delitem__(self, key): 85 | self.__check_key_is_not_fixed(key) 86 | del _get(self, "__items")[key] 87 | _get(self, "__run_context").set_parameter(key, None) 88 | 89 | def __iter__(self): 90 | return iter(_get(self, "__items")) 91 | 92 | def __len__(self): 93 | return len(_get(self, "__items")) 94 | 95 | def __repr__(self): 96 | return repr(_get(self, "__items")) 97 | 98 | def __str__(self): 99 | return str(_get(self, "__items")) 100 | 101 | 102 | class GlobalRunParameters(MutableMapping): 103 | def __init__(self, global_run_context): 104 | _set(self, "__global_run_context", global_run_context) 105 | _set(self, "__global_params", RunParameters(global_run_context, dict())) 106 | 107 | @property 108 | def __params(self): 109 | run_context = _get(self, "__global_run_context").run_context 110 | if run_context is None: 111 | return _get(self, "__global_params") 112 | return run_context.params 113 | 114 | def __getattribute__(self, attr): 115 | # public methods like update and pop should pass to the underlying __params, 116 | # but attributes beginning with "_" should at least attempt to be resolved. 117 | if attr.startswith("_"): 118 | try: 119 | return _get(self, attr) 120 | except AttributeError: 121 | pass 122 | params = self.__params 123 | return getattr(params, attr) 124 | 125 | def __setattr__(self, attr, value): 126 | params = self.__params 127 | setattr(params, attr, value) 128 | 129 | def __dir__(self): 130 | params = self.__params 131 | return sorted(set(dir(super())) | set(dir(params))) 132 | 133 | def __getitem__(self, key): 134 | params = self.__params 135 | return params[key] 136 | 137 | def __setitem__(self, key, value): 138 | params = self.__params 139 | params[key] = value 140 | 141 | def __delitem__(self, key): 142 | params = self.__params 143 | del params[key] 144 | 145 | def __iter__(self): 146 | params = self.__params 147 | return iter(params) 148 | 149 | def __len__(self): 150 | params = self.__params 151 | return len(params) 152 | 153 | def __repr__(self): 154 | params = self.__params 155 | return repr(params) 156 | 157 | def __str__(self): 158 | params = self.__params 159 | return str(params) 160 | -------------------------------------------------------------------------------- /sigopt/sigopt_logging.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import sys 5 | from logging import INFO, StreamHandler, getLogger 6 | 7 | 8 | print_logger = getLogger("sigopt.print") 9 | print_logger.setLevel(INFO) 10 | 11 | stdout_handler = StreamHandler(stream=sys.stdout) 12 | 13 | 14 | def enable_print_logging(): 15 | global print_logger, stdout_handler 16 | print_logger.removeHandler(stdout_handler) 17 | print_logger.addHandler(stdout_handler) 18 | -------------------------------------------------------------------------------- /sigopt/urllib3_patch.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import logging 5 | import time 6 | 7 | from urllib3.connection import HTTPConnection, HTTPSConnection 8 | from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool 9 | 10 | 11 | logger = logging.getLogger("sigopt.urllib3_patch") 12 | 13 | 14 | class SigOptHTTPConnection(HTTPConnection): 15 | """ 16 | Tracks the time since the last activity of the connection. 17 | """ 18 | 19 | def __init__(self, *args, **kwargs): 20 | super().__init__(*args, **kwargs) 21 | self.reset_activity() 22 | 23 | def reset_activity(self): 24 | self.last_activity = time.time() 25 | 26 | def request(self, *args, **kwargs): 27 | super().request(*args, **kwargs) 28 | self.reset_activity() 29 | 30 | def request_chunked(self, *args, **kwargs): 31 | super().request_chunked(*args, **kwargs) 32 | self.reset_activity() 33 | 34 | def close(self, *args, **kwargs): 35 | super().close() 36 | self.reset_activity() 37 | 38 | 39 | class SigOptHTTPSConnection(SigOptHTTPConnection, HTTPSConnection): 40 | pass 41 | 42 | 43 | class ExpiringHTTPConnectionPool(HTTPConnectionPool): 44 | """ 45 | Returns a new connection when the time since the connection was last used is greater than the expiration time. 46 | """ 47 | 48 | ConnectionCls = SigOptHTTPConnection 49 | 50 | def __init__(self, *args, expiration_seconds=30, **kwargs): 51 | super().__init__(*args, **kwargs) 52 | self.expiration_seconds = expiration_seconds 53 | 54 | def _get_conn(self, timeout=None): 55 | conn = super()._get_conn(timeout=timeout) 56 | if time.time() - conn.last_activity > self.expiration_seconds: 57 | logger.debug("Abandoning expired connection") 58 | return self._new_conn() 59 | return conn 60 | 61 | 62 | class ExpiringHTTPSConnectionPool(ExpiringHTTPConnectionPool, HTTPSConnectionPool): 63 | ConnectionCls = SigOptHTTPSConnection 64 | -------------------------------------------------------------------------------- /sigopt/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | 6 | def batcher(alist, n=1): 7 | l = len(alist) 8 | for ndx in range(0, l, n): 9 | yield alist[ndx : min(ndx + n, l)] 10 | -------------------------------------------------------------------------------- /sigopt/validate/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .aiexperiment_input import validate_aiexperiment_input 5 | from .common import validate_top_level_dict 6 | from .exceptions import ValidationError 7 | from .run_input import validate_run_input 8 | -------------------------------------------------------------------------------- /sigopt/validate/common.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .exceptions import ValidationError 5 | 6 | 7 | def validate_top_level_dict(input_data): 8 | if input_data is None: 9 | return {} 10 | if not isinstance(input_data, dict): 11 | raise ValidationError("The top level should be a mapping of keys to values") 12 | return input_data 13 | -------------------------------------------------------------------------------- /sigopt/validate/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | class ValidationError(TypeError): 5 | pass 6 | -------------------------------------------------------------------------------- /sigopt/validate/keys.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | PROJECT_KEY = "project" 5 | -------------------------------------------------------------------------------- /sigopt/validate/run_input.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from sigopt.lib import is_mapping, is_sequence, is_string, validate_name 5 | 6 | from .common import validate_top_level_dict 7 | from .exceptions import ValidationError 8 | 9 | 10 | def validate_run_input(run_input): 11 | run_input = validate_top_level_dict(run_input) 12 | validated = {} 13 | name = run_input.get("name") 14 | if name is not None: 15 | try: 16 | validate_name("run name", name) 17 | except ValueError as ve: 18 | raise ValidationError(str(ve)) from ve 19 | validated["name"] = name 20 | args = run_input.get("run") 21 | if args is None: 22 | args = [] 23 | elif is_string(args): 24 | args = ["sh", "-c", args] 25 | elif is_sequence(args): 26 | args = list(args) 27 | for arg in args: 28 | if not is_string(arg): 29 | raise ValidationError("'run' has some non-string arguments") 30 | else: 31 | raise ValidationError("'run' must be a command string or list of command arguments") 32 | validated["run"] = args 33 | image = run_input.get("image") 34 | if image is not None: 35 | try: 36 | validate_name("run image", image) 37 | except ValueError as ve: 38 | raise ValidationError(str(ve)) from ve 39 | validated["image"] = image 40 | resources = run_input.get("resources") 41 | if resources is None: 42 | resources = {} 43 | else: 44 | if not is_mapping(resources): 45 | raise ValidationError("'resources' must be a mapping of key strings to values") 46 | for key in resources: 47 | if not is_string(key): 48 | raise ValidationError("'resources' can only have string keys") 49 | validated["resources"] = resources 50 | unknown_keys = set(run_input) - set(validated) 51 | if unknown_keys: 52 | joined_keys = ", ".join(unknown_keys) 53 | raise ValidationError(f"unknown run options: {joined_keys}") 54 | return validated 55 | -------------------------------------------------------------------------------- /sigopt/version.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | VERSION = "8.8.3" 5 | -------------------------------------------------------------------------------- /sigopt/xgboost/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from .experiment import experiment 5 | from .run import run 6 | -------------------------------------------------------------------------------- /sigopt/xgboost/checkpoint_callback.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from ..decorators import public 5 | from .compat import xgboost 6 | 7 | 8 | class SigOptCheckpointCallback(xgboost.callback.TrainingCallback): 9 | def __init__(self, run, period=1): 10 | self.run = run 11 | self.period = period 12 | self._latest = None 13 | super().__init__() 14 | 15 | @public 16 | def after_iteration(self, model, epoch, evals_log): 17 | if not evals_log: 18 | return False 19 | 20 | checkpoint_logs = {} 21 | for dataset, metric_dict in evals_log.items(): 22 | for metric_label, metric_record in metric_dict.items(): 23 | if isinstance(metric_record[-1], tuple): 24 | chkpt_value = metric_record[-1][0] 25 | else: 26 | chkpt_value = metric_record[-1] 27 | checkpoint_logs.update({"-".join((dataset, metric_label)): chkpt_value}) 28 | if (epoch % self.period) == 0 or self.period == 1: 29 | self.run.log_checkpoint(checkpoint_logs) 30 | self._latest = None 31 | else: 32 | self._latest = checkpoint_logs 33 | 34 | return False 35 | 36 | @public 37 | def after_training(self, model): 38 | if self._latest is not None: 39 | self.run.log_checkpoint(self._latest) 40 | return model 41 | -------------------------------------------------------------------------------- /sigopt/xgboost/compat.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | # pylint: disable=unused-import 5 | from packaging.version import parse 6 | 7 | 8 | MIN_REQUIRED_XGBOOST_VERSION = "1.3.0" 9 | 10 | try: 11 | import xgboost 12 | from xgboost import Booster, DMatrix 13 | except ImportError as ie_xgb: 14 | raise ImportError( 15 | "xgboost needs to be installed in order to use sigopt.xgboost.run" 16 | " functionality. Try running `pip install 'sigopt[xgboost]'`." 17 | ) from ie_xgb 18 | 19 | if parse(xgboost.__version__) < parse(MIN_REQUIRED_XGBOOST_VERSION): 20 | raise ImportError( 21 | "sigopt.xgboost.run is compatible with" 22 | f" xgboost>={MIN_REQUIRED_XGBOOST_VERSION}. You have version" 23 | f" {xgboost.__version__} installed." 24 | ) 25 | -------------------------------------------------------------------------------- /sigopt/xgboost/compute_metrics.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import math 5 | 6 | import numpy 7 | 8 | from .compat import Booster 9 | 10 | 11 | def compute_positives_and_negatives(y_true, y_pred, class_label): 12 | y_true_equals = y_true == class_label 13 | y_true_notequals = y_true != class_label 14 | y_pred_equals = y_pred == class_label 15 | y_pred_notequals = y_pred != class_label 16 | tp = numpy.count_nonzero(numpy.logical_and(y_true_equals, y_pred_equals)) 17 | tn = numpy.count_nonzero(numpy.logical_and(y_true_notequals, y_pred_notequals)) 18 | fp = numpy.count_nonzero(numpy.logical_and(y_true_notequals, y_pred_equals)) 19 | fn = numpy.count_nonzero(numpy.logical_and(y_true_equals, y_pred_notequals)) 20 | return tp, tn, fp, fn 21 | 22 | 23 | def compute_accuracy(y_true, y_pred): 24 | accuracy = numpy.count_nonzero(y_true == y_pred) / len(y_true) 25 | return accuracy 26 | 27 | 28 | def compute_classification_report(y_true, y_pred): 29 | classes = numpy.unique(y_true) 30 | classification_report = {} 31 | classification_report["weighted avg"] = { 32 | "f1-score": 0, 33 | "recall": 0, 34 | "precision": 0, 35 | } 36 | for class_label in classes: 37 | tp, _, fp, fn = compute_positives_and_negatives(y_true, y_pred, class_label) 38 | precision = tp / (tp + fp) if (tp + fp) != 0 else 0 39 | recall = tp / (tp + fn) if (tp + fn) != 0 else 0 40 | f1 = tp / (tp + 0.5 * (fp + fn)) if not math.isclose((tp + 0.5 * (fp + fn)), 0, rel_tol=1e-09, abs_tol=0.0) else 0 41 | support = numpy.count_nonzero(y_true == class_label) 42 | classification_report[str(class_label)] = { 43 | "precision": precision, 44 | "recall": recall, 45 | "f1-score": f1, 46 | "support": support, 47 | } 48 | classification_report["weighted avg"]["precision"] += (support / len(y_pred)) * precision 49 | classification_report["weighted avg"]["recall"] += (support / len(y_pred)) * recall 50 | classification_report["weighted avg"]["f1-score"] += (support / len(y_pred)) * f1 51 | return classification_report 52 | 53 | 54 | def compute_mae(y_true, y_pred): 55 | d = y_true - y_pred 56 | return numpy.mean(abs(d)) 57 | 58 | 59 | def compute_mse(y_true, y_pred): 60 | d = y_true - y_pred 61 | return numpy.mean(d**2) 62 | 63 | 64 | def compute_classification_metrics(model, D_matrix_pair): 65 | assert isinstance(model, Booster) 66 | D_matrix, D_name = D_matrix_pair 67 | preds = model.predict(D_matrix) 68 | # Check shape of preds 69 | if len(preds.shape) == 2: 70 | preds = numpy.argmax(preds, axis=1) 71 | else: 72 | preds = numpy.round(preds) 73 | y_test = D_matrix.get_label() 74 | accuracy = compute_accuracy(y_test, preds) 75 | rep = compute_classification_report(y_test, preds) 76 | other_metrics = rep["weighted avg"] 77 | return { 78 | f"{D_name}-accuracy": accuracy, 79 | f"{D_name}-F1": other_metrics["f1-score"], 80 | f"{D_name}-recall": other_metrics["recall"], 81 | f"{D_name}-precision": other_metrics["precision"], 82 | } 83 | 84 | 85 | def compute_regression_metrics(model, D_matrix_pair): 86 | assert isinstance(model, Booster) 87 | D_matrix, D_name = D_matrix_pair 88 | preds = model.predict(D_matrix) 89 | preds = numpy.round(preds) 90 | y_test = D_matrix.get_label() 91 | return { 92 | f"{D_name}-mean absolute error": compute_mae(y_test, preds), 93 | f"{D_name}-mean squared error": compute_mse(y_test, preds), 94 | } 95 | -------------------------------------------------------------------------------- /sigopt/xgboost/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | DEFAULT_EVALS_NAME = "TestSet" 5 | USER_SOURCE_NAME = "User Specified" 6 | SIGOPT_DEFAULTS_SOURCE_NAME = "SigOpt Defaults" 7 | XGBOOST_DEFAULTS_SOURCE_NAME = "XGBoost Defaults" 8 | 9 | USER_SOURCE_PRIORITY = 2 10 | SIGOPT_DEFAULTS_SOURCE_PRIORITY = 3 11 | XGBOOST_DEFAULTS_SOURCE_PRIORITY = 4 12 | MAX_BO_ITERATIONS = 200 13 | 14 | # defaults 15 | DEFAULT_ITERS_PER_DIM = 10 16 | DEFAULT_CLASSIFICATION_METRIC = "accuracy" 17 | DEFAULT_NUM_BOOST_ROUND = 10 18 | DEFAULT_EARLY_STOPPING_ROUNDS = 10 19 | DEFAULT_REGRESSION_METRIC = "mean squared error" 20 | DEFAULT_SEARCH_PARAMS = [ 21 | "eta", 22 | "gamma", 23 | "max_depth", 24 | "min_child_weight", 25 | "num_boost_round", 26 | ] 27 | 28 | # optimization metrics 29 | CLASSIFICATION_METRIC_CHOICES = ["accuracy", "F1", "precision", "recall"] 30 | REGRESSION_METRIC_CHOICES = ["mean absolute error", "mean squared error"] 31 | SUPPORTED_METRICS_TO_OPTIMIZE = CLASSIFICATION_METRIC_CHOICES + REGRESSION_METRIC_CHOICES 32 | METRICS_OPTIMIZATION_STRATEGY = { 33 | **dict(zip(CLASSIFICATION_METRIC_CHOICES, ["maximize"] * len(CLASSIFICATION_METRIC_CHOICES))), 34 | **dict(zip(REGRESSION_METRIC_CHOICES, ["minimize"] * len(REGRESSION_METRIC_CHOICES))), 35 | } 36 | 37 | # Note: only the XGB general params. Omitted monotone_constraints and interaction_constraints b/c more complex. 38 | # Also omitting refresh_leaf b/c it's a boolean value. Only some of these have bounds which will be autofilled. 39 | # We can add arbitrary bounds to the rest (and maybe we should). 40 | PARAMETER_INFORMATION = { 41 | # Numerical Values 42 | "eta": { 43 | "type": "double", 44 | "limits": "(0, Inf]", 45 | "bounds": {"min": 0.001, "max": 1}, 46 | "transformation": "log", 47 | }, 48 | "max_delta_step": { 49 | "type": "double", 50 | "limits": "[0, Inf]", 51 | "bounds": {"min": 0.001, "max": 10}, 52 | "transformation": "log", 53 | }, 54 | "alpha": {"type": "double", "limits": "[0, Inf]", "bounds": {"min": 0, "max": 10}}, 55 | "gamma": {"type": "double", "limits": "[0, Inf]", "bounds": {"min": 0, "max": 5}}, 56 | "lambda": {"type": "double", "limits": "[0, Inf]", "bounds": {"min": 0, "max": 10}}, 57 | "max_depth": {"type": "int", "limits": "[1, Inf]", "bounds": {"min": 2, "max": 16}}, 58 | "min_child_weight": { 59 | "type": "double", 60 | "limits": "[0, Inf]", 61 | "bounds": {"min": 0, "max": 10}, 62 | }, 63 | "num_boost_round": { 64 | "type": "int", 65 | "limits": "[1, Inf]", 66 | "bounds": {"min": 10, "max": 200}, 67 | }, 68 | "colsample_bylevel": { 69 | "type": "double", 70 | "limits": "(0, 1]", 71 | "bounds": {"min": 0.5, "max": 1}, 72 | }, 73 | "colsample_bynode": { 74 | "type": "double", 75 | "limits": "(0, 1]", 76 | "bounds": {"min": 0.5, "max": 1}, 77 | }, 78 | "colsample_bytree": { 79 | "type": "double", 80 | "limits": "(0, 1]", 81 | "bounds": {"min": 0.5, "max": 1}, 82 | }, 83 | "max_bin": {"type": "int", "limits": "[1, Inf]"}, 84 | "max_leaves": {"type": "int", "limits": "[1, Inf]"}, 85 | "num_parallel_tree": {"type": "int", "limits": "[1, Inf]"}, 86 | "scale_pos_weight": {"type": "double", "limits": "[0, Inf]"}, 87 | "sketch_eps": {"type": "double", "limits": "(0, 1)"}, 88 | "subsample": { 89 | "type": "double", 90 | "limits": "(0, 1]", 91 | "bounds": {"min": 0.5, "max": 1}, 92 | }, 93 | # String values 94 | "grow_policy": {"type": "categorical", "values": ["depthwise", "lossguide"]}, 95 | "predictor": { 96 | "type": "categorical", 97 | "values": ["auto", "cpu_predictor", "gpu_predictor"], 98 | }, 99 | "process_type": {"type": "categorical", "values": ["default", "update"]}, 100 | "sampling_method": {"type": "categorical", "values": ["uniform", "gradient_based"]}, 101 | "tree_method": { 102 | "type": "categorical", 103 | "values": ["auto", "exact", "approx", "hist", "gpu_hist"], 104 | }, 105 | "updater": { 106 | "type": "categorical", 107 | "values": [ 108 | "grow_colmaker", 109 | "grow_histmaker", 110 | "grow_local_histmaker", 111 | "grow_quantile_histmaker", 112 | "grow_gpu_hist", 113 | "sync", 114 | "refresh", 115 | "prune", 116 | ], 117 | }, 118 | } 119 | 120 | SUPPORTED_AUTOBOUND_PARAMS = [name for name, info in PARAMETER_INFORMATION.items() if "bounds" in info] 121 | -------------------------------------------------------------------------------- /sigopt/xgboost/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import json 5 | 6 | from .compat import Booster 7 | 8 | 9 | def get_booster_params(booster): 10 | # refer: 11 | # https://github.com/dmlc/xgboost/blob/release_1.5.0/python-package/xgboost/sklearn.py#L522 12 | 13 | def parse_json_parameter_value(value): 14 | for convert_type in (int, float, str): 15 | try: 16 | ret = convert_type(value) 17 | return ret 18 | except ValueError: 19 | continue 20 | return str(value) 21 | 22 | assert isinstance(booster, Booster) 23 | config = json.loads(booster.save_config()) 24 | stack = [config] 25 | all_xgboost_params = {} 26 | while stack: 27 | obj = stack.pop() 28 | for k, v in obj.items(): 29 | if k.endswith("_param"): 30 | for p_k, p_v in v.items(): 31 | all_xgboost_params[p_k] = p_v 32 | elif isinstance(v, dict): 33 | stack.append(v) 34 | 35 | params = {} 36 | for k, v in all_xgboost_params.items(): 37 | params[k] = parse_json_parameter_value(v) 38 | 39 | return params 40 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-python/855b9a7a590162627ea2ef934f793a567a31b8be/test/__init__.py -------------------------------------------------------------------------------- /test/cli/test_cli_config.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import mock 5 | import pytest 6 | from click.testing import CliRunner 7 | 8 | from sigopt.cli import cli 9 | 10 | 11 | class TestRunCli(object): 12 | @pytest.mark.parametrize("opt_into_log_collection", [False, True]) 13 | @pytest.mark.parametrize("opt_into_cell_tracking", [False, True]) 14 | def test_config_command(self, opt_into_log_collection, opt_into_cell_tracking): 15 | runner = CliRunner() 16 | log_collection_arg = "--enable-log-collection" if opt_into_log_collection else "--no-enable-log-collection" 17 | cell_tracking_arg = "--enable-cell-tracking" if opt_into_cell_tracking else "--no-enable-cell-tracking" 18 | with mock.patch( 19 | "sigopt.cli.commands.config._config.persist_configuration_options" 20 | ) as persist_configuration_options: 21 | result = runner.invoke( 22 | cli, 23 | [ 24 | "config", 25 | "--api-token=some_test_token", 26 | log_collection_arg, 27 | cell_tracking_arg, 28 | ], 29 | ) 30 | persist_configuration_options.assert_called_once_with( 31 | { 32 | "api_token": "some_test_token", 33 | "code_tracking_enabled": opt_into_cell_tracking, 34 | "log_collection_enabled": opt_into_log_collection, 35 | } 36 | ) 37 | assert result.exit_code == 0 38 | assert result.output == "" 39 | -------------------------------------------------------------------------------- /test/cli/test_files/import_hello.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import print_hello # pylint: disable=unused-import 5 | 6 | 7 | del print_hello 8 | -------------------------------------------------------------------------------- /test/cli/test_files/invalid_sigopt.yml: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test/cli/test_files/print_args.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from __future__ import print_function 5 | 6 | import sys 7 | 8 | 9 | for arg in sys.argv: 10 | print(arg) 11 | -------------------------------------------------------------------------------- /test/cli/test_files/print_hello.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from __future__ import print_function 5 | 6 | 7 | print("hello") 8 | -------------------------------------------------------------------------------- /test/cli/test_files/valid_sigopt.yml: -------------------------------------------------------------------------------- 1 | name: e1 2 | parameters: 3 | - name: p1 4 | type: double 5 | bound: 6 | min: 0 7 | max: 1 8 | metrics: 9 | - name: m1 10 | -------------------------------------------------------------------------------- /test/cli/test_files/warning_sigopt.yml: -------------------------------------------------------------------------------- 1 | wrongkey: 2 | experiment: 3 | name: top level 4 | parameters: 5 | - name: this is not right 6 | type: double 7 | bounds: 8 | min: 0 9 | max: 1 10 | -------------------------------------------------------------------------------- /test/cli/test_init.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | 6 | from click.testing import CliRunner 7 | 8 | from sigopt.cli import cli 9 | 10 | 11 | class TestRunCli(object): 12 | def test_init_command_empty_dir(self): 13 | files = [ 14 | "run.yml", 15 | "experiment.yml", 16 | "Dockerfile", 17 | ".dockerignore", 18 | ] 19 | runner = CliRunner() 20 | with runner.isolated_filesystem(): 21 | result = runner.invoke(cli, ["init"]) 22 | for filename in files: 23 | assert os.path.exists(filename) 24 | assert result.exit_code == 0 25 | lines = result.output.splitlines() 26 | assert len(lines) == len(files) 27 | for line, filename in zip(lines, files): 28 | assert line == f"Wrote file contents for {filename}" 29 | -------------------------------------------------------------------------------- /test/cli/test_optimize.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | import shutil 6 | 7 | import mock 8 | import pytest 9 | from click.testing import CliRunner 10 | 11 | from sigopt.aiexperiment_context import AIExperimentContext 12 | from sigopt.cli import cli 13 | from sigopt.run_context import RunContext 14 | 15 | 16 | class TestRunCli(object): 17 | @pytest.fixture 18 | def run_context(self): 19 | run = RunContext(mock.Mock(), mock.Mock(assignments={"fixed1": 0, "fixed2": "test"})) 20 | run.to_json = mock.Mock(return_value={"run": {}}) 21 | run._end = mock.Mock() 22 | run._log_source_code = mock.Mock() 23 | return run 24 | 25 | @pytest.fixture(autouse=True) 26 | def patch_experiment(self, run_context): 27 | with mock.patch( 28 | "sigopt.cli.commands.local.optimize.create_aiexperiment_from_validated_data" 29 | ) as create_aiexperiment: 30 | experiment = AIExperimentContext(mock.Mock(project="test-project"), mock.Mock()) 31 | experiment.create_run = mock.Mock(return_value=run_context) 32 | experiment.refresh = mock.Mock() 33 | experiment.is_finished = mock.Mock(side_effect=[False, True]) 34 | create_aiexperiment.return_value = experiment 35 | yield 36 | 37 | @pytest.fixture 38 | def runner(self): 39 | runner = CliRunner() 40 | root = os.path.abspath("test/cli/test_files") 41 | with runner.isolated_filesystem(): 42 | for file in [ 43 | "print_hello.py", 44 | "print_args.py", 45 | "import_hello.py", 46 | "valid_sigopt.yml", 47 | "invalid_sigopt.yml", 48 | ]: 49 | shutil.copy(os.path.join(root, file), ".") 50 | yield runner 51 | 52 | def test_optimize_command(self, runner): 53 | result = runner.invoke( 54 | cli, 55 | [ 56 | "optimize", 57 | "--experiment-file=valid_sigopt.yml", 58 | "python", 59 | "print_hello.py", 60 | ], 61 | ) 62 | assert result.output == "hello\n" 63 | assert result.exit_code == 0 64 | 65 | def test_optimize_command_with_args(self, runner): 66 | result = runner.invoke( 67 | cli, 68 | [ 69 | "optimize", 70 | "--experiment-file=valid_sigopt.yml", 71 | "python", 72 | "print_args.py", 73 | "--kwarg=value", 74 | "positional_arg", 75 | "--", 76 | "after -- arg", 77 | ], 78 | ) 79 | assert result.output == "print_args.py\n--kwarg=value\npositional_arg\n--\nafter -- arg\n" 80 | assert result.exit_code == 0 81 | 82 | def test_optimize_command_track_source_code(self, runner, run_context): 83 | runner.invoke( 84 | cli, 85 | [ 86 | "optimize", 87 | "--experiment-file=valid_sigopt.yml", 88 | "--source-file=print_args.py", 89 | "python", 90 | "print_args.py", 91 | ], 92 | ) 93 | with open("print_args.py") as fp: 94 | content = fp.read() 95 | run_context._log_source_code.assert_called_once_with({"content": content}) 96 | 97 | def test_optimize_command_needs_existing_sigopt_yaml(self, runner): 98 | runner = CliRunner() 99 | result = runner.invoke(cli, ["optimize", "python", "print_hello.py"]) 100 | assert "Path 'experiment.yml' does not exist" in result.output 101 | assert result.exit_code == 2 102 | 103 | def test_optimize_command_needs_valid_sigopt_yaml(self, runner): 104 | runner = CliRunner() 105 | result = runner.invoke( 106 | cli, 107 | [ 108 | "optimize", 109 | "--experiment-file=invalid_sigopt.yml", 110 | "python", 111 | "print_hello.py", 112 | ], 113 | ) 114 | assert "The top level should be a mapping of keys to values" in str(result.output) 115 | assert result.exit_code == 2 116 | -------------------------------------------------------------------------------- /test/cli/test_run.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | import shutil 6 | 7 | import mock 8 | import pytest 9 | from click.testing import CliRunner 10 | 11 | from sigopt.cli import cli 12 | from sigopt.run_context import RunContext 13 | 14 | 15 | class TestRunCli(object): 16 | @pytest.fixture 17 | def run_context(self): 18 | run = RunContext(mock.Mock(), mock.Mock(assignments={"fixed1": 0, "fixed2": "test"})) 19 | run.to_json = mock.Mock(return_value={"run": {}}) 20 | run._end = mock.Mock() 21 | run._log_source_code = mock.Mock() 22 | return run 23 | 24 | @pytest.fixture(autouse=True) 25 | def patch_run_factory(self, run_context): 26 | with mock.patch("sigopt.cli.commands.local.run.SigOptFactory") as factory: 27 | instance = mock.Mock() 28 | instance.create_run.return_value = run_context 29 | factory.return_value = instance 30 | yield 31 | 32 | @pytest.fixture 33 | def runner(self): 34 | runner = CliRunner() 35 | root = os.path.abspath("test/cli/test_files") 36 | with runner.isolated_filesystem(): 37 | for file in [ 38 | "print_hello.py", 39 | "print_args.py", 40 | "import_hello.py", 41 | ]: 42 | shutil.copy(os.path.join(root, file), ".") 43 | yield runner 44 | 45 | def test_run_command_echo(self, runner): 46 | result = runner.invoke( 47 | cli, 48 | [ 49 | "run", 50 | "echo", 51 | "hello", 52 | ], 53 | ) 54 | assert result.exit_code == 0 55 | assert result.output == "hello\n" 56 | 57 | def test_run_command(self, runner): 58 | runner = CliRunner() 59 | result = runner.invoke( 60 | cli, 61 | [ 62 | "run", 63 | "python", 64 | "print_hello.py", 65 | ], 66 | ) 67 | assert result.exit_code == 0 68 | assert result.output == "hello\n" 69 | 70 | def test_run_command_with_args(self, runner): 71 | result = runner.invoke( 72 | cli, 73 | [ 74 | "run", 75 | "python", 76 | "print_args.py", 77 | "--kwarg=value", 78 | "positional_arg", 79 | ], 80 | ) 81 | assert result.output == "print_args.py\n--kwarg=value\npositional_arg\n" 82 | assert result.exit_code == 0 83 | 84 | def test_run_command_import_sibling(self, runner): 85 | result = runner.invoke( 86 | cli, 87 | [ 88 | "run", 89 | "python", 90 | "import_hello.py", 91 | ], 92 | ) 93 | assert result.output == "hello\n" 94 | assert result.exit_code == 0 95 | 96 | def test_run_command_track_source_code(self, runner, run_context): 97 | runner.invoke( 98 | cli, 99 | [ 100 | "run", 101 | "--source-file=print_hello.py", 102 | "python", 103 | "print_hello.py", 104 | ], 105 | ) 106 | with open("print_hello.py") as fp: 107 | content = fp.read() 108 | run_context._log_source_code.assert_called_once_with({"content": content}) 109 | -------------------------------------------------------------------------------- /test/cli/test_start_worker.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | import shutil 6 | 7 | import mock 8 | import pytest 9 | from click.testing import CliRunner 10 | 11 | from sigopt.aiexperiment_context import AIExperimentContext 12 | from sigopt.cli import cli 13 | from sigopt.run_context import RunContext 14 | 15 | 16 | class TestRunCli(object): 17 | @pytest.fixture 18 | def run_context(self): 19 | run = RunContext(mock.Mock(), mock.Mock(assignments={"fixed1": 0, "fixed2": "test"})) 20 | run.to_json = mock.Mock(return_value={"run": {}}) 21 | run._end = mock.Mock() 22 | run._log_source_code = mock.Mock() 23 | return run 24 | 25 | @pytest.fixture(autouse=True) 26 | def patch_run_factory(self, run_context): 27 | with mock.patch("sigopt.cli.commands.local.start_worker.SigOptFactory") as factory: 28 | experiment = AIExperimentContext(mock.Mock(project="test-project"), mock.Mock()) 29 | experiment.create_run = mock.Mock(return_value=run_context) 30 | experiment.refresh = mock.Mock() 31 | experiment.is_finished = mock.Mock(side_effect=[False, True]) 32 | instance = mock.Mock() 33 | instance.get_aiexperiment.return_value = experiment 34 | factory.from_default_project = mock.Mock(return_value=instance) 35 | yield 36 | 37 | @pytest.fixture 38 | def runner(self): 39 | runner = CliRunner() 40 | root = os.path.abspath("test/cli/test_files") 41 | with runner.isolated_filesystem(): 42 | for file in [ 43 | "print_hello.py", 44 | "print_args.py", 45 | "import_hello.py", 46 | ]: 47 | shutil.copy(os.path.join(root, file), ".") 48 | yield runner 49 | 50 | def test_start_worker_command(self, runner): 51 | result = runner.invoke( 52 | cli, 53 | [ 54 | "start-worker", 55 | "1234", 56 | "python", 57 | "print_hello.py", 58 | ], 59 | ) 60 | assert result.output == "hello\n" 61 | assert result.exit_code == 0 62 | 63 | def test_start_worker_command_with_args(self, runner): 64 | result = runner.invoke( 65 | cli, 66 | [ 67 | "start-worker", 68 | "1234", 69 | "python", 70 | "print_args.py", 71 | "--kwarg=value", 72 | "positional_arg", 73 | "--", 74 | "after -- arg", 75 | ], 76 | ) 77 | assert result.output == "print_args.py\n--kwarg=value\npositional_arg\n--\nafter -- arg\n" 78 | assert result.exit_code == 0 79 | 80 | def test_start_worker_command_track_source_code(self, runner, run_context): 81 | runner.invoke( 82 | cli, 83 | [ 84 | "start-worker", 85 | "--source-file=print_args.py", 86 | "1234", 87 | "python", 88 | "print_args.py", 89 | ], 90 | ) 91 | with open("print_args.py") as fp: 92 | content = fp.read() 93 | run_context._log_source_code.assert_called_once_with({"content": content}) 94 | -------------------------------------------------------------------------------- /test/cli/test_truncate.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from sigopt.run_context import maybe_truncate_log 5 | 6 | 7 | def test_short_logs_dont_get_truncated(): 8 | short_logs = "hello there\n" 9 | content = maybe_truncate_log(short_logs) 10 | assert content == short_logs 11 | 12 | 13 | def test_long_logs_get_truncated(): 14 | max_size = 1024 15 | long_logs = "a" * (max_size * 2) + "\n" 16 | content = maybe_truncate_log(long_logs) 17 | assert max_size < len(content) < max_size * 2 18 | assert content.startswith("[ WARNING ] ") 19 | assert "... truncated ..." in content 20 | assert content.endswith("aaaa\n") 21 | -------------------------------------------------------------------------------- /test/cli/test_version.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | from click.testing import CliRunner 5 | 6 | from sigopt.cli import cli 7 | from sigopt.version import VERSION 8 | 9 | 10 | class TestVersionCli(object): 11 | def test_version_command(self): 12 | runner = CliRunner() 13 | result = runner.invoke(cli, ["version"]) 14 | assert result.exit_code == 0 15 | assert result.output == VERSION + "\n" 16 | -------------------------------------------------------------------------------- /test/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-python/855b9a7a590162627ea2ef934f793a567a31b8be/test/client/__init__.py -------------------------------------------------------------------------------- /test/client/json_data/client.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": "client", 3 | "id": "1", 4 | "name": "Client", 5 | "created": 123, 6 | "organization": "2" 7 | } 8 | -------------------------------------------------------------------------------- /test/client/json_data/experiment.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": "experiment", 3 | "id": "123", 4 | "name": "Test Experiment", 5 | "parallel_bandwidth": 2, 6 | "created": 321, 7 | "state": "active", 8 | "linear_constraints": [ 9 | { 10 | "type": "greater_than", 11 | "terms": [ 12 | { 13 | "name": "a", 14 | "weight": 2 15 | } 16 | ], 17 | "threshold": 5 18 | } 19 | ], 20 | "conditionals": [ 21 | { 22 | "object": "conditional", 23 | "name": "num_hidden_layers", 24 | "values": [ 25 | "1", 26 | "3" 27 | ] 28 | } 29 | ], 30 | "metrics": [ 31 | { 32 | "object": "metric", 33 | "name": "Revenue", 34 | "objective": "maximize", 35 | "strategy": "optimize", 36 | "threshold": null 37 | }, 38 | { 39 | "object": "metric", 40 | "name": "Sales", 41 | "strategy": "optimize", 42 | "threshold": -3.0 43 | } 44 | ], 45 | "client": "678", 46 | "progress": { 47 | "object": "progress", 48 | "observation_count": 3, 49 | "observation_budget_consumed": 3.0, 50 | "first_observation": { 51 | "object": "observation", 52 | "id": "1", 53 | "assignments": { 54 | "a": 1, 55 | "b": "c" 56 | }, 57 | "values": [ 58 | { 59 | "object": "value", 60 | "name": "Revenue", 61 | "value": 3.1, 62 | "value_stddev": null 63 | }, 64 | { 65 | "object": "value", 66 | "name": "Sales", 67 | "value": 2.5, 68 | "value_stddev": null 69 | } 70 | ], 71 | "failed": false, 72 | "created": 451, 73 | "suggestion": "11", 74 | "experiment": "123" 75 | }, 76 | "last_observation": { 77 | "object": "observation", 78 | "id": "2", 79 | "assignments": { 80 | "a": 2, 81 | "b": "d" 82 | }, 83 | "values": [ 84 | { 85 | "object": "value", 86 | "name": "Revenue", 87 | "value": 3.1, 88 | "value_stddev": 0.5 89 | }, 90 | { 91 | "object": "value", 92 | "name": "Sales", 93 | "value": 2.5, 94 | "value_stddev": 0.8 95 | } 96 | ], 97 | "failed": false, 98 | "created": 452, 99 | "suggestion": "12", 100 | "experiment": "123" 101 | }, 102 | "best_observation": { 103 | "object": "observation", 104 | "id": "3", 105 | "assignments": { 106 | "a": 3, 107 | "b": "d" 108 | }, 109 | "values": [ 110 | { 111 | "object": "value", 112 | "name": "Revenue", 113 | "value": null, 114 | "value_stddev": null 115 | }, 116 | { 117 | "object": "value", 118 | "name": "Sales", 119 | "value": null, 120 | "value_stddev": null 121 | } 122 | ], 123 | "failed": true, 124 | "created": 453, 125 | "suggestion": "13", 126 | "experiment": "123", 127 | "metadata": { 128 | "abc": "def", 129 | "ghi": 123 130 | } 131 | } 132 | }, 133 | "parameters": [ 134 | { 135 | "object": "parameter", 136 | "name": "a", 137 | "type": "double", 138 | "bounds": { 139 | "object": "bounds", 140 | "min": 1, 141 | "max": 2 142 | }, 143 | "categorical_values": null, 144 | "precision": 3, 145 | "default_value": 2, 146 | "conditions": { 147 | "num_hidden_layers": [] 148 | } 149 | }, 150 | { 151 | "object": "parameter", 152 | "name": "b", 153 | "type": "categorical", 154 | "bounds": null, 155 | "categorical_values": [ 156 | { 157 | "name": "c", 158 | "enum_index": 1 159 | }, 160 | { 161 | "name": "d", 162 | "enum_index": 2 163 | } 164 | ], 165 | "precision": null, 166 | "default_value": null, 167 | "conditions": { 168 | "num_hidden_layers": [ 169 | "1", 170 | "3" 171 | ] 172 | } 173 | } 174 | ], 175 | "metadata": { 176 | "abc": "def", 177 | "ghi": 123 178 | }, 179 | "tasks": [ 180 | { 181 | "cost": 0.567, 182 | "name": "task 1", 183 | "object": "task" 184 | }, 185 | { 186 | "cost": 1.0, 187 | "name": "task 2", 188 | "object": "task" 189 | } 190 | ], 191 | "updated": 453, 192 | "user": "789" 193 | } 194 | -------------------------------------------------------------------------------- /test/client/json_data/importances.json: -------------------------------------------------------------------------------- 1 | { 2 | "importances": { 3 | "a": 0.92, 4 | "b": 0.03 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /test/client/json_data/metric.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Test", 3 | "objective": "maximize", 4 | "strategy": "optimize", 5 | "threshold": null 6 | } 7 | -------------------------------------------------------------------------------- /test/client/json_data/metric_importances.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": "metric_importances", 3 | "metric": "metric1", 4 | "importances": { 5 | "parameter_1": 0.92, 6 | "parameter_2": 0.65, 7 | "parameter_3": 0.03 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /test/client/json_data/organization.json: -------------------------------------------------------------------------------- 1 | { 2 | "created": 123456, 3 | "id": "7890", 4 | "name": "SigOpt", 5 | "object": "organization" 6 | } 7 | -------------------------------------------------------------------------------- /test/client/json_data/pagination.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": "pagination", 3 | "count": 2, 4 | "data": [ 5 | { 6 | "object": "experiment" 7 | } 8 | ], 9 | "paging": { 10 | "before": "1", 11 | "after": "2" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /test/client/json_data/plan.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": "plan", 3 | "id": "premium", 4 | "name": "SigOpt Premium", 5 | "rules": { 6 | "max_dimension": 1, 7 | "max_experiments": 2, 8 | "max_observations": 3, 9 | "max_parallelism": 4 10 | }, 11 | "current_period": { 12 | "start": 0, 13 | "end": 1000, 14 | "experiments": [ 15 | "1", 16 | "2" 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test/client/json_data/queued_suggestion.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": "queued_suggestion", 3 | "id": "1", 4 | "assignments": { 5 | "a": 1, 6 | "b": "c" 7 | }, 8 | "experiment": "1", 9 | "created": 123, 10 | "task": { 11 | "cost": 0.567, 12 | "name": "task 1", 13 | "object": "task" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test/client/json_data/suggestion.json: -------------------------------------------------------------------------------- 1 | { 2 | "object": "suggestion", 3 | "id": "1", 4 | "assignments": { 5 | "a": 1, 6 | "b": "c" 7 | }, 8 | "state": "open", 9 | "experiment": "1", 10 | "created": 123, 11 | "metadata": { 12 | "abc": "def", 13 | "ghi": 123 14 | }, 15 | "task": { 16 | "cost": 0.567, 17 | "name": "task 1", 18 | "object": "task" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /test/client/json_data/task.json: -------------------------------------------------------------------------------- 1 | { 2 | "cost": 0.567, 3 | "name": "task 1", 4 | "object": "task" 5 | } 6 | -------------------------------------------------------------------------------- /test/client/json_data/token.json: -------------------------------------------------------------------------------- 1 | { 2 | "all_experiments": false, 3 | "development": true, 4 | "permissions": "read", 5 | "token_type": "client-dev", 6 | "expires": 1547139000, 7 | "token": "123", 8 | "client": "456", 9 | "experiment": "1", 10 | "user": "789" 11 | } 12 | -------------------------------------------------------------------------------- /test/client/json_data/training_run.json: -------------------------------------------------------------------------------- 1 | { 2 | "assignments": { 3 | "m": 1, 4 | "n": 2 5 | }, 6 | "best_checkpoint": "cp0", 7 | "checkpoint_count": 2, 8 | "client": "11674", 9 | "completed": 1631654369, 10 | "created": 1631654365, 11 | "datasets": { 12 | "dataset1": { 13 | "object": "dataset" 14 | }, 15 | "dataset2": { 16 | "object": "dataset" 17 | } 18 | }, 19 | "deleted": false, 20 | "experiment": "432979", 21 | "favorite": false, 22 | "files": ["f0", "f1"], 23 | "finished": true, 24 | "id": "95658", 25 | "logs": { 26 | "stderr": { 27 | "content": "message stderr\n", 28 | "object": "log" 29 | }, 30 | "stdout": { 31 | "content": "message stdout\n", 32 | "object": "log" 33 | } 34 | }, 35 | "metadata": { 36 | "m0": "v0", 37 | "m1": "v1" 38 | }, 39 | "model": { 40 | "object": "model", 41 | "type": "type0" 42 | }, 43 | "name": "run-examples 2021-09-14 14:19:26", 44 | "object": "training_run", 45 | "observation": "31818393", 46 | "project": "run-examples", 47 | "source_code": { 48 | "content": "c0", 49 | "hash": "h0", 50 | "object": "source_code" 51 | }, 52 | "state": "completed", 53 | "suggestion": "46227605", 54 | "tags": ["t0", "t1"], 55 | "updated": 1631654369, 56 | "user": "53628", 57 | "values": { 58 | "accuracy": { 59 | "name": "accuracy", 60 | "object": "metric_evaluation", 61 | "value": 1, 62 | "value_stddev": 0.1 63 | }, 64 | "f1": { 65 | "name": "f1", 66 | "object": "metric_evaluation", 67 | "value": 2, 68 | "value_stddev": 0.2 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /test/client/test_interface.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import os 5 | 6 | import mock 7 | import pytest 8 | 9 | from sigopt.config import config 10 | from sigopt.interface import Connection 11 | from sigopt.request_driver import DEFAULT_HTTP_TIMEOUT 12 | from sigopt.resource import ApiResource 13 | 14 | 15 | class TestInterface(object): 16 | @pytest.yield_fixture 17 | def config_dict(self, autouse=True): 18 | del autouse 19 | with mock.patch.dict(config._configuration, {}): 20 | yield config._configuration 21 | 22 | def test_create(self): 23 | conn = Connection(client_token="client_token") 24 | assert conn.impl.driver.api_url == "https://api.sigopt.com" 25 | assert conn.impl.driver.verify_ssl_certs is None 26 | assert conn.impl.driver.proxies is None 27 | assert conn.impl.driver.timeout == DEFAULT_HTTP_TIMEOUT 28 | assert conn.impl.driver.auth.username == "client_token" 29 | assert isinstance(conn.clients, ApiResource) 30 | assert isinstance(conn.experiments, ApiResource) 31 | 32 | def test_create_uses_session_if_provided(self): 33 | session = mock.Mock() 34 | conn = Connection(client_token="client_token", session=session) 35 | assert conn.impl.driver.session is session 36 | 37 | response = mock.Mock() 38 | session.request.return_value = response 39 | response.status_code = 200 40 | response.text = "{}" 41 | session.request.assert_not_called() 42 | conn.experiments().fetch() 43 | session.request.assert_called_once() 44 | 45 | def test_environment_variable(self): 46 | with mock.patch.dict(os.environ, {"SIGOPT_API_TOKEN": "client_token"}): 47 | conn = Connection() 48 | assert conn.impl.driver.auth.username == "client_token" 49 | 50 | def test_token_in_config(self, config_dict): 51 | with mock.patch.dict(config_dict, {"api_token": "test_token_in_config"}), mock.patch.dict(os.environ, {}): 52 | conn = Connection() 53 | assert conn.impl.driver.auth.username == "test_token_in_config" 54 | 55 | def test_api_url(self): 56 | conn = Connection("client_token") 57 | conn.set_api_url("https://api-test.sigopt.com") 58 | assert conn.impl.driver.api_url == "https://api-test.sigopt.com" 59 | 60 | def test_api_url_env(self): 61 | with mock.patch.dict(os.environ, {"SIGOPT_API_URL": "https://api-env.sigopt.com"}): 62 | conn = Connection("client_token") 63 | assert conn.impl.driver.api_url == "https://api-env.sigopt.com" 64 | 65 | def test_verify(self): 66 | conn = Connection("client_token") 67 | conn.set_verify_ssl_certs(False) 68 | assert conn.impl.driver.verify_ssl_certs is False 69 | 70 | def test_proxies(self): 71 | conn = Connection("client_token") 72 | conn.set_proxies({"http": "http://127.0.0.1:6543"}) 73 | assert conn.impl.driver.proxies["http"] == "http://127.0.0.1:6543" 74 | 75 | def test_error(self): 76 | with mock.patch.dict(os.environ, {"SIGOPT_API_TOKEN": ""}): 77 | with pytest.raises(ValueError): 78 | Connection() 79 | 80 | def test_timeout(self): 81 | conn = Connection("client_token") 82 | conn.set_timeout(30) 83 | assert conn.impl.driver.timeout == 30 84 | -------------------------------------------------------------------------------- /test/client/test_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import warnings 5 | 6 | import numpy 7 | import pytest 8 | 9 | from sigopt.lib import * 10 | 11 | 12 | LONG_NUMBER = 100000000000000000000000 13 | 14 | 15 | class TestBase(object): 16 | @pytest.fixture(autouse=True) 17 | def set_warnings(self): 18 | warnings.simplefilter("error") 19 | 20 | def test_is_integer(self): 21 | assert is_integer(int("3")) is True 22 | assert is_integer(0) is True 23 | assert is_integer(LONG_NUMBER) is True 24 | assert is_integer(numpy.int32()) is True 25 | assert is_integer(numpy.int64()) is True 26 | 27 | assert is_integer([]) is False 28 | assert is_integer([2]) is False 29 | assert is_integer({1: 2}) is False 30 | assert is_integer(None) is False 31 | assert is_integer(True) is False 32 | assert is_integer(False) is False 33 | assert is_integer(4.0) is False 34 | assert is_integer("3") is False 35 | assert is_integer(3.14) is False 36 | assert is_integer(numpy.float32()) is False 37 | assert is_integer(numpy.float64()) is False 38 | assert is_integer(numpy.nan) is False 39 | 40 | def test_is_number(self): 41 | assert is_number(int("3")) is True 42 | assert is_number(0) is True 43 | assert is_number(LONG_NUMBER) is True 44 | assert is_number(4.0) is True 45 | assert is_number(3.14) is True 46 | assert is_number(numpy.int32()) is True 47 | assert is_number(numpy.int64()) is True 48 | assert is_number(numpy.float32()) is True 49 | assert is_number(numpy.float64()) is True 50 | 51 | assert is_number([]) is False 52 | assert is_number([2]) is False 53 | assert is_number({1: 2}) is False 54 | assert is_number(None) is False 55 | assert is_number(True) is False 56 | assert is_number(False) is False 57 | assert is_number("3") is False 58 | assert is_number(numpy.nan) is False 59 | 60 | def test_is_numpy_array(self): 61 | assert is_numpy_array(numpy.array([])) 62 | assert is_numpy_array(numpy.array([1, 2, 3])) 63 | 64 | assert not is_numpy_array([]) 65 | assert not is_numpy_array([1, 2, 3]) 66 | assert not is_numpy_array(()) 67 | assert not is_numpy_array((1, 2, 3)) 68 | assert not is_numpy_array(None) 69 | assert not is_numpy_array(False) 70 | assert not is_numpy_array(True) 71 | assert not is_numpy_array(0) 72 | assert not is_numpy_array(1.0) 73 | assert not is_numpy_array("abc") 74 | assert not is_numpy_array("abc") 75 | assert not is_numpy_array(b"abc") 76 | assert not is_numpy_array({}) 77 | assert not is_numpy_array({"a": 123}) 78 | assert not is_numpy_array(set()) 79 | assert not is_numpy_array(set((1, "a"))) 80 | assert not is_numpy_array({1, "a"}) 81 | assert not is_numpy_array(frozenset((1, "a"))) 82 | 83 | def test_is_sequence(self): 84 | assert is_sequence([]) 85 | assert is_sequence([1, 2, 3]) 86 | assert is_sequence(()) 87 | assert is_sequence((1, 2, 3)) 88 | assert is_sequence(numpy.array([])) 89 | assert is_sequence(numpy.array([1, 2, 3])) 90 | 91 | assert not is_sequence(None) 92 | assert not is_sequence(False) 93 | assert not is_sequence(True) 94 | assert not is_sequence(0) 95 | assert not is_sequence(1.0) 96 | assert not is_sequence("abc") 97 | assert not is_sequence("abc") 98 | assert not is_sequence(b"abc") 99 | assert not is_sequence({}) 100 | assert not is_sequence({"a": 123}) 101 | assert not is_sequence(set()) 102 | assert not is_sequence(set((1, "a"))) 103 | assert not is_sequence({1, "a"}) 104 | assert not is_sequence(frozenset((1, "a"))) 105 | 106 | def test_is_mapping(self): 107 | assert is_mapping({}) 108 | assert is_mapping({"a": 123}) 109 | 110 | assert not is_mapping([]) 111 | assert not is_mapping([1, 2, 3]) 112 | assert not is_mapping(()) 113 | assert not is_mapping((1, 2, 3)) 114 | assert not is_mapping(numpy.array([])) 115 | assert not is_mapping(numpy.array([1, 2, 3])) 116 | assert not is_mapping(None) 117 | assert not is_mapping(False) 118 | assert not is_mapping(True) 119 | assert not is_mapping(0) 120 | assert not is_mapping(1.0) 121 | assert not is_mapping("abc") 122 | assert not is_mapping("abc") 123 | assert not is_mapping(b"abc") 124 | assert not is_mapping(set()) 125 | assert not is_mapping(set((1, "a"))) 126 | assert not is_mapping({1, "a"}) 127 | assert not is_mapping(frozenset((1, "a"))) 128 | 129 | def test_is_string(self): 130 | assert is_string("abc") 131 | assert is_string("abc") 132 | 133 | assert not is_string(b"abc") 134 | assert not is_string({}) 135 | assert not is_string({"a": 123}) 136 | assert not is_string([]) 137 | assert not is_string([1, 2, 3]) 138 | assert not is_string(()) 139 | assert not is_string((1, 2, 3)) 140 | assert not is_string(numpy.array([])) 141 | assert not is_string(numpy.array([1, 2, 3])) 142 | assert not is_string(None) 143 | assert not is_string(False) 144 | assert not is_string(True) 145 | assert not is_string(0) 146 | assert not is_string(1.0) 147 | assert not is_string(set()) 148 | assert not is_string(set((1, "a"))) 149 | assert not is_string({1, "a"}) 150 | assert not is_string(frozenset((1, "a"))) 151 | -------------------------------------------------------------------------------- /test/client/test_request_driver.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import mock 5 | import pytest 6 | 7 | from sigopt.request_driver import RequestDriver 8 | from sigopt.version import VERSION 9 | 10 | 11 | class TestRequestDriver: 12 | api_token = "test_api_token" 13 | api_url = "https://test.api.sigopt.ninja" 14 | timeout = -1 15 | user_agent_info = ("test", "info") 16 | 17 | @pytest.fixture(autouse=True) 18 | def patch_config(self): 19 | with mock.patch("sigopt.config.get_user_agent_info") as get_user_agent_info: 20 | get_user_agent_info.side_effect = [self.user_agent_info] 21 | yield 22 | 23 | @pytest.fixture 24 | def mock_session(self): 25 | return mock.Mock() 26 | 27 | @pytest.fixture 28 | def driver(self, mock_session): 29 | return RequestDriver( 30 | self.api_token, 31 | api_url=self.api_url, 32 | session=mock_session, 33 | timeout=self.timeout, 34 | ) 35 | 36 | @pytest.mark.parametrize( 37 | "method,expected_method,uses_params", 38 | [ 39 | ("get", "GET", True), 40 | ("Get", "GET", True), 41 | ("GET", "GET", True), 42 | ("DELETE", "DELETE", True), 43 | ("PUT", "PUT", False), 44 | ("POST", "POST", False), 45 | ("MERGE", "MERGE", False), 46 | ], 47 | ) 48 | @pytest.mark.parametrize( 49 | "path,expected_url", 50 | [ 51 | (["experiments"], "/v1/experiments"), 52 | (["experiments", 1], "/v1/experiments/1"), 53 | (["experiments", "1"], "/v1/experiments/1"), 54 | (["experiments", "1", "suggestions", "2"], "/v1/experiments/1/suggestions/2"), 55 | ], 56 | ) 57 | @pytest.mark.parametrize( 58 | "data,expected_params,expected_json", 59 | [ 60 | (None, {}, None), 61 | ({}, {}, {}), 62 | ({"id": 1}, {"id": "1"}, {"id": 1}), 63 | ({"id": "1"}, {"id": "1"}, {"id": "1"}), 64 | ( 65 | {"assignments": {"x": 1, "y": 2}}, 66 | {"assignments": '{"x":1,"y":2}'}, 67 | {"assignments": {"x": 1, "y": 2}}, 68 | ), 69 | ( 70 | {"datasets": {"mnist": {}}, "source_code": {"hash": "xyz123"}}, 71 | {"datasets": '{"mnist":{}}', "source_code": '{"hash":"xyz123"}'}, 72 | {"datasets": {"mnist": {}}, "source_code": {"hash": "xyz123"}}, 73 | ), 74 | ( 75 | {"after": "", "before": "1234", "limit": 100}, 76 | {"after": "", "before": "1234", "limit": "100"}, 77 | {"after": "", "before": "1234", "limit": 100}, 78 | ), 79 | ], 80 | ) 81 | @pytest.mark.parametrize( 82 | "headers,expected_headers", 83 | [ 84 | (None, {}), 85 | ({}, {}), 86 | ({"X-Response-Content": "skip"}, {"X-Response-Content": "skip"}), 87 | ], 88 | ) 89 | @pytest.mark.parametrize( 90 | "response_code,response_data,returned_data", 91 | [ 92 | (200, "{}", {}), 93 | (204, "", None), 94 | ], 95 | ) 96 | def test_request( 97 | self, 98 | driver, 99 | mock_session, 100 | method, 101 | path, 102 | data, 103 | headers, 104 | uses_params, 105 | response_code, 106 | response_data, 107 | returned_data, 108 | expected_method, 109 | expected_url, 110 | expected_params, 111 | expected_json, 112 | expected_headers, 113 | ): 114 | mock_session.request = mock.Mock( 115 | side_effect=[ 116 | mock.Mock( 117 | status_code=response_code, 118 | text=response_data, 119 | ) 120 | ] 121 | ) 122 | response = driver.request(method, path, data, headers) 123 | assert response == returned_data 124 | if uses_params: 125 | expected_json = None 126 | else: 127 | expected_params = None 128 | expected_headers.update(driver.default_headers) 129 | user_agent_info = "; ".join(self.user_agent_info) 130 | expected_headers.update( 131 | { 132 | "User-Agent": f"sigopt-python/{VERSION} ({user_agent_info})", 133 | } 134 | ) 135 | mock_session.request.assert_called_once_with( 136 | method=expected_method, 137 | url=f"{self.api_url}{expected_url}", 138 | params=expected_params, 139 | json=expected_json, 140 | auth=driver.auth, 141 | headers=expected_headers, 142 | verify=None, 143 | proxies=None, 144 | timeout=self.timeout, 145 | cert=None, 146 | ) 147 | -------------------------------------------------------------------------------- /test/client/test_requestor.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import pytest 5 | 6 | from sigopt.exception import ApiException, ConnectionException 7 | from sigopt.interface import ConnectionImpl 8 | from sigopt.objects import Experiment, Observation, Pagination 9 | 10 | 11 | class MockRequestor(object): 12 | def __init__(self, response): 13 | self.response = response 14 | 15 | def __getattr__(self, name): 16 | def func(*args, **kwargs): 17 | if isinstance(self.response, Exception): 18 | raise self.response 19 | return self.response 20 | 21 | func.__name__ = name 22 | return func 23 | 24 | 25 | MESSAGE = "This is an exception message." 26 | 27 | SAMPLE_EXCEPTION = { 28 | "message": MESSAGE, 29 | } 30 | 31 | SAMPLE_RESPONSE = { 32 | "number": 1.2, 33 | "string": "abc", 34 | "list": [1, 2, 3], 35 | "object": { 36 | "key": "value", 37 | }, 38 | } 39 | 40 | 41 | class TestRequestor(object): 42 | def returns(self, response): 43 | return MockRequestor(response) 44 | 45 | def test_ok(self): 46 | connection = ConnectionImpl(self.returns(SAMPLE_RESPONSE)) 47 | assert connection.experiments(1).fetch() == Experiment(SAMPLE_RESPONSE) 48 | assert connection.experiments().create() == Experiment(SAMPLE_RESPONSE) 49 | assert connection.experiments(1).update() == Experiment(SAMPLE_RESPONSE) 50 | assert connection.experiments(1).delete() is None 51 | assert connection.experiments(1).observations().create_batch() == Pagination(Observation, SAMPLE_RESPONSE) 52 | 53 | def test_ok_code(self): 54 | connection = ConnectionImpl(self.returns(SAMPLE_RESPONSE)) 55 | assert connection.experiments(1).fetch() == Experiment(SAMPLE_RESPONSE) 56 | 57 | def test_response(self): 58 | connection = ConnectionImpl(self.returns(SAMPLE_RESPONSE)) 59 | assert connection.experiments(1).fetch() == Experiment(SAMPLE_RESPONSE) 60 | 61 | def test_no_response(self): 62 | connection = ConnectionImpl(self.returns(None)) 63 | assert connection.experiments(1).fetch() is None 64 | 65 | def test_client_error(self): 66 | connection = ConnectionImpl(self.returns(ApiException(SAMPLE_RESPONSE, 400))) 67 | with pytest.raises(ApiException) as e: 68 | connection.experiments(1).fetch() 69 | e = e.value 70 | assert str(e) == "ApiException (400): " 71 | assert e.status_code == 400 72 | assert e.to_json() == SAMPLE_RESPONSE 73 | 74 | def test_client_error_message(self): 75 | connection = ConnectionImpl(self.returns(ApiException(SAMPLE_EXCEPTION, 400))) 76 | with pytest.raises(ApiException) as e: 77 | connection.experiments(1).fetch() 78 | e = e.value 79 | assert str(e) == "ApiException (400): " + MESSAGE 80 | assert e.status_code == 400 81 | assert e.to_json() == SAMPLE_EXCEPTION 82 | 83 | def test_server_error(self): 84 | connection = ConnectionImpl(self.returns(ApiException(SAMPLE_EXCEPTION, status_code=500))) 85 | with pytest.raises(ApiException) as e: 86 | connection.experiments(1).fetch() 87 | e = e.value 88 | assert str(e) == "ApiException (500): " + MESSAGE 89 | assert e.status_code == 500 90 | assert e.to_json() == SAMPLE_EXCEPTION 91 | 92 | def test_connection_error(self): 93 | connection = ConnectionImpl(self.returns(ConnectionException("fake connection exception"))) 94 | with pytest.raises(ConnectionException) as e: 95 | connection.experiments(1).fetch() 96 | e = e.value 97 | assert str(e) == "ConnectionException: fake connection exception" 98 | -------------------------------------------------------------------------------- /test/client/test_resource.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import pytest 5 | 6 | from sigopt.endpoint import BoundApiEndpoint 7 | from sigopt.interface import ConnectionImpl 8 | from sigopt.resource import BoundApiResource, PartiallyBoundApiResource 9 | 10 | 11 | class TestResource(object): 12 | @pytest.fixture 13 | def connection(self): 14 | return ConnectionImpl(driver=None) 15 | 16 | def test_partial_bind_resource(self, connection): 17 | assert isinstance(connection.experiments().observations, PartiallyBoundApiResource) 18 | assert isinstance(connection.experiments(1).observations, PartiallyBoundApiResource) 19 | assert isinstance(connection.experiments().observations, PartiallyBoundApiResource) 20 | assert isinstance(connection.experiments(1).observations, PartiallyBoundApiResource) 21 | 22 | def test_bind_resource(self, connection): 23 | api_resource = connection.experiments 24 | assert isinstance(api_resource(), BoundApiResource) 25 | assert isinstance(api_resource(1), BoundApiResource) 26 | 27 | partially_bound_api_resource = connection.experiments().observations 28 | assert isinstance(partially_bound_api_resource(), BoundApiResource) 29 | assert isinstance(partially_bound_api_resource(1), BoundApiResource) 30 | 31 | def test_bind_endpoint(self, connection): 32 | assert isinstance(connection.experiments().fetch, BoundApiEndpoint) 33 | assert isinstance(connection.experiments(1).fetch, BoundApiEndpoint) 34 | 35 | def test_get_bound_entity(self, connection): 36 | assert isinstance(connection.experiments().get_bound_entity("fetch"), BoundApiEndpoint) 37 | assert isinstance( 38 | connection.experiments().get_bound_entity("observations"), 39 | PartiallyBoundApiResource, 40 | ) 41 | -------------------------------------------------------------------------------- /test/client/test_urllib3_patch.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import pytest 5 | 6 | from sigopt.urllib3_patch import ExpiringHTTPConnectionPool, ExpiringHTTPSConnectionPool 7 | 8 | 9 | @pytest.mark.parametrize("pool_cls", [ExpiringHTTPConnectionPool, ExpiringHTTPSConnectionPool]) 10 | def test_pool_reuses_connections(pool_cls): 11 | pool = pool_cls(host="sigopt.com", expiration_seconds=30) 12 | conn1 = pool._get_conn() 13 | pool._put_conn(conn1) 14 | conn2 = pool._get_conn() 15 | assert conn1 is conn2 16 | 17 | 18 | @pytest.mark.parametrize("pool_cls", [ExpiringHTTPConnectionPool, ExpiringHTTPSConnectionPool]) 19 | def test_pool_expires_connections(pool_cls): 20 | pool = pool_cls(host="sigopt.com", expiration_seconds=0) 21 | conn1 = pool._get_conn() 22 | pool._put_conn(conn1) 23 | conn2 = pool._get_conn() 24 | assert conn1 is not conn2 25 | -------------------------------------------------------------------------------- /test/runs/test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import base64 5 | import os 6 | 7 | import mock 8 | 9 | from sigopt.config import Config 10 | 11 | 12 | fake_context = base64.b64encode(b'{"a": "b"}').decode("utf-8") 13 | 14 | 15 | class FakeConfigContext(object): 16 | def __init__(self, key): 17 | self.CONFIG_CONTEXT_KEY = key 18 | 19 | 20 | class TestConfig(object): 21 | def test_load_json_config(self): 22 | with mock.patch.dict(os.environ, {"SIGOPT_CONTEXT": fake_context}): 23 | config = Config() 24 | 25 | assert config.get_context_data(FakeConfigContext("a")) == "b" 26 | assert config.get_context_data(FakeConfigContext("none")) is None 27 | -------------------------------------------------------------------------------- /test/runs/test_defaults.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import pytest 5 | 6 | from sigopt.defaults import normalize_project_id 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "project_id,expected", 11 | [ 12 | ("simple", "simple"), 13 | ("CAPS", "caps"), 14 | ("CamelCase", "camelcase"), 15 | ("snake_case", "snake_case"), 16 | ("numbers0123", "numbers0123"), 17 | ("h-yphen", "h-yphen"), 18 | ("that's`~!@#$%^&*()+[]{};:'\",<>/?illegal!", "thatsillegal"), 19 | ], 20 | ) 21 | def test_normalize_project_id(project_id, expected): 22 | normalized = normalize_project_id(project_id) 23 | assert normalized == expected 24 | -------------------------------------------------------------------------------- /test/runs/test_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import warnings 5 | 6 | import mock 7 | import pytest 8 | 9 | from sigopt.factory import SigOptFactory 10 | 11 | 12 | warnings.simplefilter("always") 13 | 14 | 15 | class TestSigOptFactory(object): 16 | @pytest.fixture 17 | def api_connection(self): 18 | conn = mock.Mock() 19 | conn.clients().projects().training_runs().create.return_value = mock.Mock( 20 | assignments={"fixed1": 0, "fixed2": "test"} 21 | ) 22 | return conn 23 | 24 | @pytest.fixture(autouse=True) 25 | def patched_connection(self, api_connection): 26 | with mock.patch("sigopt.factory.get_connection") as connection_singleton: 27 | connection_singleton.return_value = api_connection 28 | yield 29 | 30 | @pytest.fixture 31 | def factory(self): 32 | factory = SigOptFactory("test-project") 33 | return factory 34 | 35 | def test_create_run(self, factory): 36 | run_context = factory.create_run() 37 | assert run_context is not None 38 | 39 | def test_create_context_with_name(self, factory, api_connection): 40 | run_context = factory.create_run(name="test context") 41 | assert run_context is not None 42 | api_connection.clients().projects().training_runs().create.assert_called_once() 43 | assert api_connection.clients().projects().training_runs().create.call_args[1]["name"] == "test context" 44 | 45 | def test_local_run_context_methods(self, factory): 46 | with factory.create_run(name="test-run") as local_run: 47 | local_run._update_run = mock.Mock() 48 | local_run.params.p1 = 1 49 | local_run.params.setdefault("p2", 2) 50 | local_run.params.update({"p3": 3, "p4": 4}) 51 | local_run.params.setdefault("p3", 0) 52 | local_run.params.pop("p2") 53 | local_run.log_metric("metric", 1, 0.1) 54 | local_run._update_run.assert_has_calls( 55 | [ 56 | mock.call({"assignments": {"p1": 1}}), 57 | mock.call({"assignments": {"p2": 2}}), 58 | mock.call({"assignments": {"p3": 3, "p4": 4}}), 59 | mock.call({"assignments": {"p2": None}}), 60 | mock.call({"values": {"metric": {"value": 1, "value_stddev": 0.1}}}), 61 | mock.call({"state": "completed"}), 62 | ] 63 | ) 64 | 65 | def test_local_run_context_exception(self, factory): 66 | class TestException(Exception): 67 | pass 68 | 69 | with pytest.raises(TestException): 70 | with factory.create_run(name="test-run") as local_run: 71 | local_run._update_run = mock.Mock() 72 | raise TestException() 73 | local_run._update_run.assert_has_calls( 74 | [ 75 | mock.call({"state": "failed"}), 76 | ] 77 | ) 78 | -------------------------------------------------------------------------------- /test/runs/test_files/test.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-python/855b9a7a590162627ea2ef934f793a567a31b8be/test/runs/test_files/test.bmp -------------------------------------------------------------------------------- /test/runs/test_files/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-python/855b9a7a590162627ea2ef934f793a567a31b8be/test/runs/test_files/test.png -------------------------------------------------------------------------------- /test/runs/test_local_run_context.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import pytest 5 | 6 | from sigopt.local_run_context import LocalRunContext 7 | 8 | 9 | class TestLocalRunContext(object): 10 | @pytest.fixture 11 | def context(self): 12 | return LocalRunContext() 13 | 14 | @pytest.fixture 15 | def params(self): 16 | return {"x": 1.0, "y": 2.0} 17 | 18 | @pytest.fixture 19 | def metrics(self): 20 | return { 21 | "v0": 1, 22 | "v1": 2.0, 23 | } 24 | 25 | def test_init(self): 26 | name = "test0" 27 | metadata = {"m0": 1, "m2": 2.0} 28 | context = LocalRunContext(name=name, metadata=metadata) 29 | run = context.get() 30 | assert run["name"] == name 31 | assert run["metadata"] == metadata 32 | 33 | @pytest.mark.parametrize("state", ["completed", "failed"]) 34 | def test_log_state(self, context, state): 35 | context.log_state(state) 36 | run = context.get() 37 | assert run["state"] == state 38 | 39 | def test_log_failure(self, context): 40 | context.log_failure() 41 | run = context.get() 42 | assert run["state"] == "failed" 43 | 44 | def test_log_metrics(self, context, metrics): 45 | context.log_metrics(metrics) 46 | run = context.get() 47 | assert run["values"] == {k: {"value": v} for k, v in metrics.items()} 48 | 49 | @pytest.mark.parametrize( 50 | "source, source_sort, source_default_show", 51 | [ 52 | ("s0", 10, True), 53 | ("s0", 20, False), 54 | ("s0", None, None), 55 | (None, 20, False), 56 | ], 57 | ) 58 | def test_log_parameters(self, context, params, source, source_sort, source_default_show): 59 | if source_sort is not None: 60 | source_meta = {"sort": source_sort, "default_show": source_default_show} 61 | else: 62 | source_meta = None 63 | context.log_parameters(params, source, source_meta) 64 | run = context.get() 65 | assert run["assignments"] == params 66 | if source is not None: 67 | assert run["assignments_meta"] == {p: {"source": source} for p in params} 68 | if source_sort is not None: 69 | assert run["assignments_sources"][source] == { 70 | "sort": source_sort, 71 | "default_show": source_default_show, 72 | } 73 | else: 74 | assert "assignments_sources" not in run 75 | else: 76 | assert "assignments_meta" not in run and "assignments_sources" not in run 77 | -------------------------------------------------------------------------------- /test/runs/test_set_project.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import warnings 5 | 6 | import mock 7 | import pytest 8 | 9 | from sigopt import _global_factory, set_project 10 | 11 | 12 | def test_set_project_with_global_run(): 13 | with mock.patch("sigopt.get_run_id", mock.Mock(return_value="1")): 14 | with pytest.warns(UserWarning): 15 | set_project("test-123") 16 | assert _global_factory.project == "test-123" 17 | 18 | 19 | def test_set_project_without_run(): 20 | with mock.patch("sigopt.get_run_id", mock.Mock(return_value=None)): 21 | with warnings.catch_warnings(): 22 | warnings.simplefilter("error") 23 | set_project("test-123") 24 | assert _global_factory.project == "test-123" 25 | -------------------------------------------------------------------------------- /test/runs/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import matplotlib 5 | import pytest 6 | 7 | 8 | matplotlib.use("Agg") 9 | 10 | import io 11 | import os 12 | import xml.etree.ElementTree as ET 13 | 14 | import numpy 15 | from matplotlib import pyplot as plt 16 | from PIL import Image 17 | from utils import ObserveWarnings 18 | 19 | from sigopt.file_utils import ( 20 | create_api_image_payload, 21 | get_blob_properties, 22 | try_load_matplotlib_image, 23 | try_load_numpy_image, 24 | try_load_pil_image, 25 | ) 26 | 27 | 28 | @pytest.fixture 29 | def pil_image(): 30 | return Image.new("RGB", (16, 16), (255, 0, 0)) 31 | 32 | 33 | @pytest.fixture 34 | def matplotlib_figure(): 35 | figure = plt.figure() 36 | plt.plot([1, 2, 3, 4]) 37 | plt.ylabel("for testing") 38 | return figure 39 | 40 | 41 | def test_load_pil_image(pil_image): 42 | data = try_load_pil_image(pil_image) 43 | 44 | assert data is not None 45 | 46 | filename, image_data, content_type = data 47 | assert filename is None 48 | 49 | with Image.open(image_data) as loaded_image: 50 | assert numpy.all(numpy.array(loaded_image) == numpy.array(pil_image)) 51 | 52 | assert content_type == "image/png" 53 | 54 | 55 | def test_load_matplotlib_image(matplotlib_figure): 56 | data = try_load_matplotlib_image(matplotlib_figure) 57 | 58 | assert data is not None 59 | 60 | filename, image_data, content_type = data 61 | assert filename is None 62 | 63 | image_data.seek(0) 64 | contents = image_data.read() 65 | assert b"for testing" in contents 66 | 67 | # check that the svg is at least valid xml 68 | ET.fromstring(contents) 69 | 70 | assert content_type == "image/svg+xml" 71 | 72 | 73 | def test_load_HxW_numpy_image(): 74 | numpy_img = numpy.random.randint(0, 255, (32, 16)) 75 | data = try_load_numpy_image(numpy_img) 76 | 77 | assert data is not None 78 | 79 | filename, image_data, content_type = data 80 | assert filename is None 81 | 82 | image_data.seek(0) 83 | with Image.open(image_data) as pil_image: 84 | loaded_numpy_img = numpy.array(pil_image) 85 | 86 | assert loaded_numpy_img.shape[:2] == (32, 16) 87 | 88 | assert numpy.all(numpy_img == loaded_numpy_img) 89 | 90 | assert content_type == "image/png" 91 | 92 | 93 | @pytest.mark.parametrize("N", [1, 3, 4]) 94 | def test_load_HxWxN_numpy_image(N): 95 | numpy_img = numpy.random.randint(0, 255, (32, 16, N)) 96 | data = try_load_numpy_image(numpy_img) 97 | 98 | assert data is not None 99 | 100 | filename, image_data, content_type = data 101 | assert filename is None 102 | 103 | image_data.seek(0) 104 | with Image.open(image_data) as pil_image: 105 | loaded_numpy_img = numpy.array(pil_image) 106 | 107 | assert loaded_numpy_img.shape[:2] == (32, 16) 108 | loaded_numpy_img = loaded_numpy_img.reshape((32, 16, N)) 109 | 110 | assert numpy.all(numpy_img == loaded_numpy_img) 111 | 112 | assert content_type == "image/png" 113 | 114 | 115 | def test_load_numpy_image_clipping(): 116 | numpy_img = numpy.ones((32, 16)) * 512 117 | numpy_img[:16] = -256 118 | data = try_load_numpy_image(numpy_img) 119 | 120 | assert data is not None 121 | 122 | filename, image_data, content_type = data 123 | assert filename is None 124 | 125 | image_data.seek(0) 126 | with Image.open(image_data) as pil_image: 127 | loaded_numpy_img = numpy.array(pil_image) 128 | 129 | assert loaded_numpy_img.shape[:2] == (32, 16) 130 | 131 | assert numpy.all(loaded_numpy_img[:16] == 0) 132 | assert numpy.all(loaded_numpy_img[16:] == 255) 133 | 134 | assert content_type == "image/png" 135 | 136 | 137 | @pytest.mark.parametrize( 138 | "image_path,expected_type", 139 | [ 140 | ("test.png", "image/png"), 141 | ("test.svg", "image/svg+xml"), 142 | ("test.bmp", "image/bmp"), 143 | ], 144 | ) 145 | def test_create_api_image_payload_string_path(image_path, expected_type): 146 | image_path = os.path.join("./test/runs/test_files", image_path) 147 | data = create_api_image_payload(image_path) 148 | assert data is not None 149 | filepath, image_data, content_type = data 150 | with image_data: 151 | image_data.seek(0) 152 | with open(image_path, "rb") as fp: 153 | original_contents = fp.read() 154 | assert image_data.read() == original_contents 155 | assert filepath == image_path 156 | assert content_type == expected_type 157 | 158 | 159 | def test_create_api_image_payload_string_path_bad_type(): 160 | with ObserveWarnings() as w: 161 | path = "./test/runs/test_files/test.txt" 162 | data = create_api_image_payload(path) 163 | assert data is None 164 | assert len(w) == 1 165 | assert issubclass(w[-1].category, RuntimeWarning) 166 | 167 | 168 | def test_create_api_image_payload_pil_image(pil_image): 169 | data = create_api_image_payload(pil_image) 170 | assert data is not None 171 | filepath, _, content_type = data 172 | assert filepath is None 173 | assert content_type == "image/png" 174 | 175 | 176 | def test_create_api_image_payload_matplotlib_figure(matplotlib_figure): 177 | data = create_api_image_payload(matplotlib_figure) 178 | assert data is not None 179 | filepath, _, content_type = data 180 | assert filepath is None 181 | assert content_type == "image/svg+xml" 182 | 183 | 184 | def test_create_api_image_payload_numpy_image(): 185 | numpy_image = numpy.random.randint(0, 255, (16, 16)) 186 | data = create_api_image_payload(numpy_image) 187 | assert data is not None 188 | filepath, _, content_type = data 189 | assert filepath is None 190 | assert content_type == "image/png" 191 | 192 | 193 | def test_create_api_image_payload_unsupported_type(): 194 | with ObserveWarnings() as w: 195 | data = create_api_image_payload({}) 196 | assert data is None 197 | assert len(w) >= 1 198 | assert issubclass(w[-1].category, RuntimeWarning) 199 | 200 | 201 | def test_get_blob_properties(): 202 | data = "some\nblob\ndata\n".encode() 203 | blob = io.BytesIO(data) 204 | expected_b64_md5 = "hlXKMpBfPY7uZV7oFfHr2w==" 205 | length, b64_md5 = get_blob_properties(blob) 206 | assert length == len(data) 207 | assert b64_md5 == expected_b64_md5 208 | -------------------------------------------------------------------------------- /test/runs/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import warnings 5 | from contextlib import contextmanager 6 | 7 | 8 | @contextmanager 9 | def ObserveWarnings(): 10 | with warnings.catch_warnings(record=True) as e: 11 | warnings.simplefilter("always") 12 | yield e 13 | warnings.simplefilter("error") 14 | -------------------------------------------------------------------------------- /test/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import warnings 5 | from contextlib import contextmanager 6 | 7 | 8 | @contextmanager 9 | def ObserveWarnings(): 10 | with warnings.catch_warnings(record=True) as e: 11 | warnings.simplefilter("always") 12 | yield e 13 | warnings.simplefilter("error") 14 | -------------------------------------------------------------------------------- /test/validate/test_validate_experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import copy 5 | 6 | import pytest 7 | 8 | from sigopt.validate.aiexperiment_input import validate_aiexperiment_input 9 | from sigopt.validate.exceptions import ValidationError 10 | 11 | 12 | VALID_EXPERIMENT_INPUT = { 13 | "name": "test experiment", 14 | "parameters": [ 15 | { 16 | "name": "p1", 17 | "type": "int", 18 | "bounds": { 19 | "min": 0, 20 | "max": 1, 21 | }, 22 | }, 23 | ], 24 | "metrics": [{"name": "m1"}], 25 | "budget": 10, 26 | "parallel_bandwidth": 4, 27 | } 28 | 29 | 30 | class TestValidateExperiment: 31 | @pytest.mark.parametrize( 32 | "mutator,expected_message", 33 | [ 34 | (lambda e: e.__delitem__("name"), "name is required"), 35 | (lambda e: e.__setitem__("name", None), "name must be a string"), 36 | (lambda e: e.__setitem__("name", ""), "name cannot be an empty string"), 37 | (lambda e: e.__setitem__("name", 1), "name must be a string"), 38 | (lambda e: e.__setitem__("name", {}), "name must be a string"), 39 | (lambda e: e.__delitem__("parameters"), "parameters is required"), 40 | ( 41 | lambda e: e.__setitem__("parameters", None), 42 | "parameters must be a non-empty list", 43 | ), 44 | ( 45 | lambda e: e.__setitem__("parameters", {}), 46 | "parameters must be a non-empty list", 47 | ), 48 | ( 49 | lambda e: e.__setitem__("parameters", []), 50 | "parameters must be a non-empty list", 51 | ), 52 | (lambda e: e["parameters"].__setitem__(0, []), "parameters must be a mapping"), 53 | (lambda e: e["parameters"][0].__delitem__("name"), "parameters require a name"), 54 | ( 55 | lambda e: e["parameters"][0].__setitem__("name", None), 56 | "parameter name must be a string", 57 | ), 58 | ( 59 | lambda e: e["parameters"][0].__setitem__("name", ""), 60 | "parameter name cannot be an empty string", 61 | ), 62 | (lambda e: e["parameters"][0].__delitem__("type"), "parameters require a type"), 63 | ( 64 | lambda e: e["parameters"][0].__setitem__("type", None), 65 | "parameter type must be a string", 66 | ), 67 | ( 68 | lambda e: e["parameters"][0].__setitem__("type", {}), 69 | "parameter type must be a string", 70 | ), 71 | ( 72 | lambda e: e["parameters"][0].__setitem__("type", ""), 73 | "parameter type cannot be an empty string", 74 | ), 75 | (lambda e: e.__delitem__("metrics"), "metrics is required"), 76 | (lambda e: e.__setitem__("metrics", None), "metrics must be a non-empty list"), 77 | (lambda e: e.__setitem__("metrics", {}), "metrics must be a non-empty list"), 78 | (lambda e: e.__setitem__("metrics", []), "metrics must be a non-empty list"), 79 | (lambda e: e["metrics"].__setitem__(0, []), "metrics must be a mapping"), 80 | (lambda e: e["metrics"][0].__delitem__("name"), "metrics require a name"), 81 | ( 82 | lambda e: e["metrics"][0].__setitem__("name", None), 83 | "metric name must be a string", 84 | ), 85 | ( 86 | lambda e: e["metrics"][0].__setitem__("name", ""), 87 | "metric name cannot be an empty string", 88 | ), 89 | (lambda e: e.__setitem__("budget", []), "budget must be a non-negative number"), 90 | (lambda e: e.__setitem__("budget", -1), "budget must be a non-negative number"), 91 | (lambda e: e.__setitem__("budget", float("inf")), "budget cannot be infinity"), 92 | ( 93 | lambda e: e.__setitem__("parallel_bandwidth", []), 94 | "parallel_bandwidth must be a positive integer", 95 | ), 96 | ( 97 | lambda e: e.__setitem__("parallel_bandwidth", -1), 98 | "parallel_bandwidth must be a positive integer", 99 | ), 100 | ( 101 | lambda e: e.__setitem__("parallel_bandwidth", 0), 102 | "parallel_bandwidth must be a positive integer", 103 | ), 104 | ( 105 | lambda e: e.__setitem__("parallel_bandwidth", 0.5), 106 | "parallel_bandwidth must be a positive integer", 107 | ), 108 | ], 109 | ) 110 | def test_invalid_experiment(self, mutator, expected_message): 111 | experiment_input = copy.deepcopy(VALID_EXPERIMENT_INPUT) 112 | mutator(experiment_input) 113 | with pytest.raises(ValidationError) as validation_error: 114 | validate_aiexperiment_input(experiment_input) 115 | assert expected_message in str(validation_error) 116 | 117 | @pytest.mark.parametrize( 118 | "mutator,check", 119 | [ 120 | (lambda e: e, lambda e: e["name"] == "test experiment"), 121 | ( 122 | lambda e: e, 123 | lambda e: e["parameters"] == [{"name": "p1", "type": "int", "bounds": {"min": 0, "max": 1}}], 124 | ), 125 | (lambda e: e, lambda e: e["metrics"] == [{"name": "m1"}]), 126 | (lambda e: e, lambda e: e["parallel_bandwidth"] == 4), 127 | # support new features without needing to write new validation 128 | ( 129 | lambda e: e.__setitem__("unrecognized_key", []), 130 | lambda e: e["unrecognized_key"] == [], 131 | ), 132 | ( 133 | lambda e: e["parameters"][0].__setitem__("unrecognized_key", []), 134 | lambda e: e["parameters"][0]["unrecognized_key"] == [], 135 | ), 136 | ( 137 | lambda e: e["metrics"][0].__setitem__("unrecognized_key", []), 138 | lambda e: e["metrics"][0]["unrecognized_key"] == [], 139 | ), 140 | ], 141 | ) 142 | def test_valid_experiment(self, mutator, check): 143 | experiment_input = copy.deepcopy(VALID_EXPERIMENT_INPUT) 144 | mutator(experiment_input) 145 | validated = validate_aiexperiment_input(experiment_input) 146 | assert check(validated) 147 | -------------------------------------------------------------------------------- /test/validate/test_validate_run.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import copy 5 | 6 | import pytest 7 | 8 | from sigopt.validate.exceptions import ValidationError 9 | from sigopt.validate.run_input import validate_run_input 10 | 11 | 12 | VALID_RUN_INPUT = { 13 | "name": "test run", 14 | "run": "python test.py", 15 | "resources": {"gpus": 2}, 16 | } 17 | 18 | 19 | class TestValidateRun: 20 | @pytest.mark.parametrize( 21 | "mutator,expected_message", 22 | [ 23 | (lambda r: r.__setitem__("name", ""), "name cannot be an empty string"), 24 | (lambda r: r.__setitem__("name", 1), "name must be a string"), 25 | (lambda r: r.__setitem__("name", {}), "name must be a string"), 26 | (lambda r: r.__setitem__("run", {}), "must be a command"), 27 | (lambda r: r.__setitem__("run", [1, 2, 3]), "has some non-string arguments"), 28 | (lambda r: r.__setitem__("resources", []), "must be a mapping"), 29 | (lambda r: r.__setitem__("resources", {1: 2}), "can only have string keys"), 30 | ], 31 | ) 32 | def test_invalid_run(self, mutator, expected_message): 33 | run_input = copy.deepcopy(VALID_RUN_INPUT) 34 | mutator(run_input) 35 | with pytest.raises(ValidationError) as validation_error: 36 | validate_run_input(run_input) 37 | assert expected_message in str(validation_error) 38 | 39 | @pytest.mark.parametrize( 40 | "mutator,check", 41 | [ 42 | (lambda r: r, lambda r: r["name"] == "test run"), 43 | (lambda r: r, lambda r: r["run"] == ["sh", "-c", "python test.py"]), 44 | ( 45 | lambda r: r.__setitem__("run", ["python", "test.py"]), 46 | lambda r: r["run"] == ["python", "test.py"], 47 | ), 48 | (lambda r: r.__delitem__("run"), lambda r: r["run"] == []), 49 | (lambda r: r.__setitem__("run", None), lambda r: r["run"] == []), 50 | (lambda r: r, lambda r: r["resources"] == {"gpus": 2}), 51 | (lambda r: r.__delitem__("resources"), lambda r: r["resources"] == {}), 52 | ], 53 | ) 54 | def test_valid_run(self, mutator, check): 55 | run_input = copy.deepcopy(VALID_RUN_INPUT) 56 | mutator(run_input) 57 | validated = validate_run_input(run_input) 58 | assert check(validated) 59 | -------------------------------------------------------------------------------- /test/xgboost/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-python/855b9a7a590162627ea2ef934f793a567a31b8be/test/xgboost/__init__.py -------------------------------------------------------------------------------- /test/xgboost/test_compute_metric.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import numpy 5 | from sklearn.metrics import accuracy_score, classification_report, mean_absolute_error, mean_squared_error 6 | 7 | from sigopt.xgboost.compute_metrics import ( 8 | compute_accuracy, 9 | compute_classification_report, 10 | compute_mae, 11 | compute_mse, 12 | compute_positives_and_negatives, 13 | ) 14 | 15 | 16 | def verify_classification_metrics_against_sklearn(y_true, y_pred): 17 | report_compute = compute_classification_report(y_true, y_pred) 18 | report_sklearn = classification_report(y_true, y_pred, output_dict=True, zero_division=0) 19 | assert numpy.isclose( 20 | report_compute["weighted avg"]["precision"], 21 | report_sklearn["weighted avg"]["precision"], 22 | ) 23 | assert numpy.isclose(report_compute["weighted avg"]["recall"], report_sklearn["weighted avg"]["recall"]) 24 | assert numpy.isclose( 25 | report_compute["weighted avg"]["f1-score"], 26 | report_sklearn["weighted avg"]["f1-score"], 27 | ) 28 | assert numpy.abs(compute_accuracy(y_true, y_pred) - accuracy_score(y_true, y_pred)) < 1e-8 29 | classes = numpy.unique(y_true) 30 | for c in classes: 31 | label = str(c) 32 | assert numpy.isclose(report_compute[label]["precision"], report_sklearn[label]["precision"]) 33 | assert numpy.isclose(report_compute[label]["recall"], report_sklearn[label]["recall"]) 34 | assert numpy.isclose(report_compute[label]["f1-score"], report_sklearn[label]["f1-score"]) 35 | 36 | 37 | class TestComputeMetrics(object): 38 | def test_compute_positives_and_negatives(self): 39 | y_true = numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) 40 | y_pred = numpy.array([1, 0, 0, 0, 0, 0, 0, 1, 0, 0]) 41 | tp, tn, fp, fn = compute_positives_and_negatives(y_true, y_pred, 1) 42 | assert sum([tp, tn, fp, fn]) == len(y_true) 43 | assert tp == 1 44 | assert tn == 6 45 | assert fp == 1 46 | assert fn == 2 47 | 48 | y_true = numpy.array([1, 1, 1, 2, 2, 2, 0, 0, 0, 0]) 49 | y_pred = numpy.array([1, 0, 0, 0, 0, 0, 1, 1, 0, 0]) 50 | tp, tn, fp, fn = compute_positives_and_negatives(y_true, y_pred, 2) 51 | assert sum([tp, tn, fp, fn]) == len(y_true) 52 | assert tp == 0 53 | assert tn == 7 54 | assert fp == 0 55 | assert fn == 3 56 | 57 | def test_binary_classification_one_true_label(self): 58 | y_true = numpy.zeros(10, dtype=int) 59 | y_pred = numpy.ones(10, dtype=int) 60 | assert compute_accuracy(y_true, y_pred) == 0 61 | report = compute_classification_report(y_true, y_pred) 62 | assert "1" not in report.keys() 63 | assert report["0"]["precision"] == 0 64 | assert report["0"]["recall"] == 0 65 | assert report["0"]["f1-score"] == 0 66 | assert report["weighted avg"]["precision"] == 0 67 | assert report["weighted avg"]["recall"] == 0 68 | assert report["weighted avg"]["f1-score"] == 0 69 | 70 | def test_binary_classification_one_pred_label(self): 71 | y_true = numpy.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0]) 72 | y_pred = numpy.ones(10, dtype=int) 73 | assert compute_accuracy(y_true, y_pred) == 0.3 74 | report = compute_classification_report(y_true, y_pred) 75 | assert report["0"]["precision"] == 0 76 | assert report["0"]["recall"] == 0 77 | assert report["0"]["f1-score"] == 0 78 | assert report["1"]["precision"] == 0.3 79 | assert report["1"]["recall"] == 1.0 80 | assert numpy.isclose(report["1"]["f1-score"], 2 * (1.0 * 0.3) / (1.0 + 0.3)) 81 | assert report["weighted avg"]["precision"] == 0.3 * 0.3 82 | assert report["weighted avg"]["recall"] == 0.3 83 | assert numpy.isclose(report["weighted avg"]["f1-score"], 2 * (1.0 * 0.3) / (1.0 + 0.3) * 0.3) 84 | 85 | def test_binary_classification_against_sklearn(self): 86 | n_samples = 30 87 | y_true = numpy.random.randint(2, size=n_samples) 88 | y_pred = numpy.random.randint(2, size=n_samples) 89 | verify_classification_metrics_against_sklearn(y_true, y_pred) 90 | 91 | def test_multiclass_classification_metrics(self): 92 | n_samples = 50 93 | y_true = numpy.random.randint(3, size=n_samples) 94 | y_pred = numpy.random.randint(2, size=n_samples) 95 | verify_classification_metrics_against_sklearn(y_true, y_pred) 96 | 97 | y_pred = numpy.random.randint(3, size=n_samples) 98 | verify_classification_metrics_against_sklearn(y_true, y_pred) 99 | 100 | def test_regression_metrics(self): 101 | # Check regression metrics 102 | n_samples = 10 103 | y_true = 2 * numpy.ones(n_samples) 104 | assert numpy.isclose(compute_mae(y_true, numpy.zeros(n_samples)), 2.0) 105 | assert numpy.isclose(compute_mse(y_true, numpy.zeros(n_samples)), 4.0) 106 | 107 | y_true = numpy.random.randn(n_samples) 108 | assert numpy.isclose(compute_mae(y_true, y_true), 0) 109 | assert numpy.isclose(compute_mse(y_true, y_true), 0) 110 | 111 | y_pred = numpy.random.randn(n_samples) 112 | # pylint: disable=arguments-out-of-order 113 | assert numpy.isclose(compute_mae(y_true, y_pred), compute_mae(y_pred, y_true)) 114 | assert numpy.isclose(compute_mse(y_true, y_pred), compute_mse(y_pred, y_true)) 115 | # pylint: enable=arguments-out-of-order 116 | assert numpy.isclose(compute_mae(y_true, y_pred), numpy.sum(numpy.abs(y_true - y_pred)) / n_samples) 117 | assert numpy.isclose(compute_mse(y_true, y_pred), numpy.sum((y_true - y_pred) ** 2) / n_samples) 118 | 119 | assert numpy.isclose(compute_mae(y_true, y_pred), mean_absolute_error(y_true, y_pred)) 120 | assert numpy.isclose(compute_mse(y_true, y_pred), mean_squared_error(y_true, y_pred)) 121 | -------------------------------------------------------------------------------- /test/xgboost/test_run_options_parsing.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import random 5 | 6 | import pytest 7 | from mock import Mock 8 | 9 | from sigopt.objects import TrainingRun 10 | from sigopt.run_context import RunContext 11 | from sigopt.xgboost.run import DEFAULT_RUN_OPTIONS, XGBRunHandler, parse_run_options 12 | 13 | from ..utils import ObserveWarnings 14 | 15 | 16 | class TestXGBoostKwargs(object): 17 | def test_xgboost_kwargs_remove_wrong_key(self): 18 | kwargs = { 19 | "WRONG_KEY_1": True, 20 | "WRONG_KEY_2": 3.14, 21 | } 22 | with ObserveWarnings() as ws: 23 | xgb_run_handler = XGBRunHandler( 24 | params={"max_depth": 2}, 25 | dtrain=Mock(), 26 | num_boost_round=21, 27 | evals=None, 28 | early_stopping_rounds=10, 29 | evals_result=None, 30 | verbose_eval=True, 31 | xgb_model=Mock(), 32 | callbacks=None, 33 | run_options=None, 34 | **kwargs 35 | ) 36 | assert not xgb_run_handler.kwargs 37 | assert len(ws) == len(kwargs) 38 | for w in ws: # pylint: disable=not-an-iterable 39 | assert issubclass(w.category, RuntimeWarning) 40 | 41 | def test_xgboost_kwargs_keep_right_key(self): 42 | xgb_run_handler = XGBRunHandler( 43 | params={"max_depth": 2}, 44 | dtrain=Mock(), 45 | num_boost_round=21, 46 | evals=None, 47 | early_stopping_rounds=None, 48 | evals_result=None, 49 | verbose_eval=True, 50 | xgb_model=None, 51 | callbacks=None, 52 | maximize=True, 53 | run_options={"autolog_metrics": True}, 54 | ) 55 | assert len(xgb_run_handler.kwargs) == 1 56 | assert "maximize" in xgb_run_handler.kwargs 57 | assert xgb_run_handler.kwargs["maximize"] is True 58 | 59 | 60 | class TestRunOptionsParsing(object): 61 | def test_run_options_wrong_type(self): 62 | run_options = Mock(log_params=True) 63 | with pytest.raises(TypeError): 64 | parse_run_options(run_options) 65 | 66 | def test_run_options_wrong_keys(self): 67 | run_options = { 68 | "autolog_metric": True, 69 | } 70 | with pytest.raises(ValueError): 71 | parse_run_options(run_options) 72 | 73 | def test_run_options_autolog_not_bool(self): 74 | run_options = { 75 | "autolog_metrics": 12, 76 | } 77 | with pytest.raises(TypeError): 78 | parse_run_options(run_options) 79 | 80 | def test_run_options_run_and_name_keys(self): 81 | run_options = { 82 | "name": "test-run", 83 | "run": Mock(), 84 | } 85 | with pytest.raises(ValueError): 86 | parse_run_options(run_options) 87 | 88 | run_options = { 89 | "name": None, 90 | "run": None, 91 | } 92 | assert parse_run_options(run_options) 93 | 94 | run_options = { 95 | "name": "", 96 | "run": None, 97 | } 98 | assert parse_run_options(run_options) 99 | 100 | run_options = { 101 | "name": "", 102 | "run": RunContext(Mock(), Mock(assignments={"a": 1})), 103 | } 104 | assert parse_run_options(run_options) 105 | 106 | def test_run_options_run_context_object(self): 107 | run_options = {"run": TrainingRun(Mock())} 108 | with pytest.raises(TypeError): 109 | parse_run_options(run_options) 110 | 111 | run_options = { 112 | "run": RunContext(Mock(), Mock(assignments={"a": 1})), 113 | } 114 | assert parse_run_options(run_options) 115 | 116 | def test_run_options_fully_parsed(self): 117 | num_of_options = random.randint(1, len(DEFAULT_RUN_OPTIONS)) 118 | run_options_keys = random.sample(sorted(DEFAULT_RUN_OPTIONS.keys()), num_of_options) 119 | run_options = {k: DEFAULT_RUN_OPTIONS[k] for k in run_options_keys} 120 | parsed_options = parse_run_options(run_options) 121 | assert set(parsed_options.keys()) == set(DEFAULT_RUN_OPTIONS.keys()) 122 | -------------------------------------------------------------------------------- /tools/generate_feature_importances.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2022 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | import string 5 | 6 | import numpy as np 7 | 8 | import sigopt 9 | from sigopt.interface import get_connection 10 | from sigopt.run_context import RunContext 11 | 12 | 13 | def rand_str(n, chars=string.ascii_letters + string.digits): 14 | chars = np.array(list(chars)) 15 | idx = np.random.choice(len(chars), n) 16 | return "".join(chars[idx]) 17 | 18 | 19 | def generate_feature_importances(num_feature=50, max_feature_len=100, score_type="exp"): 20 | lens = np.random.choice(np.arange(1, max_feature_len + 1), num_feature) 21 | features = [rand_str(n) for n in lens] 22 | scores = np.random.uniform(size=num_feature) 23 | if score_type == "exp": 24 | scores = np.exp((scores - 0.5) * 10) 25 | return {"type": "weight", "scores": dict(zip(features, scores))} 26 | 27 | 28 | if __name__ == "__main__": 29 | import argparse 30 | 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument("--run", default=19, help="run id") 33 | parser.add_argument("--score_type", default="exp", help="score type") 34 | parser.add_argument("--num_feature", default=50, help="number of defauts") 35 | parser.add_argument("--max_feature_length", default=100, help="max feature name length") 36 | args = parser.parse_args() 37 | fp = generate_feature_importances(args.num_feature, args.max_feature_length, args.score_type) 38 | context = RunContext(connection=get_connection(), run=sigopt.get_run(args.run)) 39 | context.log_sys_metadata("feature_importances", fp) 40 | -------------------------------------------------------------------------------- /tools/generate_vulture_allowlist: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import subprocess 4 | 5 | 6 | cmd = "./tools/run_vulture.sh . --make-whitelist" 7 | proc = subprocess.run(cmd, shell=True, capture_output=True, text=True, check=False) 8 | pwd = os.getcwd() 9 | print(proc.stdout.rstrip().replace(pwd + "/", "")) 10 | -------------------------------------------------------------------------------- /tools/run_vulture.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright © 2023 Intel Corporation 3 | # 4 | # SPDX-License-Identifier: MIT 5 | set -e 6 | set -o pipefail 7 | 8 | exec vulture --exclude="build,venv" --ignore-decorators="@click.*,@sigopt_cli.*,@public,@pytest.*" --ignore-names="side_effect" "$@" 9 | -------------------------------------------------------------------------------- /trivy.yaml: -------------------------------------------------------------------------------- 1 | db: 2 | download-only: false 3 | light: false 4 | no-progress: true 5 | repository: ghcr.io/aquasecurity/trivy-db 6 | skip-update: false 7 | debug: false 8 | exit-code: 1 9 | format: table 10 | image: 11 | removed-pkgs: false 12 | insecure: false 13 | license: 14 | forbidden: [] 15 | full: false 16 | ignored: [] 17 | notice: [] 18 | permissive: [] 19 | reciprocal: [] 20 | restricted: [] 21 | unencumbered: [] 22 | list-all-pkgs: false 23 | quiet: false 24 | scan: 25 | file-patterns: [] 26 | scanners: 27 | - vuln 28 | - secret 29 | skip-dirs: [] 30 | skip-files: [] 31 | severity: LOW,MEDIUM,HIGH,CRITICAL 32 | timeout: 10m0s 33 | vulnerability: 34 | ignore-unfixed: false 35 | type: os,library 36 | --------------------------------------------------------------------------------