├── .devcontainer └── devcontainer.json ├── .github ├── ISSUE_TEMPLATE │ ├── issue.md │ └── request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── deploy.yml ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── SECURITY.md └── website ├── .gitignore ├── README.md ├── babel.config.js ├── blog ├── 2020-10-08-run-config.md ├── 2020-10-09-run-history.md └── img │ └── logging-metrics.png ├── docs ├── cheatsheets │ ├── cli │ │ └── .gitkeep │ └── python │ │ ├── v1 │ │ ├── cheatsheet.md │ │ ├── ci-dev.md │ │ ├── compute-targets.md │ │ ├── data.md │ │ ├── debugging.md │ │ ├── distributed-training.md │ │ ├── docker-build.md │ │ ├── environment.md │ │ ├── img │ │ │ ├── compute-target.png │ │ │ ├── create-compute-1.png │ │ │ ├── create-compute-lp.png │ │ │ ├── create-compute-lp.pngZone.Identifier │ │ │ ├── create-compute-ssh.png │ │ │ ├── create-compute-ssh.pngZone.Identifier │ │ │ ├── create-compute.png │ │ │ ├── log-files.png │ │ │ ├── logging-metrics.png │ │ │ └── run-ex-sine.png │ │ ├── installation.md │ │ ├── logging.md │ │ ├── run-history.md │ │ ├── run.md │ │ ├── script-run-config.md │ │ ├── templates.md │ │ ├── troubleshooting.md │ │ └── workspace.md │ │ └── v2 │ │ └── .gitkeep ├── misc │ ├── contributing.md │ ├── deployment.md │ └── issues.md └── vs-code-snippets │ ├── python.json │ ├── readme.md │ ├── snippets-parser.py │ ├── snippets.md │ └── vs-code-snippets-demo.gif ├── docusaurus.config.js ├── i18n └── ja │ ├── code.json │ ├── docusaurus-plugin-content-docs │ ├── current.json │ └── current │ │ ├── cheatsheets │ │ ├── cli │ │ │ └── .gitkeep │ │ └── python │ │ │ ├── v1 │ │ │ ├── cheatsheet.md │ │ │ ├── ci-dev.md │ │ │ ├── compute-targets.md │ │ │ ├── data.md │ │ │ ├── debugging.md │ │ │ ├── distributed-training.md │ │ │ ├── docker-build.md │ │ │ ├── environment.md │ │ │ ├── img │ │ │ │ ├── compute-target.png │ │ │ │ ├── create-compute-1.png │ │ │ │ ├── create-compute-lp.png │ │ │ │ ├── create-compute-lp.pngZone.Identifier │ │ │ │ ├── create-compute-ssh.png │ │ │ │ ├── create-compute-ssh.pngZone.Identifier │ │ │ │ ├── create-compute.png │ │ │ │ ├── log-files.png │ │ │ │ ├── logging-metrics.png │ │ │ │ └── run-ex-sine.png │ │ │ ├── installation.md │ │ │ ├── logging.md │ │ │ ├── run-history.md │ │ │ ├── run.md │ │ │ ├── script-run-config.md │ │ │ ├── templates.md │ │ │ ├── troubleshooting.md │ │ │ └── workspace.md │ │ │ └── v2 │ │ │ └── .gitkeep │ │ ├── misc │ │ ├── contributing.md │ │ ├── deployment.md │ │ └── issues.md │ │ └── vs-code-snippets │ │ ├── python.json │ │ ├── readme.md │ │ ├── snippets-parser.py │ │ ├── snippets.md │ │ └── vs-code-snippets-demo.gif │ └── docusaurus-theme-classic │ ├── footer.json │ └── navbar.json ├── package-lock.json ├── package.json ├── plugins ├── appinsights │ └── index.js └── readme.md ├── sidebars.js ├── src ├── css │ └── custom.css └── pages │ ├── index.backup │ ├── index.js │ └── styles.module.css ├── static ├── .nojekyll └── img │ ├── aml-logo.png │ ├── custom-view.png │ ├── favicon.ico │ ├── logo.svg │ ├── undraw_docusaurus_mountain.svg │ ├── undraw_docusaurus_react.svg │ ├── undraw_docusaurus_tree.svg │ ├── view-metrics.png │ └── vs-code-snippets-demo.gif └── yarn.lock /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | # devcontainer json from docusaurus github repo 2 | { 3 | "name": "Docusaurus Dev Container", 4 | "image": "mcr.microsoft.com/vscode/devcontainers/typescript-node:14-buster", 5 | "settings": { 6 | "terminal.integrated.shell.linux": "/bin/bash" 7 | }, 8 | "extensions": ["dbaeumer.vscode-eslint", "orta.vscode-jest"], 9 | "forwardPorts": [3000], 10 | "postCreateCommand": "cd website && yarn install" 11 | } -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue 3 | about: Report an issue with a cheat sheet 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Which cheat sheet? Describe the issue 11 | 12 | cheat sheet: 13 | description: 14 | 15 | ## Additional context 16 | 17 | - 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Request 3 | about: Request a cheat sheet 4 | title: '' 5 | labels: feature-request 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Describe the request 11 | 12 | description: 13 | 14 | ## Additional context 15 | 16 | - 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # PR into Azure/azureml-cheatsheets 2 | 3 | ## Checklist 4 | 5 | I have: 6 | 7 | - [ ] read and followed the [contributing guidelines](https://azure.github.io/azureml-cheatsheets/docs/misc/contributing) 8 | 9 | ## Changes 10 | 11 | - 12 | 13 | fixes # 14 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | on: 3 | workflow_dispatch: 4 | pull_request: 5 | branches: [main] 6 | paths: [website/**] 7 | push: 8 | branches: [main] 9 | paths: [website/**] 10 | jobs: 11 | checks: 12 | if: github.event_name != 'push' 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v1 16 | - uses: actions/setup-node@v1 17 | with: 18 | node-version: '14.x' 19 | - name: Test Build 20 | run: | 21 | cd website 22 | if [ -e yarn.lock ]; then 23 | yarn install --frozen-lockfile 24 | elif [ -e package-lock.json ]; then 25 | npm ci 26 | else 27 | npm i 28 | fi 29 | yarn build 30 | gh-release: 31 | if: github.event_name != 'pull_request' 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v1 35 | - uses: actions/setup-node@v1 36 | with: 37 | node-version: '14.x' 38 | - name: Add key to allow access to repository 39 | env: 40 | SSH_AUTH_SOCK: /tmp/ssh_agent.sock 41 | run: | 42 | mkdir -p ~/.ssh 43 | ssh-keyscan github.com >> ~/.ssh/known_hosts 44 | echo "${{ secrets.GH_PAGES_DEPLOY }}" > ~/.ssh/id_rsa 45 | chmod 600 ~/.ssh/id_rsa 46 | cat <> ~/.ssh/config 47 | Host github.com 48 | HostName github.com 49 | IdentityFile ~/.ssh/id_rsa 50 | EOT 51 | - name: Release to GitHub Pages 52 | env: 53 | USE_SSH: true 54 | GIT_USER: git 55 | run: | 56 | git config --global user.email "actions@gihub.com" 57 | git config --global user.name "gh-actions" 58 | cd website 59 | if [ -e yarn.lock ]; then 60 | yarn install --frozen-lockfile 61 | elif [ -e package-lock.json ]; then 62 | npm ci 63 | else 64 | npm i 65 | fi 66 | yarn deploy 67 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Machine Learning Cheat Sheets 2 | 3 | [![deploy](https://github.com/Azure/azureml-cheatsheets/workflows/deploy/badge.svg)](https://github.com/azure/azureml-cheatsheets/actions?query=workflow%3Adeploy) 4 | 5 | https://azure.github.io/azureml-cheatsheets 6 | 7 | This website is built using [Docusaurus 2](https://v2.docusaurus.io/), a modern static website generator. 8 | 9 | ## Contributions 10 | 11 | Please see the [contributing guide](https://azure.github.io/azureml-cheatsheets/docs/misc/contributing). 12 | 13 | ### Contributor License Agreement 14 | 15 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 16 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 17 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 18 | 19 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 20 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 21 | provided by the bot. You will only need to do this once across all repos using our CLA. 22 | 23 | ### Code of Conduct 24 | 25 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 26 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 27 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 28 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /website/.gitignore: -------------------------------------------------------------------------------- 1 | # dependencies 2 | /node_modules 3 | 4 | # production 5 | /build 6 | 7 | # generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # misc 12 | .DS_Store 13 | .vscode 14 | .env.local 15 | .env.development.local 16 | .env.test.local 17 | .env.production.local 18 | 19 | npm-debug.log* 20 | yarn-debug.log* 21 | yarn-error.log* -------------------------------------------------------------------------------- /website/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This is the source code for the website. 4 | 5 | This website is built using [Docusaurus 2](https://v2.docusaurus.io/), a modern static website generator. 6 | 7 | Layout: 8 | 9 | - `docs/`: Contains the markdown files used to generate the webpage. Main sections are: 10 | - `cheatsheet/` 11 | - `vs-code-snippets/` 12 | - `tempaltes/` 13 | - `docusaurus.config.js`: Configuration file for the webpage generation. Controls the nav bar and some 14 | other global information (e.g. AppInsights logging is configured here) 15 | - `sidebar.js`: Modify the sidebars in the webpages 16 | 17 | Typically you shouldn't need to modify any of the other files and folders. 18 | -------------------------------------------------------------------------------- /website/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /website/blog/2020-10-08-run-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: run-config 3 | title: What is a Run Configuration? 4 | author: Amin Saied 5 | author_title: Data Scientist @ Azure ML 6 | author_url: https://github.com/aminsaied 7 | author_image_url: https://avatars.githubusercontent.com/aminsaied 8 | tags: [run] 9 | --- 10 | 11 | What _is_ a `RunConfiguration`? In practice you should never have to build one by hand. 12 | Azure ML provides higher-level concepts that to handle this for you - for example, the 13 | `ScriptRunConfig`. It is however useful to know what aspects of a run it controls, and to see how to modify 14 | the parts you may have cause to change. 15 | 16 | 17 | 18 | ## ScriptRunConfig 19 | 20 | The `ScriptRunConfig` is a good place to start. The clue is in the name 21 | here: this class combines your script together with your desired run-time configuration: 22 | 23 | - Script: 24 | - Source code: usually a directory e.g. `src/` consisting of models and training code 25 | - Script arguments: a series of command-line arguments that control various hyperparameters 26 | - Run Configuration: Everything you need to make your script run as intended in a remote environment. -------------------------------------------------------------------------------- /website/blog/2020-10-09-run-history.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: run-history 3 | title: What is Run History? 4 | author: Amin Saied 5 | author_title: Data Scientist @ Azure ML 6 | author_url: https://github.com/aminsaied 7 | author_image_url: https://avatars.githubusercontent.com/aminsaied 8 | tags: [run] 9 | --- 10 | 11 | Azure ML can supercharge your ML workloads in (at least!) two ways: 12 | 13 | - AML Compute: Providing powerful compute resoures to train larger models faster 14 | - Run history: Best-in-class lineage and reproducability 15 | 16 | In this article we focus on Run History - and why you need it in your life! 17 | 18 | 19 | 20 | As teams progress to running dozens, and eventually hundreds of experiments, having 21 | some way to organize them is essential. Run History is a service that provides a number 22 | features that quickly become essential to your ML-model builders. 23 | 24 | ### Experiments and Runs 25 | 26 | When you are running dozens of experiments in multiple different projects, having a clear 27 | way to organize and search though the results is key. Azure ML provides two concepts to help 28 | with this: `Run`s and `Experiment`s. 29 | 30 | #### Runs 31 | A run is a single execution of your code - usually a training script. The run has a life-cycle: 32 | the code is prepared to be submited to Azure ML (e.g. via a ScriptRunConfig), then the code is 33 | submitted 34 | 35 | Once the code is submitted to Azure ML (for example, via a `ScriptRunConfig`) a `Run` object is 36 | created. This compute target is prepared (nodes are provisioned, containers hosting your Python 37 | environment are fired up), the entry point script is called (`$ python run.py [args]`) and logs 38 | start being generated: 39 | 40 | ```console 41 | Files already downloaded and verified 42 | epoch=1, batch= 2000: loss 2.19 43 | epoch=1, batch= 4000: loss 1.82 44 | epoch=1, batch= 6000: loss 1.66 45 | ... 46 | ``` 47 | 48 | You may log metrics to Azure ML with `run.log('', metric_value)` and monitor them in the studio: 49 | 50 | ![](img/logging-metrics.png) 51 | 52 | The training concludes, usually some model files are saved, and the nodes are 53 | released. 54 | 55 | But the story doesn't end there. The run persists even after the nodes are returned 56 | to Azure. You can always return, either in code or via the studio, to see a history 57 | of your runs, all their outputs and metrics, and the exact code that was used to generate them. 58 | 59 | #### Experiments 60 | 61 | An Experiment is a collection of runs. All runs belongs to an Experiment. Usually 62 | an Experiment is tied to a specific work item, for example, "Finetune Bert-Large", 63 | and will posses a number of runs as you iterate toward this goal. 64 | 65 | ### Snapshot 66 | 67 | When you submit your code to run in Azure ML, a _snapshot_ is taken. This is a copy of the exact 68 | code that ran. Think of this as version control for your experiments. Want to reproduce the 69 | results from that experiment 2-months ago even though you've iterated on the model and the 70 | training script in the meantime? No problem, snapshot has you covered! 71 | 72 | You have total control of what goes into the snapshot with the `.amlignore` file. This plays 73 | the same role as a `.gitignore` so you can efficiently manage what to include in the snapshot. 74 | 75 | ### Metrics 76 | 77 | As you run experiments, you track metrics - from validation loss through to GPU load. Analysing these metrics is essential to determining your best model. With Run History, these metrics are stored for all your runs. -------------------------------------------------------------------------------- /website/blog/img/logging-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/blog/img/logging-metrics.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/cli/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/cheatsheet.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Cheat Sheet 3 | id: cheatsheet 4 | description: A cheat sheet for Azure ML. 5 | keywords: 6 | - azure machine learning 7 | - aml 8 | - cheatsheet 9 | - overview 10 | --- 11 | 12 | ## Basic setup 13 | 14 | ### Connect to workspace 15 | 16 | ```python 17 | from azureml.core import Workspace 18 | ws = Workspace.from_config() 19 | ``` 20 | 21 | The workspace object is the fundamental handle on your Azure ML assets and is used 22 | throughout (often simply referred to by `ws`). 23 | 24 | For more details: [Workspaces](./workspace.md) 25 | 26 | ### Connect to compute target 27 | 28 | ```python 29 | compute_target = ws.compute_targets[''] 30 | ``` 31 | 32 | **Sample usage.** 33 | 34 | ```python 35 | compute_target = ws.compute_targets['powerful-gpu'] 36 | 37 | config = ScriptRunConfig( 38 | compute_target=compute_target, # compute target used to run train.py script 39 | source_directory='.', 40 | script='train.py', 41 | ) 42 | ``` 43 | 44 | For more details: [Compute Target](./compute-targets.md) 45 | 46 | ### Prepare Python environment 47 | 48 | You can use a pip `requirements.txt` file or a Conda `env.yml` file to define a Python environment on your compute. 49 | 50 | ```python 51 | from azureml.core import Environment 52 | # Option 1. From pip 53 | environment = Environment.from_pip_requirements('', '') 54 | # Option 2. From Conda 55 | environment = Environment.from_conda_specification('', '') 56 | ``` 57 | 58 | You can also use docker images to prepare your environments. 59 | 60 | **Sample usage.** 61 | 62 | ```python 63 | environment = Environment.from_pip_requirements('', '') 64 | 65 | config = ScriptRunConfig( 66 | environment=environment, # set the python environment 67 | source_directory='.', 68 | script='train.py', 69 | ) 70 | ``` 71 | 72 | For more details: [Environment](./environment.md) 73 | 74 | 75 | ## Submit code 76 | 77 | To run code in Azure ML you need to: 78 | 79 | 1. **Configure**: Configuration includes specifying the code to run, the compute 80 | target to run on and the Python environment to run in. 81 | 2. **Submit**: Create or reuse an Azure ML Experiment and submit the run. 82 | 83 | ### ScriptRunConfig 84 | 85 | A typical directory may have the following structure: 86 | 87 | ```bash 88 | source_directory/ 89 | script.py # entry point to your code 90 | module1.py # modules called by script.py 91 | ... 92 | ``` 93 | 94 | To run `$ (env) python /script.py [arguments]` on a remote compute 95 | cluster `target: ComputeTarget` with an environment `env: Environment` we can use 96 | the `ScriptRunConfig` class. 97 | 98 | ```python 99 | from azureml.core import ScriptRunConfig 100 | 101 | config = ScriptRunConfig( 102 | source_directory='', # relative paths okay 103 | script='script.py', 104 | compute_target=compute_target, 105 | environment=environment, 106 | arguments=arguments, 107 | ) 108 | ``` 109 | 110 | For more details on arguments: [Command line arguments](./script-run-config.md#command-line-arguments) 111 | 112 | :::info 113 | - `compute_target`: If not provided the script will run on your local machine. 114 | - `environment`: If not provided, uses a default Python environment managed by Azure ML. See [Environment](./environment.md) for more details. 115 | ::: 116 | 117 | #### Commands 118 | 119 | It is possible to provide the explicit command to run. 120 | 121 | ```python 122 | command = 'echo cool && python script.py'.split() 123 | 124 | config = ScriptRunConfig( 125 | source_directory='', # relative paths okay 126 | command=command, 127 | compute_target=compute_target, 128 | environment=environment, 129 | arguments=arguments, 130 | ) 131 | ``` 132 | 133 | For more details: [Commands](./script-run-config.md#commands) 134 | 135 | ### Experiment 136 | 137 | To submit this code, create an `Experiment`: a light-weight container that helps to 138 | organize our submissions and keep track of code (See [Run History](./run-history.md)). 139 | 140 | ```python 141 | exp = Experiment(ws, '') 142 | run = exp.submit(config) 143 | print(run.get_portal_url()) 144 | ``` 145 | 146 | This link will take you to the Azure ML Studio where you can monitor your run. 147 | 148 | For more details: [ScriptRunConfig](./script-run-config.md) 149 | 150 | ### Sample usage 151 | 152 | Here is a fairly typical example using a Conda environment to run a training 153 | script `train.py` on our local machine from the command line. 154 | 155 | ```bash 156 | $ conda env create -f env.yml # create environment called pytorch 157 | $ conda activate pytorch 158 | (pytorch) $ cd 159 | (pytorch) $ python train.py --learning_rate 0.001 --momentum 0.9 160 | ``` 161 | 162 | Suppose you want to run this on a GPU in Azure. 163 | 164 | ```python 165 | ws = Workspace.from_config() 166 | compute_target = ws.compute_targets['powerful-gpu'] 167 | environment = Environment.from_conda_specification('pytorch', 'env.yml') 168 | 169 | config = ScriptRunConfig( 170 | source_directory='', 171 | script='train.py', 172 | environment=environment, 173 | arguments=['--learning_rate', 0.001, '--momentum', 0.9], 174 | ) 175 | 176 | run = Experiment(ws, 'PyTorch model training').submit(config) 177 | ``` 178 | 179 | ## Distributed GPU Training 180 | 181 | Adapt your `ScriptRunConfig` to enable distributed GPU training. 182 | 183 | ```python {3,8-9,12,19} 184 | from azureml.core import Workspace, Experiment, ScriptRunConfig 185 | from azureml.core import Environment 186 | from azureml.core.runconfig import MpiConfiguration 187 | 188 | ws = Workspace.from_config() 189 | compute_target = ws.compute_targets['powerful-gpu'] 190 | environment = Environment.from_conda_specification('pytorch', 'env.yml') 191 | environment.docker.enabled = True 192 | environment.docker.base_image = 'mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.1-cudnn7-ubuntu18.04' 193 | 194 | # train on 2 nodes each with 4 GPUs 195 | mpiconfig = MpiConfiguration(process_count_per_node=4, node_count=2) 196 | 197 | config = ScriptRunConfig( 198 | source_directory='', # directory containing train.py 199 | script='train.py', 200 | environment=environment, 201 | arguments=['--learning_rate', 0.001, '--momentum', 0.9], 202 | distributed_job_config=mpiconfig, # add the distributed configuration 203 | ) 204 | 205 | run = Experiment(ws, 'PyTorch model training').submit(config) 206 | ``` 207 | 208 | :::info 209 | - `mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.1-cudnn7-ubuntu18.04` is a docker image 210 | with OpenMPI. This is required for distributed training on Azure ML. 211 | - `MpiConfiguration` is where you specify the number of nodes and GPUs (per node) you 212 | want to train on. 213 | ::: 214 | 215 | For more details: [Distributed GPU Training](./distributed-training.md) 216 | 217 | ## Connect to data 218 | 219 | To work with data in your training scripts using your workspace `ws` and its default datastore: 220 | 221 | ```python 222 | datastore = ws.get_default_datastore() 223 | dataset = Dataset.File.from_files(path=(datastore, '')) 224 | ``` 225 | For more details see: [Data](./data.md) 226 | 227 | Pass this to your training script as a command line argument. 228 | 229 | ```python 230 | arguments=['--data', dataset.as_mount()] 231 | ``` 232 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/ci-dev.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Developing on Azure ML 3 | description: Guide to developing your code on Azure ML. 4 | keywords: 5 | - ssh 6 | - development 7 | - compute 8 | --- 9 | 10 | This guide gives some pointers for developing your code on Azure ML. A typical 11 | scenario might be testing your distributed training code, or some other aspect 12 | of your code that isn't well represented on your local devbox. 13 | 14 | A common pain-point in these scenarios is that iteration on Azure ML can feel 15 | slow - especially when compared to developing on a VM. 16 | 17 | **Learning objective.** To improve the development experience on Azure ML 18 | to match - or even exceed - that of a "bare" VM. 19 | 20 | ## 🚧 The hurdles 21 | 22 | Two main reasons developing on Azure ML can feel slow as compared to a VM are: 23 | 24 | - Any changes to my Python environment force Docker image rebuild which can 25 | take >5 minutes. 26 | 27 | - Compute resources are _released_ between iterations, forcing me to wait for 28 | new compute to warm up (e.g. pulling Docker images). 29 | 30 | Below we provide some techniques to address these issues, as well as some advantages 31 | to working with Azure ML compute directly. We also provide a [example](#example) applying these 32 | techniques. 33 | 34 | ## 🕰️ Prepare compute for development 35 | 36 | When creating your _compute instance / cluster_ there are a fews things you can 37 | do to prepare for development: 38 | 39 | 1. **Enable SSH on compute.** 40 | 41 | Supported on both _compute instance_ and _compute targets_. This will allow you to 42 | use your compute just like you would a VM. 43 | 44 | :::tip VS Code Remote Extension. 45 | VS Code's [remote extension](https://code.visualstudio.com/docs/remote/ssh) 46 | allows you to connect to your Azure ML compute resources via SSH. 47 | This way you can develop directly in the cloud. 48 | ::: 49 | 50 | 2. **Increase "Idle seconds before scale down".** 51 | 52 | For compute targets you can increase this parameter e.g. to 30 minutes. This means 53 | the cluster won't be released between runs while you iterate. 54 | 55 | :::warning 56 | Don't forget to roll this back when you're done iterating. 57 | ::: 58 | 59 | ## 🏃‍♀️ Commands 60 | 61 | Typically you will submit your code to Azure ML via a `ScriptRunConfig` a little like this: 62 | 63 | ```python 64 | config = ScriptRunConfig( 65 | source_directory='', 66 | script='script.py', 67 | compute_target=target, 68 | environment=env, 69 | ... 70 | ) 71 | ``` 72 | 73 | :::info 74 | For more details on using `ScriptRunConfig` to submit your code see 75 | [Running Code in the cloud](script-run-config). 76 | ::: 77 | 78 | By using the [`command`](script-run-config#commands) argument you can improve your agility. 79 | Commands allow you to chain together several steps in one e.g.: 80 | 81 | ```python 82 | command = "pip install torch && python script.py --learning_rate 2e-5".split() 83 | ``` 84 | 85 | Another example would be to include a setup script: 86 | 87 | ```bash title="setup.sh" 88 | echo "Running setup script" 89 | pip install torch 90 | pip install -r requirements.txt 91 | export PYTHONPATH=$PWD 92 | ``` 93 | 94 | and then calling it in your command 95 | 96 | ```python 97 | command = "bash setup.sh && python script.py --learning_rate 2e-5".split() 98 | ``` 99 | 100 | This way Azure ML doesn't have to rebuild the docker image with incremental changes. 101 | 102 | ## Advantages 103 | 104 | In addition to matching the development experience on a VM, there are certain benefits to 105 | developing on Azure ML compute directly. 106 | 107 | - **Production-ready.** By developing directly in Azure ML you avoid the additional step of porting your 108 | VM-developed code to Azure ML later. This is particularly relevant if you intend to 109 | run your production code on Azure ML. 110 | - **Data access.** If your training script makes use of data in Azure you can use the Azure ML 111 | Python SDK to read it (see [Data](data) for examples). The alternative is that you might have to 112 | find some way of getting your data onto the VM you are developing on. 113 | - **Notebooks.** Azure ML's _compute insances_ come with Jupyter notebooks which can help with quick 114 | debugging. Moreover, these notebooks can easily be run against different compute infrastructure 115 | and can be a great way to collaborate. 116 | 117 | ## Example 118 | 119 | We provide a simple example demonstrating the mechanics of the above steps. Consider the following 120 | setup: 121 | 122 | ```bash 123 | src/ 124 | .azureml/ 125 | config.json # workspace connection config 126 | train.py # python script we are developing 127 | setup.sh # to run on compute before train.py 128 | azureml_run.py # submit job to azure 129 | ``` 130 | 131 | ```bash title="setup.sh" 132 | echo "Running setup script" 133 | pip install numpy 134 | ``` 135 | 136 | ```python title="train.py" 137 | import numpy as np 138 | print(np.random.rand()) 139 | ``` 140 | 141 | Now from your local machine you can use the Azure ML Python SDK 142 | to execute your command in the cloud: 143 | 144 | ```python title="azureml_run.py" 145 | from azureml.core import Workspace, Experiment, ScriptRunConfig 146 | 147 | # get workspace 148 | ws = Workspace.from_config() 149 | target = ws.compute_targets['cpucluster'] 150 | exp = Experiment(ws, 'dev-example') 151 | 152 | command = "bash setup.sh && python script.py".split() 153 | 154 | # set up script run configuration 155 | config = ScriptRunConfig( 156 | source_directory='.', 157 | command=command, 158 | compute_target=target, 159 | ) 160 | 161 | # submit script to AML 162 | run = exp.submit(config) 163 | print(run.get_portal_url()) # link to ml.azure.com 164 | run.wait_for_completion(show_output=True) 165 | ``` 166 | 167 | Now if you needed to update your Python environment for example you can simply 168 | add commands to `setup.sh`: 169 | 170 | ```bash title="setup.sh" 171 | echo "Running setup script" 172 | pip install numpy 173 | pip install pandas # add additional libraries 174 | export CUDA_VISIBLE_DEVICES="0,1" # set environment variables 175 | nvidia-smi # run helpful command-line tools 176 | ``` 177 | 178 | without having to rebuild any Docker images. -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/compute-targets.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Compute Target 3 | description: Guide to setting up and using Azure compute resources in Azure ML. 4 | keywords: 5 | - compute 6 | - cpu 7 | - gpu 8 | --- 9 | 10 | Compute targets are an AML abstraction around the concept of a compute resource. 11 | This can range from your local machine to a cluster of Azure VMs. 12 | 13 | ### Get Compute Target 14 | 15 | To get an existing compute target within a workspace `ws`: 16 | 17 | ```python 18 | from azureml.core import ComputeTarget 19 | target = ComputeTarget(ws, '') 20 | ``` 21 | 22 | ### List Existing Compute Targets 23 | 24 | To get a list of compute targets within a workspace `ws` 25 | 26 | ```python 27 | ComputeTarget.list(ws): List[ComputeTarget] 28 | ``` 29 | 30 | ### Check availabilty 31 | 32 | A very common task - especially if you are sharing a workspace within a team - is 33 | to check the compute resources available within a workspace `ws` prior to submitting 34 | a job. 35 | 36 | The easiest way to do this is via the [studio](https://ml.azure.com). 37 | 38 | ![](img/compute-target.png) 39 | 40 | ## Creating Compute Targets 41 | 42 | The easiest way to create a new Compute Target is via the [studio](https://ml.azure.com) 43 | 44 | Select the "Compute" menu > Select "Compute clusters" tab > Select the "+ New" button: 45 | 46 | ![](img/create-compute.png) 47 | 48 | Complete the following info: 49 | 50 | - **Compute name**: This will be used later to reference the compute. Name is required. Name must be between 2 to 16 characters. Valid characters are letters, digits, and the - character 51 | - **Virtual Machine type**: Either CPU or GPU 52 | - **Virtual Machine Priority**: Either "Dedicated" or "Low priority"> Low priority virtual machines are cheaper but don't guarantee the compute nodes. Your job may be pre-empted. 53 | - **Virtual Machine Size**: Select from a drop down list. See the full list [here](https://azure.microsoft.com/global-infrastructure/services/?products=virtual-machines) 54 | - **Min / Max number of nodes**: Compute will autoscale between the min and max node count depending on the number of jobs submitted. By setting min nodes = 0 the cluster will scale to 0 when there are no running jobs on the compute - saving you money. 55 | - **Idle seconds before scale down**: Idle time before we scale down the cluster to the minimum node count. 56 | 57 | Note: Machine Learning compute is always created in the same region as the Machine Learning service workspace. 58 | 59 | ### with SSH 60 | 61 | Enable SSH access on your compute target by selecting the "Advanced" drop-down and entering the admin 62 | username/password as well as providing the SSH key. 63 | 64 | ![](img/create-compute-ssh.png) 65 | 66 | ### Low Priority Compute Targets 67 | 68 | Low priority compute targets are cheaper but don't guarantee the compute nodes. Your job may be pre-empted. 69 | 70 | ![](img/create-compute-lp.png) 71 | 72 | ### via the SDK 73 | 74 | To create a compute target in code with the SDK: 75 | 76 | ```python 77 | from azureml.core import Workspace 78 | from azureml.core.compute import ComputeTarget, AmlCompute 79 | from azureml.core.compute_target import ComputeTargetException 80 | 81 | ws = Workspace.from_config() # This automatically looks for a directory .azureml 82 | 83 | # Choose a name for your CPU cluster 84 | cpu_cluster_name = "cpu-cluster" 85 | 86 | # Verify that the cluster does not exist already 87 | try: 88 | cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name) 89 | print('Found existing cluster, use it.') 90 | except ComputeTargetException: 91 | compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', 92 | max_nodes=4, 93 | idle_seconds_before_scaledown=2400) 94 | cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, compute_config) 95 | 96 | cpu_cluster.wait_for_completion(show_output=True) 97 | ``` -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/data.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Data 3 | description: Guide to working with data in Azure ML. 4 | keywords: 5 | - data 6 | - dataset 7 | - datastore 8 | --- 9 | 10 | ## Concepts 11 | 12 | AzureML provides two basic assets for working with data: 13 | 14 | - Datastore 15 | - Dataset 16 | 17 | ### Datastore 18 | 19 | Provides an interface for numerous Azure Machine Learning storage accounts. 20 | 21 | Each Azure ML workspace comes with a default datastore: 22 | 23 | ```python 24 | from azureml.core import Workspace 25 | ws = Workspace.from_config() 26 | datastore = ws.get_default_datastore() 27 | ``` 28 | 29 | which can also be accessed directly from the [Azure Portal](https://portal.azure.com) (under the same 30 | resource group as your Azure ML Workspace). 31 | 32 | Datastores are attached to workspaces and are used to store connection information to Azure storage services so you can refer to them by name and don't need to remember the connection information and secret used to connect to the storage services. 33 | 34 | Use this class to perform management operations, including register, list, get, and remove datastores. 35 | 36 | ### Dataset 37 | 38 | A dataset is a reference to data - either in a datastore or behind a public URL. 39 | 40 | Datasets provide enhaced capabilities including data lineage (with the notion of versioned datasets). 41 | 42 | 43 | ## Get Datastore 44 | 45 | ### Default datastore 46 | 47 | Each workspace comes with a default datastore. 48 | 49 | ```python 50 | datastore = ws.get_default_datastore() 51 | ``` 52 | 53 | ### Register datastore 54 | 55 | Connect to, or create, a datastore backed by one of the multiple data-storage options 56 | that Azure provides. For example: 57 | 58 | - Azure Blob Container 59 | - Azure Data Lake (Gen1 or Gen2) 60 | - Azure File Share 61 | - Azure MySQL 62 | - Azure PostgreSQL 63 | - Azure SQL 64 | - Azure Databricks File System 65 | 66 | See the SDK for a comprehensive list of datastore types and authentication options: 67 | [Datastores (SDK)](https://docs.microsoft.com/python/api/azureml-core/azureml.core.datastore(class)?view=azure-ml-py). 68 | 69 | #### Register a new datastore 70 | 71 | - To register a store via an **account key**: 72 | 73 | ```python 74 | datastores = Datastore.register_azure_blob_container( 75 | workspace=ws, 76 | datastore_name='', 77 | container_name='', 78 | account_name='', 79 | account_key='', 80 | ) 81 | ``` 82 | 83 | - To register a store via a **SAS token**: 84 | 85 | ```python 86 | datastores = Datastore.register_azure_blob_container( 87 | workspace=ws, 88 | datastore_name='', 89 | container_name='', 90 | account_name='', 91 | sas_token='', 92 | ) 93 | ``` 94 | 95 | ### Connect to datastore 96 | 97 | The workspace object `ws` has access to its datastores via 98 | 99 | ```python 100 | ws.datastores: Dict[str, Datastore] 101 | ``` 102 | 103 | Any datastore that is registered to workspace can thus be accessed by name. 104 | 105 | ```python 106 | datastore = ws.datastores[''] 107 | ``` 108 | 109 | ### Link datastore to Azure Storage Explorer 110 | 111 | The workspace object `ws` is a very powerful handle when it comes to managing assets the 112 | workspace has access to. For example, we can use the workspace to connect to a datastore 113 | in Azure Storage Explorer. 114 | 115 | ```python 116 | from azureml.core import Workspace 117 | ws = Workspace.from_config() 118 | datastore = ws.datastores[''] 119 | ``` 120 | 121 | - For a datastore that was created using an **account key** we can use: 122 | 123 | ```python 124 | account_name, account_key = datastore.account_name, datastore.account_key 125 | ``` 126 | 127 | - For a datastore that was created using a **SAS token** we can use: 128 | 129 | ```python 130 | sas_token = datastore.sas_token 131 | ``` 132 | 133 | The account_name and account_key can then be used directly in Azure Storage Explorer to 134 | connect to the Datastore. 135 | 136 | ## Blob Datastore 137 | 138 | Move data to and from your [AzureBlobDatastore](https://docs.microsoft.com/python/api/azureml-core/azureml.data.azure_storage_datastore.azureblobdatastore?view=azure-ml-py) object `datastore`. 139 | 140 | ### Upload to Blob Datastore 141 | 142 | The AzureBlobDatastore provides APIs for data upload: 143 | 144 | ```python 145 | datastore.upload( 146 | src_dir='./data', 147 | target_path='', 148 | overwrite=True, 149 | ) 150 | ``` 151 | 152 | Alternatively, if you are working with multiple files in different locations you can use 153 | 154 | ```python 155 | datastore.upload_files( 156 | files, # List[str] of absolute paths of files to upload 157 | target_path='', 158 | overwrite=False, 159 | ) 160 | ``` 161 | 162 | ### Download from Blob Datastore 163 | 164 | Download the data from the blob container to the local file system. 165 | 166 | ```python 167 | datastore.download( 168 | target_path, # str: local directory to download to 169 | prefix='', 170 | overwrite=False, 171 | ) 172 | ``` 173 | 174 | ### Via Storage Explorer 175 | 176 | Azure Storage Explorer is free tool to easily manage your Azure cloud storage 177 | resources from Windows, macOS, or Linux. Download it from [here](https://azure.microsoft.com/features/storage-explorer/). 178 | 179 | Azure Storage Explorer gives you a (graphical) file exporer, so you can literally drag-and-drop 180 | files into and out of your datastores. 181 | 182 | See ["Link datastore to Azure Storage Explorer"](#link-datastore-to-azure-storage-explorer) 183 | above for more details. 184 | 185 | ## Read from Datastore 186 | 187 | Reference data in a `Datastore` in your code, for example to use in a remote setting. 188 | 189 | ### DataReference 190 | 191 | First, connect to your basic assets: `Workspace`, `ComputeTarget` and `Datastore`. 192 | 193 | ```python 194 | from azureml.core import Workspace 195 | ws: Workspace = Workspace.from_config() 196 | compute_target: ComputeTarget = ws.compute_targets[''] 197 | ds: Datastore = ws.get_default_datastore() 198 | ``` 199 | 200 | Create a `DataReference`, either as mount: 201 | 202 | ```python 203 | data_ref = ds.path('').as_mount() 204 | ``` 205 | 206 | or as download: 207 | 208 | ```python 209 | data_ref = ds.path('').as_download() 210 | ``` 211 | :::info 212 | To mount a datastore the workspace need to have read and write access to the underlying storage. For readonly datastore `as_download` is the only option. 213 | ::: 214 | 215 | #### Consume DataReference in ScriptRunConfig 216 | 217 | Add this DataReference to a ScriptRunConfig as follows. 218 | 219 | ```python 220 | config = ScriptRunConfig( 221 | source_directory='.', 222 | script='script.py', 223 | arguments=[str(data_ref)], # returns environment variable $AZUREML_DATAREFERENCE_example_data 224 | compute_target=compute_target, 225 | ) 226 | 227 | config.run_config.data_references[data_ref.data_reference_name] = data_ref.to_config() 228 | ``` 229 | 230 | The command-line argument `str(data_ref)` returns the environment variable `$AZUREML_DATAREFERENCE_example_data`. 231 | Finally, `data_ref.to_config()` instructs the run to mount the data to the compute target and to assign the 232 | above environment variable appropriately. 233 | 234 | #### Without specifying argument 235 | 236 | Specify a `path_on_compute` to reference your data without the need for command-line arguments. 237 | 238 | ```python 239 | data_ref = ds.path('').as_mount() 240 | data_ref.path_on_compute = '/tmp/data' 241 | 242 | config = ScriptRunConfig( 243 | source_directory='.', 244 | script='script.py', 245 | compute_target=compute_target, 246 | ) 247 | 248 | config.run_config.data_references[data_ref.data_reference_name] = data_ref.to_config() 249 | ``` 250 | 251 | ## Create Dataset 252 | 253 | ### From local data 254 | You could create and register a dataset directly from a folder on your local machine. Note that `src_dir` must point to a **folder**, not file. 255 | 256 | :warning: Method `upload_directory`: This is an experimental method, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information. 257 | ```python 258 | from azureml.core import Dataset 259 | 260 | # upload the data to datastore and create a FileDataset from it 261 | folder_data = Dataset.File.upload_directory(src_dir="path/to/folder", target=(datastore, "self-defined/path/on/datastore")) 262 | dataset = folder_data.register(workspace=ws, name="") 263 | ``` 264 | 265 | ### From a datastore 266 | The code snippet below shows how to create a `Dataset` given a relative path on `datastore`. Note that the path could either point to a folder (e.g. `local/test/`) or a single file (e.g. `local/test/data.tsv`). 267 | ```python 268 | from azureml.core import Dataset 269 | 270 | # create input dataset 271 | data = Dataset.File.from_files(path=(datastore, "path/on/datastore")) 272 | dataset = data.register(workspace=ws, name="") 273 | ``` 274 | 275 | ### From outputs using `OutputFileDatasetConfig` 276 | ```python 277 | from azureml.core import ScriptRunConfig 278 | from azureml.data import OutputFileDatasetConfig 279 | 280 | output_data = OutputFileDatasetConfig( 281 | destination=(datastore, "path/on/datastore"), 282 | name="", 283 | ) 284 | 285 | config = ScriptRunConfig( 286 | source_directory=".", 287 | script="run.py", 288 | arguments=["--output_dir", output_data.as_mount()], 289 | ) 290 | 291 | # register your OutputFileDatasetConfig as a dataset 292 | output_data_dataset = output_data.register_on_complete(name="", description = "") 293 | ``` 294 | 295 | #### Upload to datastore 296 | 297 | To upload a local directory `./data/`: 298 | 299 | ```python 300 | datastore = ws.get_default_datastore() 301 | datastore.upload(src_dir='./data', target_path='', overwrite=True) 302 | ``` 303 | 304 | This will upload the entire directory `./data` from local to the default datastore associated 305 | to your workspace `ws`. 306 | 307 | #### Create dataset from files in datastore 308 | 309 | To create a dataset from a directory on a datastore at ``: 310 | 311 | ```python 312 | datastore = ws.get_default_datastore() 313 | dataset = Dataset.File.from_files(path=(datastore, '')) 314 | ``` 315 | 316 | ## Use Dataset 317 | 318 | ### ScriptRunConfig 319 | 320 | To reference data from a dataset in a ScriptRunConfig you can either mount or download the 321 | dataset using: 322 | 323 | - `dataset.as_mount(path_on_compute)` : mount dataset to a remote run 324 | - `dataset.as_download(path_on_compute)` : download the dataset to a remote run 325 | 326 | **Path on compute** Both `as_mount` and `as_download` accept an (optional) parameter `path_on_compute`. 327 | This defines the path on the compute target where the data is made available. 328 | 329 | - If `None`, the data will be downloaded into a temporary directory. 330 | - If `path_on_compute` starts with a `/` it will be treated as an **absolute path**. (If you have 331 | specified an absolute path, please make sure that the job has permission to write to that directory.) 332 | - Otherwise it will be treated as relative to the working directory 333 | 334 | Reference this data in a remote run, for example in mount-mode: 335 | 336 | ```python title="run.py" 337 | arguments=[dataset.as_mount()] 338 | config = ScriptRunConfig(source_directory='.', script='train.py', arguments=arguments) 339 | experiment.submit(config) 340 | ``` 341 | 342 | and consumed in `train.py`: 343 | 344 | ```python title="train.py" 345 | import sys 346 | data_dir = sys.argv[1] 347 | 348 | print("===== DATA =====") 349 | print("DATA PATH: " + data_dir) 350 | print("LIST FILES IN DATA DIR...") 351 | print(os.listdir(data_dir)) 352 | print("================") 353 | ``` 354 | 355 | For more details: [ScriptRunConfig](script-run-config) 356 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/debugging.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Debugging 3 | description: Guide to debugging in Azure ML. 4 | keywords: 5 | - debug 6 | - log files 7 | --- 8 | 9 | ## Azure ML Log Files 10 | 11 | Azure ML's log files are an essential resource for debugging your Azure ML workloads. 12 | 13 | | Log file | Description | 14 | | - | - | 15 | | `20_image_build_log*.txt` | Docker build logs. Only applicable when updating your Environment. Otherwise Azure ML will reuse cached image.

If successful, contains image registry details for the corresponding image.| 16 | | `55_azureml-execution*.txt` | Pulls image to compute target. Note, this log only appears once you have secured compute resources.| 17 | | `65_job_prep*.txt` | Job preparation: Download your code to compute target and datastores (if requested). | 18 | | **`70_driver_log.txt`** | **The standard output from your script. This is where your code's logs (e.g. print statements) show up.**

In the majority of cases you will monitor the logs here. | 19 | | `75_job_post*.txt` | Job release: Send logs, release the compute resources back to Azure. | 20 | 21 | :::info 22 | You will not necessarily see every file for every run. For example, the `20_image_build_log*.txt` only appears when a new image is built (e.g. when you change you environment). 23 | ::: 24 | 25 | ### Find logs in the Studio 26 | 27 | These log files are available via the Studio UI at https://ml.azure.com under Workspace > Experiment > 28 | Run > "Outputs and logs". 29 | 30 | ![](img/log-files.png) 31 | 32 | ### Streaming logs 33 | 34 | It is also possible to stream these logs directly to your local terminal using a `Run` object, 35 | for example: 36 | 37 | ```python 38 | from azureml.core import Workspace, Experiment, ScriptRunConfig 39 | ws = Workspace.from_config() 40 | config = ScriptRunConfig(...) 41 | run = Experiment(ws, 'my-amazing-experiment').submit(config) 42 | run.wait_for_completion(show_output=True) 43 | ``` 44 | 45 | ## SSH 46 | 47 | It can be useful to SSH into your compute for a variety of reasons - including to assist in debugging. 48 | 49 | :::warning Enable SSH at compute creation 50 | SSH needs to be enabled when you create the compute instance / target - see [Compute Targets](compute-targets#with-ssh) for details. 51 | ::: 52 | 53 | 1. Get **public ip** and **port number** for your compute. 54 | 55 | Visit [ml.azure.com](https://ml.azure.com/) > select "Compute" tab > Locate the desired compute instance / target. 56 | 57 | **Note.** The compute needs to be running in order to connect. 58 | - In the case of compute instance this just requires turning it on. 59 | - For compute targets there should be something running on the cluster. In this case you can select the "Nodes" tab of the cluster ([ml.azure.com](https://ml.azure.com/) > Compute > _your compute target_ > Nodes) to get Public IP & port number for each node. 60 | 61 | 2. Open your favorite shell and run: 62 | 63 | ```bash 64 | ssh azureuser@ -p 65 | ``` 66 | 67 | 68 | :::info SSH key pair using RSA 69 | We recommend setting up SSH public-private key pair: see [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys) for more details. 70 | ::: 71 | 72 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/docker-build.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Azure ML Containers' 3 | description: Guide to containers in Azure ML. 4 | keywords: 5 | - containers 6 | - dockerfile 7 | - docker 8 | - environment 9 | --- 10 | 11 | In this post we explain how Azure ML builds the containers used to run your code. 12 | 13 | ## Dockerfile 14 | 15 | Each job in Azure ML runs with an associated `Environment`. In practice, each environment 16 | corresponds to a Docker image. 17 | 18 | There are numerous ways to define an environment - from specifying a set of required Python packages 19 | through to directly providing a custom Docker image. In each case the contents of the associated 20 | dockerfile are available directly from the environment object. 21 | 22 | For more background: [Environment](environment) 23 | 24 | #### Example 25 | 26 | Suppose you create an environment - in this example we will work with Conda: 27 | 28 | ```yml title="env.yml" 29 | name: pytorch 30 | channels: 31 | - defaults 32 | - pytorch 33 | dependencies: 34 | - python=3.7 35 | - pytorch 36 | - torchvision 37 | ``` 38 | 39 | We can create and register this as an `Environment` in our workspace `ws` as follows: 40 | 41 | ```python 42 | from azureml.core import Environment 43 | env = Environment.from_conda_specification('pytorch', 'env.yml') 44 | env.register(ws) 45 | ``` 46 | 47 | In order to consume this environment in a remote run, Azure ML builds a docker image 48 | that creates the corresponding python environment. 49 | 50 | The dockerfile used to build this image is available directly from the environment object. 51 | 52 | ```python 53 | details = env.get_image_details(ws) 54 | print(details['ingredients']['dockerfile']) 55 | ``` 56 | 57 | Let's take a look: 58 | 59 | ```docker title="Dockerfile" {1,7-12} 60 | FROM mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1@sha256:8cee6f674276dddb23068d2710da7f7f95b119412cc482675ac79ba45a4acf99 61 | USER root 62 | RUN mkdir -p $HOME/.cache 63 | WORKDIR / 64 | COPY azureml-environment-setup/99brokenproxy /etc/apt/apt.conf.d/ 65 | RUN if dpkg --compare-versions `conda --version | grep -oE '[^ ]+$'` lt 4.4.11; then conda install conda==4.4.11; fi 66 | COPY azureml-environment-setup/mutated_conda_dependencies.yml azureml-environment-setup/mutated_conda_dependencies.yml 67 | RUN ldconfig /usr/local/cuda/lib64/stubs && conda env create -p /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6 - 68 | f azureml-environment-setup/mutated_conda_dependencies.yml && rm -rf "$HOME/.cache/pip" && conda clean -aqy && CONDA_ROO 69 | T_DIR=$(conda info --root) && rm -rf "$CONDA_ROOT_DIR/pkgs" && find "$CONDA_ROOT_DIR" -type d -name __pycache__ -exec rm 70 | -rf {} + && ldconfig 71 | # AzureML Conda environment name: azureml_7459a71437df47401c6a369f49fbbdb6 72 | ENV PATH /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6/bin:$PATH 73 | ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6 74 | ENV LD_LIBRARY_PATH /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6/lib:$LD_LIBRARY_PATH 75 | COPY azureml-environment-setup/spark_cache.py azureml-environment-setup/log4j.properties /azureml-environment-setup/ 76 | RUN if [ $SPARK_HOME ]; then /bin/bash -c '$SPARK_HOME/bin/spark-submit /azureml-environment-setup/spark_cache.py'; fi 77 | ENV AZUREML_ENVIRONMENT_IMAGE True 78 | CMD ["bash"] 79 | ``` 80 | 81 | Notice: 82 | 83 | - The base image here is a standard image maintained by Azure ML. Dockerfiles for all base images are available on 84 | github: https://github.com/Azure/AzureML-Containers 85 | - The dockerfile references `mutated_conda_dependencies.yml` to build the Python environment via Conda. 86 | 87 | Get the contents of `mutated_conda_dependencies.yml` from the environment: 88 | 89 | ```python 90 | print(env.python.conda_dependencies.serialize_to_string()) 91 | ``` 92 | 93 | Which looks like 94 | 95 | ```bash title="mutated_conda_dependencies.yml" 96 | channels: 97 | - defaults 98 | - pytorch 99 | dependencies: 100 | - python=3.7 101 | - pytorch 102 | - torchvision 103 | name: azureml_7459a71437df47401c6a369f49fbbdb6 104 | ``` -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/compute-target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/compute-target.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/create-compute-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/create-compute-1.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/create-compute-lp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/create-compute-lp.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/create-compute-lp.pngZone.Identifier: -------------------------------------------------------------------------------- 1 | [ZoneTransfer] 2 | LastWriterPackageFamilyName=Microsoft.ScreenSketch_8wekyb3d8bbwe 3 | ZoneId=3 4 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/create-compute-ssh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/create-compute-ssh.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/create-compute-ssh.pngZone.Identifier: -------------------------------------------------------------------------------- 1 | [ZoneTransfer] 2 | LastWriterPackageFamilyName=Microsoft.ScreenSketch_8wekyb3d8bbwe 3 | ZoneId=3 4 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/create-compute.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/create-compute.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/log-files.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/log-files.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/logging-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/logging-metrics.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/img/run-ex-sine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/cheatsheets/python/v1/img/run-ex-sine.png -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/installation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Installation 3 | description: Guide to installing Azure ML Python SDK and setting up key resources. 4 | keywords: 5 | - azureml-sdk 6 | - installation 7 | - workspace 8 | - compute 9 | - cpu 10 | - gpu 11 | --- 12 | 13 | Install the Azure ML Python SDK: 14 | 15 | ```console 16 | pip install azureml-sdk 17 | ``` 18 | 19 | ### Create Workspace 20 | 21 | Please prepare your Azure subscription, resource group, and decide your Workspace name in order to create `Workspace`. 22 | If you don't have subscription, please [generate it for free](https://azure.microsoft.com/en-us/free/). 23 | 24 | ```python 25 | from azureml.core import Workspace 26 | 27 | ws = Workspace.create(name='', # provide a name for your workspace 28 | subscription_id='', # provide your subscription ID 29 | resource_group='', # provide a resource group name 30 | create_resource_group=True, 31 | location='') # e.g. 'westeurope' or 'eastus2' or 'westus2' or 'southeastasia'. 32 | 33 | # write out the workspace details to a configuration file: .azureml/config.json 34 | ws.write_config(path='.azureml') 35 | ``` 36 | 37 | :::info 38 | You can easily access this later with 39 | ```python 40 | from azureml.core import Workspace 41 | ws = Workspace.from_config() 42 | ``` 43 | ::: 44 | 45 | ### Create Compute Target 46 | 47 | The following example creates a compute target in your workspace with: 48 | 49 | - VM type: CPU 50 | - VM size: STANDARD_D2_V2 51 | - Cluster size: up to 4 nodes 52 | - Idle time: 2400s before the node scales down automatically 53 | 54 | Modify this code to update to GPU, or to change the SKU of your VMs. 55 | 56 | ```python 57 | from azureml.core import Workspace 58 | from azureml.core.compute import ComputeTarget, AmlCompute 59 | from azureml.core.compute_target import ComputeTargetException 60 | 61 | ws = Workspace.from_config() # automatically looks for a directory .azureml/ 62 | 63 | # name for your cluster 64 | cpu_cluster_name = "cpu-cluster" 65 | 66 | try: 67 | # check if cluster already exists 68 | cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name) 69 | print('Found existing cluster, use it.') 70 | except ComputeTargetException: 71 | # if not, create it 72 | compute_config = AmlCompute.provisioning_configuration( 73 | vm_size='STANDARD_D2_V2', 74 | max_nodes=4, 75 | idle_seconds_before_scaledown=2400,) 76 | cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, compute_config) 77 | cpu_cluster.wait_for_completion(show_output=True) 78 | ``` 79 | 80 | :::info 81 | You can access this later with 82 | 83 | ```python 84 | from azureml.core import ComputeTarget 85 | cpu_cluster = ComputeTarget(ws, 'cpu-cluster') 86 | ``` 87 | ::: -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/logging.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Metrics 3 | description: Guide to metric logging in Azure ML. 4 | keywords: 5 | - metric 6 | - logging 7 | --- 8 | 9 | ## Logging metrics 10 | 11 | Logging a metric to a run causes that metric to be stored in the run record in the experiment. 12 | Visualize and keep a history of all logged metrics. 13 | 14 | 15 | ### `log` 16 | 17 | Log a single metric value to a run. 18 | 19 | ```python 20 | from azureml.core import Run 21 | run = Run.get_context() 22 | run.log('metric-name', metric_value) 23 | ``` 24 | 25 | You can log the same metric multiple times within a run; the results will be displayed as a chart. 26 | 27 | ### `log_row` 28 | 29 | Log a metric with multiple columns. 30 | 31 | ```python 32 | from azureml.core import Run 33 | run = Run.get_context() 34 | run.log_row("Y over X", x=1, y=0.4) 35 | ``` 36 | 37 | :::info More logging options 38 | These are probably the most common APIs used for logging metrics, but see [here](https://docs.microsoft.com/azure/machine-learning/how-to-log-view-metrics#data-types) for a complete 39 | list, including logging lists, tables and images. 40 | ::: 41 | 42 | ## Viewing metrics 43 | 44 | Metrics will be automatically available in the Azure ML Studio. Locate your run, e.g., either 45 | by visiting [ml.azure.com](https://ml.azure.com), or using the SDK: 46 | 47 | ``` 48 | run.get_workspace_url() 49 | ``` 50 | 51 | Select the "Metrics" tab and select the metric(s) to view: 52 | 53 | ![](/img/view-metrics.png) 54 | 55 | It is also possible to compare metrics between runs in a _summary_ view from the experiments page itself. Select the "Experiments" tab, add custom charts and select the runs you want to compare! 56 | 57 | ![](/img/custom-view.png) 58 | 59 | :::info Custom views 60 | Curate your view by adding charts, selecting key runs, updating the columns in the table. 61 | You can then save this view, create more, and even share them with others! 62 | ::: 63 | 64 | ### Via the SDK 65 | 66 | Viewing metrics in a run (for more details on runs: [Run](run)) 67 | 68 | ```python 69 | metrics = run.get_metrics() 70 | # metrics is of type Dict[str, List[float]] mapping mertic names 71 | # to a list of the values for that metric in the given run. 72 | 73 | metrics.get('metric-name') 74 | # list of metrics in the order they were recorded 75 | 76 | metrics_including_child_runs = run.get_metrics(recursive=True) 77 | # set 'recursive' boolean parameter to True to also get 78 | # the metrics for child runs 79 | ``` 80 | 81 | To view all recorded values for a given metric `my-metric` in a 82 | given experiment `my-experiment`: 83 | 84 | ```python 85 | experiments = ws.experiments 86 | # of type Dict[str, Experiment] mapping experiment names the 87 | # corresponding Experiment 88 | 89 | exp = experiments['my-experiment'] 90 | for run in exp.get_runs(): 91 | metrics = run.get_metrics() 92 | 93 | my_metric = metrics.get('my-metric') 94 | if my_metric: 95 | print(my_metric) 96 | ``` 97 | 98 | ## Examples 99 | 100 | ### Logging with MLFlow 101 | 102 | Use [MLFlow](https://mlflow.org/) to log metrics in Azure ML. 103 | 104 | ```python 105 | from azureml.core import Run 106 | 107 | # connect to the workspace from within your running code 108 | run = Run.get_context() 109 | ws = run.experiment.workspace 110 | 111 | # workspace has associated ml-flow-tracking-uri 112 | mlflow_url = ws.get_mlflow_tracking_uri() 113 | ``` 114 | 115 | ### Logging with PyTorch Lightning 116 | 117 | This examples: 118 | - Includes Lightning's `TensorBoardLogger` 119 | - Sets up Lightning's `MLFlowLogger` using AzureML `Run.get_context()` 120 | - Only adds this logger when used as part of an Azure ML run 121 | 122 | ```python 123 | import pytorch_lightning as pl 124 | 125 | run = None 126 | try: 127 | from azureml.core.run import Run, _OfflineRun 128 | run = Run.get_context() 129 | if isinstance(run, _OfflineRun): 130 | run = None 131 | except ImportError: 132 | print("Couldn't import azureml.core.run.Run") 133 | 134 | def get_logger(): 135 | tb_logger = pl.loggers.TensorBoardLogger('logs/') 136 | logger = [tb_logger] 137 | 138 | if run is not None: 139 | mlflow_url = run.experiment.workspace.get_mlflow_tracking_uri() 140 | mlf_logger = pl.loggers.MLFlowLogger( 141 | experiment_name=run.experiment.name, 142 | tracking_uri=mlflow_url, 143 | ) 144 | mlf_logger._run_id = run.id 145 | logger.append(mlf_logger) 146 | 147 | return logger 148 | ``` 149 | 150 | Now include this logger in the lightning `Trainer` class: 151 | 152 | ```python 153 | logger = get_logger() 154 | 155 | trainer = pl.Trainer.from_argparse_args( 156 | args=args, 157 | logger=logger, 158 | ) 159 | trainer.fit(model) 160 | ``` -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/run-history.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Run History 3 | --- 4 | 5 | Azure ML can supercharge your ML workloads in (at least!) two ways: 6 | 7 | - AML Compute: Providing powerful compute resoures to train larger models 8 | - Run history: Best-in-class lineage and reproducability 9 | 10 | In this article we focus on Run History - and why you need it in your life! 11 | 12 | As teams progress to running dozens, and eventually hundreds of experiments, having 13 | some way to organize them is essential. Run History is a service that provides a number 14 | features that quickly become essential to your ML-model builders: 15 | 16 | ### Experiments and Runs 17 | 18 | When you are running dozens of experiments in multiple different projects, having a clear 19 | way to organize and search though the results is key. Azure ML provides two concepts to help 20 | with this: `Run`s and `Experiment`s. 21 | 22 | #### Runs 23 | A run is a single execution of your code - usually a training script. The run has a life-cycle: 24 | the code is prepared to be submited to Azure ML (e.g. via a ScriptRunConfig), then the code is 25 | submitted 26 | 27 | Once the code is submitted to Azure ML (for example, via a `ScriptRunConfig`) a `Run` object is 28 | created. This compute target is prepared (nodes are provisioned, containers hosting your Python 29 | environment are fired up), the entry point script is called (`$ python run.py [args]`) and logs 30 | start being generated: 31 | 32 | ```console 33 | Files already downloaded and verified 34 | epoch=1, batch= 2000: loss 2.19 35 | epoch=1, batch= 4000: loss 1.82 36 | epoch=1, batch= 6000: loss 1.66 37 | ... 38 | ``` 39 | 40 | You may log metrics to Azure ML with `run.log('', metric_value)` and monitor them in the studio: 41 | 42 | ![](img/logging-metrics.png) 43 | 44 | The training concludes, usually some model files are saved, and the nodes are 45 | released. 46 | 47 | But the story doesn't end there. The run persists even after the nodes are returned 48 | to Azure. You can always return, either in code or via the studio, to see a history 49 | of your runs, all their outputs and metrics, and the exact code that was used to generate them. 50 | 51 | #### Experiments 52 | 53 | An Experiment is a collection of runs. All runs belongs to an Experiment. Usually 54 | an Experiment is tied to a specific work item, for example, "Finetune Bert-Large", 55 | and will possess a number of runs as you iterate toward this goal. 56 | 57 | ### Snapshot 58 | 59 | When you submit your code to run in Azure ML, a _snapshot_ is taken. This is a copy of the exact 60 | code that ran. Think of this as version control for your experiments. Want to reproduce the 61 | results from that experiment 2-months ago even though you've iterated on the model and the 62 | training script in the meantime? No problem, snapshot has you covered! 63 | 64 | You have total control of what goes into the snapshot with the `.amlignore` file. This plays 65 | the same role as a `.gitignore` so you can efficiently manage what to include in the snapshot. 66 | 67 | ### Metrics 68 | 69 | As you run experiments, you track metrics - from validation loss through to GPU load. Analysing these metrics is essential to determining your best model. With Run History, these metrics are stored for all your runs. -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/run.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Experiment and Run 3 | description: Guide to running code with Azure ML 4 | keywords: 5 | - run 6 | - experiment 7 | - submit 8 | - remote 9 | - ScriptRunConfig 10 | --- 11 | 12 | ## Concepts 13 | 14 | ### Run 15 | 16 | A run represents a single execution of your code. 17 | 18 | Azure ML is a machine-learning service that facilitates running your code in 19 | the cloud. A `Run` is an abstraction layer around each such submission, and is used to 20 | monitor the job in real time as well as keep a history of your results. 21 | 22 | ### Experiments 23 | 24 | An experiment is a light-weight container for `Run`. Use experiments to submit 25 | and track runs. 26 | 27 | Create an experiment in your workspace `ws`. 28 | 29 | ```python 30 | from azureml.core import Experiment 31 | exp = Experiment(ws, '') 32 | ``` 33 | 34 | ## Create Run 35 | 36 | ### Via ScriptRunConfig 37 | 38 | Usually a run is created by submitting a ScriptRunConfig. 39 | 40 | ```python 41 | from azureml.core import Workspace, Experiment, ScriptRunConfig 42 | ws = Workspace.from_config() 43 | exp = Experiment(ws, '') 44 | 45 | config = ScriptRunConfig(source_directory=<''>, script='train.py', ...) 46 | run = exp.submit(config) 47 | ``` 48 | 49 | For more details: [ScriptRunConfig](script-run-config) 50 | 51 | ### Get Context 52 | 53 | Code that is running within Azure ML is associated to a `Run`. The submitted code 54 | can access its own run. 55 | 56 | ```py 57 | from azureml.core import Run 58 | run = Run.get_context() 59 | ``` 60 | 61 | #### Example: Logging metrics to current run context 62 | 63 | A common use-case is logging metrics in a training script. 64 | 65 | ```py title="train.py" 66 | from azureml.core import Run 67 | 68 | run = Run.get_context() 69 | 70 | # training code 71 | for epoch in range(n_epochs): 72 | model.train() 73 | ... 74 | val = model.evaluate() 75 | run.log('validation', val) 76 | ``` 77 | 78 | When this code is submitted to Azure ML (e.g. via ScriptRunConfig) it will log metrics to its assocaited run. 79 | 80 | For more details: [Logging Metrics](logging) 81 | 82 | ### Interactive 83 | 84 | In an interactive setting e.g. a Jupyter notebook 85 | 86 | ```python 87 | run = exp.start_logging() 88 | ``` 89 | 90 | #### Example: Jupyter notebook 91 | 92 | A common use case for interacive logging is to train a model in a notebook. 93 | 94 | ```py 95 | from azureml.core import Workspace 96 | from azureml.core import Experiment 97 | ws = Workspace.from_config() 98 | exp = Experiment(ws, 'example') 99 | 100 | run = exp.start_logging() # start interactive run 101 | print(run.get_portal_url()) # get link to studio 102 | 103 | # toy example in place of e.g. model 104 | # training or exploratory data analysis 105 | import numpy as np 106 | for x in np.linspace(0, 10): 107 | y = np.sin(x) 108 | run.log_row('sine', x=x, y=y) # log metrics 109 | 110 | run.complete() # stop interactive run 111 | ``` 112 | 113 | Follow the link to the run to see the metric logging in real time. 114 | 115 | ![](img/run-ex-sine.png) -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/script-run-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Running Code in the Cloud 3 | description: Guide to running code with Azure ML 4 | keywords: 5 | - run 6 | - experiment 7 | - submit 8 | - remote 9 | - ScriptRunConfig 10 | --- 11 | 12 | ## Experiments and Runs 13 | 14 | Azure ML is a machine-learning service that facilitates running your code in 15 | the cloud. A `Run` is an abstraction layer around each such submission, and is used to 16 | monitor the job in real time as well as keep a history of your results. 17 | 18 | - Run: A run represents a single execution of your code. See [Run](#run) for more details. 19 | - Experiments: An experiment is a light-weight container for `Run`. Use experiments to submit 20 | and track runs. 21 | 22 | Create an experiment in your workspace `ws`. 23 | 24 | ```python 25 | from azureml.core import Experiment 26 | exp = Experiment(ws, '') 27 | ``` 28 | 29 | ## ScriptRunConfig 30 | 31 | A common way to run code in the cloud is via the `ScriptRunConfig` which packages 32 | your source code (Script) and run configuration (RunConfig). 33 | 34 | Consider the following layout for your code. 35 | 36 | ```bash 37 | source_directory/ 38 | script.py # entry point to your code 39 | module1.py # modules called by script.py 40 | ... 41 | ``` 42 | 43 | To run `script.py` in the cloud via the `ScriptRunConfig` 44 | 45 | ```python 46 | config = ScriptRunConfig( 47 | source_directory='', 48 | script='script.py', 49 | compute_target=target, 50 | environment=env, 51 | arguments = [ 52 | '--learning_rate', 0.001, 53 | '--momentum', 0.9, 54 | ] 55 | ) 56 | ``` 57 | 58 | where: 59 | 60 | - `source_directory='source_directory'` : Local directory with your code. 61 | - `script='script.py'` : Script to run. This does not need to be at the root of `source_directory`. 62 | - `compute_taget=target` : See [Compute Target](copute-target) 63 | - `environment` : See [Environment](environment) 64 | - `arguments` : See [Arguments](#command-line-arguments) 65 | 66 | Submit this code to Azure with 67 | 68 | ```python 69 | exp = Experiment(ws, '') 70 | run = exp.submit(config) 71 | print(run) 72 | run.wait_for_completion(show_output=True) 73 | ``` 74 | 75 | This will present you with a link to monitor your run on the web (https://ml.azure.com) 76 | as well as streaming logs to your terminal. 77 | 78 | ## Command Line Arguments 79 | 80 | To pass command line arguments to your script use the `arguments` parameter in `ScriptRunConfig`. 81 | Arguments are specified as a list: 82 | 83 | ```python 84 | arguments = [first, second, third, ...] 85 | ``` 86 | 87 | which are then passed to the script as command-line arguments as follows: 88 | 89 | ```console 90 | $ python script.py first second third ... 91 | ``` 92 | 93 | This also supports using named arguments: 94 | 95 | ```python 96 | arguments = ['--first_arg', first_val, '--second_arg', second_val, ...] 97 | ``` 98 | 99 | Arguments can be of type `int`, `float` `str` and can also be used to reference data. 100 | 101 | For more details on referencing data via the command line: [Use dataset in a remote run](dataset#use-dataset-in-a-remote-run) 102 | 103 | ### Example: `sys.argv` 104 | 105 | In this example we pass two arguments to our script. If we were running this from the 106 | console: 107 | 108 | ```console title="console" 109 | $ python script.py 0.001 0.9 110 | ``` 111 | 112 | To mimic this command using `argument` in `ScriptRunConfig`: 113 | 114 | ```python title="run.py" 115 | arguments = [0.001, 0.9] 116 | 117 | config = ScriptRunConfig( 118 | source_directory='.', 119 | script='script.py', 120 | arguments=arguments, 121 | ) 122 | ``` 123 | 124 | which can be consumed as usual in our script: 125 | 126 | ```python title="script.py" 127 | import sys 128 | learning_rate = sys.argv[1] # gets 0.001 129 | momentum = sys.argv[2] # gets 0.9 130 | ``` 131 | 132 | ### Example: `argparse` 133 | 134 | In this example we pass two named arguments to our script. If we were running this from the 135 | console: 136 | 137 | ```console title="console" 138 | $ python script.py --learning_rate 0.001 --momentum 0.9 139 | ``` 140 | 141 | To mimic this behavior in `ScriptRunConfig`: 142 | 143 | ```python title="run.py" 144 | arguments = [ 145 | '--learning_rate', 0.001, 146 | '--momentum', 0.9, 147 | ] 148 | 149 | config = ScriptRunConfig( 150 | source_directory='.', 151 | script='script.py', 152 | arguments=arguments, 153 | ) 154 | ``` 155 | 156 | which can be consumed as usual in our script: 157 | 158 | ```python title="script.py" 159 | import argparse 160 | parser = argparse.Argparser() 161 | parser.add_argument('--learning_rate', type=float) 162 | parser.add_argument('--momentum', type=float) 163 | args = parser.parse_args() 164 | 165 | learning_rate = args.learning_rate # gets 0.001 166 | momentum = args.momentum # gets 0.9 167 | ``` 168 | 169 | ## Commands 170 | 171 | It is possible to provide the explicit command to run. 172 | 173 | ```python 174 | command = 'python script.py'.split() 175 | 176 | config = ScriptRunConfig( 177 | source_directory='', 178 | command=command, 179 | compute_target=compute_target, 180 | environment=environment, 181 | ) 182 | ``` 183 | 184 | This example is equivalent to setting the argument `script='script.py'` in place of 185 | the `command` argument. 186 | 187 | This option provides a lot of flexibility. For example: 188 | 189 | 190 | - **Set environment variables**: Some useful examples: 191 | 192 | ```python 193 | command = 'export PYTHONPATH=$PWD && python script.py'.split() 194 | ``` 195 | 196 | ```python 197 | command = f'export RANK={rank} && python script.py'.split() 198 | ``` 199 | 200 | - **Run setup script**: Run a setup script e.g. to download data, set environment variables. 201 | 202 | ```python 203 | command = 'python setup.py && python script.py'.split() 204 | ``` 205 | 206 | ## Using Datasets 207 | 208 | ### via Arguments 209 | 210 | Pass a dataset to your ScriptRunConfig as an argument 211 | 212 | ```py 213 | # create dataset 214 | datastore = ws.get_default_datastore() 215 | dataset = Dataset.File.from_files(path=(datastore, '')) 216 | 217 | arguments = ['--dataset', dataset.as_mount()] 218 | 219 | config = ScriptRunConfig( 220 | source_directory='.', 221 | script='script.py', 222 | arguments=arguments, 223 | ) 224 | ``` 225 | 226 | This mounts the dataset to the run where it can be referenced by `script.py`. 227 | 228 | ## Run 229 | 230 | ### Interactive 231 | 232 | In an interactive setting e.g. a Jupyter notebook 233 | 234 | ```python 235 | run = exp.start_logging() 236 | ``` 237 | 238 | #### Example: Jupyter notebook 239 | 240 | A common use case for interactive logging is to train a model in a notebook. 241 | 242 | ```py 243 | from azureml.core import Workspace 244 | from azureml.core import Experiment 245 | ws = Workspace.from_config() 246 | exp = Experiment(ws, 'example') 247 | 248 | run = exp.start_logging() # start interactive run 249 | print(run.get_portal_url()) # get link to studio 250 | 251 | # toy example in place of e.g. model 252 | # training or exploratory data analysis 253 | import numpy as np 254 | for x in np.linspace(0, 10): 255 | y = np.sin(x) 256 | run.log_row('sine', x=x, y=y) # log metrics 257 | 258 | run.complete() # stop interactive run 259 | ``` 260 | 261 | Follow the link to the run to see the metric logging in real time. 262 | 263 | ![](img/run-ex-sine.png) 264 | 265 | ### Get Context 266 | 267 | Code that is running within Azure ML is associated to a `Run`. The submitted code 268 | can access its own run. 269 | 270 | ```py 271 | from azureml.core import Run 272 | run = Run.get_context() 273 | ``` 274 | 275 | #### Example: Logging metrics to current run context 276 | 277 | A common use-case is logging metrics in a training script. 278 | 279 | ```py title="train.py" 280 | from azureml.core import Run 281 | 282 | run = Run.get_context() 283 | 284 | # training code 285 | for epoch in range(n_epochs): 286 | model.train() 287 | ... 288 | val = model.evaluate() 289 | run.log('validation', val) 290 | ``` 291 | 292 | When this code is submitted to Azure ML (e.g. via ScriptRunConfig) it will log metrics to its associated run. 293 | 294 | For more details: [Logging Metrics](logging) -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/templates.md: -------------------------------------------------------------------------------- 1 | # Templates 2 | 3 | ## Introduction 4 | 5 | Cookiecutter is a simple command-line tool that allows you to quickly create 6 | new projects from pre-defined templates. Let's see it in action! 7 | 8 | First go ahead and get cookiecutter using your environment manager of choice, 9 | for example: 10 | 11 | ```bash 12 | pip install cookiecutter 13 | ``` 14 | 15 | Then give this repo a home 16 | 17 | ```bash 18 | cd ~/repos # or wherever your repos call home :-) 19 | git clone 20 | ``` 21 | 22 | To create a new project from the `ScriptRunConfig` template for example, simply 23 | run 24 | 25 | ```bash 26 | cookiecutter path/to/cheatsheet/repo/templates/ScriptRunConfig 27 | ``` 28 | 29 | See [ScriptRunConfig](#ScriptRunConfig) for more details for this template. 30 | 31 | ## Project Templates 32 | 33 | - ScriptRunConfig: Create a project to run a script in AML making use of the 34 | ScriptRunConfig class. This template is well suited for smaller projects and 35 | is especially handy for testing. 36 | 37 | ### ScriptRunConfig 38 | 39 | [Cookiecutter](https://cookiecutter.readthedocs.io/en/1.7.2/README.html) 40 | template for setting up an AML 41 | [ScriptRunConfig](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.scriptrunconfig?view=azure-ml-py) 42 | used to run your script in Azure. 43 | 44 | #### Usage 45 | 46 | Run the cookiecutter command 47 | 48 | ```bash 49 | cookiecutter /ScriptRunConfig 50 | ``` 51 | 52 | to create a new `ScriptRunConfig` project. 53 | 54 | **Note.** Install with `pip install cookiecutter` (see 55 | [cookiecutter docs](https://cookiecutter.readthedocs.io/en/1.7.2/installation.html) 56 | for more installation options) 57 | 58 | You will be prompted for the following: 59 | 60 | - `directory_name`: The desired name of the directory (default: 61 | "aml-src-script") 62 | - `script_name`: The name of the python script to be run in Azure (default: 63 | "script") 64 | - `subscription_id`: Your Azure Subscription ID 65 | - `resource_group`: Your Azure resource group name 66 | - `workspace_name`: Your Azure ML workspace name 67 | - `compute_target_name`: The name of the Azure ML compute target to run the 68 | script on (default: "local", will run on your box) 69 | 70 | Cookiecutter creates a new project with the following layout. 71 | 72 | ```bash 73 | {directory_name}/ 74 | {script_name}.py # the script you want to run in the cloud 75 | run.py # wraps your script in ScriptRunConfig to send to Azure 76 | config.json # your Azure ML metadata 77 | readme.md # this readme file! 78 | ``` 79 | 80 | See 81 | [ScriptRunConfig](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.scriptrunconfig?view=azure-ml-py) 82 | for more options and details on configuring runs. 83 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/troubleshooting.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Troubleshooting 3 | id: troubleshooting 4 | description: A cheat sheet for Azure ML. 5 | keywords: 6 | - azure machine learning 7 | - aml 8 | - troubleshooting 9 | --- 10 | 11 | ### Error: az acr login- APIVersion 2020-11-01-preview is not available. 12 | **Description** 13 | NotImplementedError occurred when building image using az acr. 14 | ```bash 15 | az acr build --image $image_name --subscription $ws.subscription_id --registry $cr --file docker/Dockerfile docker/ 16 | ``` 17 | The error: 18 | ```text 19 | NotImplementedError: APIVersion 2020-11-01-preview is not available. 20 | ``` 21 | 22 | **Solution** This is a problem related with the version of az cli. Please update az cli by running 23 | ```bash 24 | az upgrade --yes 25 | ``` 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v1/workspace.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Workspace 3 | description: Overview of Azure ML Workspaces. 4 | keywords: 5 | - workspace 6 | --- 7 | 8 | Workspaces are a foundational object used throughout Azure ML and are used in the 9 | constructors of many other classes. Throughout this documentation we frequently 10 | omit the workspace object instantiation and simply refer to `ws`. 11 | 12 | See [Installation](installation) for instructions on creating a new workspace. 13 | 14 | ## Get workspace 15 | 16 | Instantiate `Workspace` object used to connect to your AML assets. 17 | 18 | ```python title="run.py" 19 | from azureml.core import Workspace 20 | ws = Workspace( 21 | subscription_id="", 22 | resource_group="", 23 | workspace_name="", 24 | ) 25 | ``` 26 | 27 | For convenience store your workspace metadata in a `config.json`. 28 | 29 | ```json title=".azureml/config.json" 30 | { 31 | "subscription_id": , 32 | "resource_group": , 33 | "workspace_name": 34 | } 35 | ``` 36 | 37 | ### Helpful methods 38 | 39 | - `ws.write_config(path, file_name)` : Write the `config.json` on your behalf. The `path` defaults to '.azureml/' in the current working directory and `file_name` defaults to 'config.json'. 40 | - `Workspace.from_config(path, _file_name)`: Read the workspace configuration from config. The parameter defaults to starting the search in the current directory. 41 | 42 | :::info 43 | It is recommended to store these in a directory `.azureml/` as this path is searched _by default_ 44 | in the `Workspace.from_config` method. 45 | ::: 46 | 47 | ## Get Workspace Assets 48 | 49 | The workspace provides a handle to your Azure ML assets: 50 | 51 | ### Compute Targets 52 | 53 | Get all compute targets attached to the workspace. 54 | 55 | ```python 56 | ws.compute_targets: Dict[str, ComputeTarget] 57 | ``` 58 | 59 | ### Datastores 60 | 61 | Get all datastores registered to the workspace. 62 | 63 | ```python 64 | ws.datastores: Dict[str, Datastore] 65 | ``` 66 | 67 | Get the workspace's default datastore. 68 | 69 | ```python 70 | ws.get_default_datastore(): Datastore 71 | ``` 72 | 73 | ### Keyvault 74 | 75 | Get workspace's default Keyvault. 76 | 77 | ```python 78 | ws.get_default_keyvault(): Keyvault 79 | ``` 80 | 81 | ### Environments 82 | 83 | Get environments registered to the workspace. 84 | 85 | ```python 86 | ws.environments: Dict[str, Environment] 87 | ``` 88 | 89 | ### MLFlow 90 | 91 | Get MLFlow tracking uri. 92 | 93 | ```python 94 | ws.get_mlflow_tracking_uri(): str 95 | ``` -------------------------------------------------------------------------------- /website/docs/cheatsheets/python/v2/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /website/docs/misc/contributing.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Contributing 3 | description: Guide to contributing. 4 | --- 5 | 6 | ## Issues 7 | 8 | All forms of feedback are welcome through [issues](https://github.com/Azure/azureml-cheatsheets/issues) - please follow the pre-defined templates where applicable. 9 | 10 | ## Pull requests 11 | 12 | Pull requests (PRs) to this repo require review and approval by the Azure Machine Learning team to merge. Please follow the pre-defined template and read all relevant sections below. 13 | 14 | Make PR's against the `main` branch. 15 | 16 | ```bash 17 | git clone git@github.com:Azure/azureml-cheatsheets.git 18 | cd azureml-cheatsheets 19 | git checkout -b user/contrib 20 | ... 21 | gh pr create 22 | ``` 23 | 24 | - When a PR arrives against `main` GitHub actions (deploy) will test the build is successful 25 | - When the PR is merged the change will be automatically deployed to `gh-pages` branch (and the webpage will be updated) 26 | 27 | 99% of contributions should only need the following: 28 | 29 | - Add markdown files to the `website/docs/cheatsheets` folder 30 | - Update the `sidebar.js` file to add a page to the sidebar 31 | - Put any images in `website/docs/cheatsheets//img/` and refer to them like this: `![](img/.png)` 32 | 33 | If you need to do anything more than adding a new page to the sidebar (e.g. 34 | modify the nav bar) then please refer to the [Docusaurus 2 documentation](https://v2.docusaurus.io/). 35 | 36 | ## Previewing changes locally 37 | 38 | - Install [npm](https://nodejs.org/en/download/) and [yarn](https://classic.yarnpkg.com/en/docs/install#windows-stable). 39 | 40 | - Initial Docusaurus installation: (**First time only**) 41 | 42 | ```bash 43 | cd website 44 | npm install 45 | ``` 46 | 47 | 48 | - Run local server while developing: 49 | 50 | ```bash 51 | cd website 52 | yarn start 53 | ``` 54 | 55 | See Docusaurus instructions [here](https://v2.docusaurus.io/docs/installation) for more details. -------------------------------------------------------------------------------- /website/docs/misc/deployment.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Deployment 3 | description: One-time website deployment setup. 4 | --- 5 | 6 | ## Deployment 7 | 8 | This article describes the one-time process for deploying the cheat sheeets as a GitHub Pages website. 9 | 10 | This repo has GitHub actions in place that automate deployment by watching the `main` branch. 11 | If you are interested in how deployment works then read on! 12 | 13 | ### GitHub Actions 14 | 15 | We use GitHub actions to automate deployment. Set up was as follows: 16 | 17 | - Generated new SSH key 18 | - NB. Since there was an existing ssh key tied the repo a new key was generated (in a different location) `/tmp/.ssh/id_rsa` 19 | - Add public key to repo's [deploy key](https://developer.github.com/v3/guides/managing-deploy-keys/) 20 | - NB. Allow write access 21 | - Add private key as [GitHub secret](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets) 22 | - We use repo-level (not org level) secret 23 | - Secret is named `GH_PAGES_DEPLOY` 24 | - `xclip -sel clip < /tmp/.ssh/id_rsa` 25 | 26 | ### Manual 27 | 28 | It is possible to make manual deployments without use of the GitHub action above. 29 | 30 | ```console 31 | GIT_USER= USE_SSH=true yarn deploy 32 | ``` 33 | 34 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 35 | -------------------------------------------------------------------------------- /website/docs/misc/issues.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Issue Triage Process 3 | id: issues 4 | description: GitHub issue triage process for Azure Machine Learning. 5 | keywords: 6 | - azure machine learning 7 | - aml 8 | - azure 9 | --- 10 | 11 | ## Overview 12 | 13 | This page defines the triage process for Azure Machine Learning (AML) repositories. 14 | 15 | ## Repositories 16 | 17 | AML examples: 18 | 19 | - https://github.com/Azure/MachineLearningNotebooks 20 | - https://github.com/Azure/azureml-examples 21 | 22 | Azure core: 23 | 24 | - https://github.com/Azure/azure-cli 25 | - https://github.com/Azure/azure-cli-extensions 26 | - https://github.com/Azure/azure-powershell 27 | - https://github.com/Azure/azure-rest-api-specs 28 | - https://github.com/Azure/azure-sdk-for-js 29 | - https://github.com/Azure/azure-sdk-for-python 30 | 31 | > To request a repository to be added, [open an issue](https://github.com/Azure/azureml-web/issues) 32 | 33 | ## Code of Conduct 34 | 35 | All interactions on GitHub must follow the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 36 | 37 | ## Priority 38 | 39 | GitHub user experience. 40 | 41 | ## Metrics 42 | 43 | - FQR: first quality response 44 | - TTC: time to close 45 | 46 | ## Goals 47 | 48 | - triage issue area and type in <3 hrs 49 | - FQR <8 hrs 50 | - TTC for questions <5 days 51 | - TTC for bugs <30 days 52 | 53 | ## SLA 54 | 55 | - triage <1 day 56 | - FQR <3 days 57 | 58 | ## Labels 59 | 60 | ### Areas 61 | 62 | #### Foundations 63 | 64 | - `Foundations/Data` 65 | - `Foundations/Compute` 66 | - `Foundations/Infrastructure` 67 | - `Foundations/Admin` 68 | 69 | #### Experiences 70 | 71 | - `Experiences/UI` 72 | - `Experiences/Lifecycle` 73 | - `Experiences/Intelligence` 74 | - `Experiences/Inference` 75 | 76 | #### Pipelines 77 | 78 | - `Pipelines/UI` 79 | - `Pipelines/Aether` 80 | 81 | ### Issue types 82 | 83 | - `bug` 84 | - `question` 85 | - `feature-request` 86 | 87 | ### Other 88 | 89 | - `needs-details`: additional details needed from author 90 | - `v2`: planned for AMLv2 91 | 92 | ## Process 93 | 94 | ### Triage 95 | 96 | Initial triage will be performed by the GitHub v-team. On initial triage, assign the correct area label and issue type. 97 | 98 | If the issue needs obvious clarification before this can be done, kindly ask the user. If the issue has no path to closing without user response, mark it as `needs-details`. 99 | 100 | After initial triage, it is up to each area (Experiences, Foundations, Pipelines) to further triage as necessary to the correct engineering team members. 101 | 102 | One type of issue may be changed to another, i.e. for an issue like “can I do X” could end up as a feature request for X. Simply change the issue labels as appropriate. In some cases, it might make sense to open a new issue and close the original instead of changing the label. 103 | 104 | Once the issue is understood, it is up to each area to appropriately route through internal tools such as ADO, maintaining the GitHub issue as the point of communication with the user. Major developments should be communicated back to the user. 105 | 106 | ### Closing 107 | 108 | Issues may be closed by their creator at anytime, which is preferred, **especially for questions**. 109 | 110 | Additionally, issues may be closed once: 111 | 112 | - `needs-details`: user/author has not responded for 5+ days with no other path to closure 113 | - `question`: the question has been thoroughly answered with relevant links, documentation, and examples and has no follow-up questions from user(s) in 48 hrs 114 | - `bug`: the bug fix has been released, tested, and the user confirms the solution or does not respond for 48 hrs after being made aware of the fix 115 | - `feature-request`: the feature has been released, tested, and the user confirms the solution or does not respond for 48 hrs after being made aware of the release 116 | -------------------------------------------------------------------------------- /website/docs/vs-code-snippets/readme.md: -------------------------------------------------------------------------------- 1 | # VS Code Snippets 2 | 3 | Notes for contributing Azure ML Snippets. 4 | 5 | For using snippets see https://azure.github.io/azureml-web/docs/vs-code-snippets/snippets. 6 | 7 | 1. Add snippets to `python.json`. For more details on VS Code snippets: [vs-code-docs](https://code.visualstudio.com/docs/editor/userdefinedsnippets) 8 | 2. Run `python snippets-parser.py` to automatically update the `snippets.md` (which will document your changes) 9 | 3. Make a PR to the `main` branch and request a review. 10 | -------------------------------------------------------------------------------- /website/docs/vs-code-snippets/snippets-parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List 3 | 4 | 5 | class Snippet: 6 | """Handle json snippets 7 | 8 | Parse json (VS Code) snippets file and generate markdown summary. 9 | """ 10 | 11 | def __init__(self, name, snippet_json): 12 | self.name = name 13 | self.description = snippet_json.get("description") 14 | self.prefix = self._read_prefix(snippet_json.get("prefix")) 15 | self.body = snippet_json.get("body") 16 | 17 | def __repr__(self): 18 | return f"Snippet({self.name})" 19 | 20 | @staticmethod 21 | def _read_prefix(prefix): 22 | """Guarentee prefix is of type List.""" 23 | if type(prefix) == list: 24 | return prefix 25 | else: 26 | assert type(prefix) == str 27 | return [prefix] 28 | 29 | def to_markdown(self) -> List[str]: 30 | """Convert snippet to markdown (as list of lines).""" 31 | lines = [] 32 | 33 | # add heading 34 | heading = f"### {self.name}" 35 | lines.append(heading) 36 | lines.append("") 37 | 38 | # add description 39 | description = f"Description: {self.description}" 40 | lines.append(description) 41 | lines.append("") 42 | 43 | # add prefix(es) 44 | if len(self.prefix) > 1: 45 | prefix = f"Prefixes: " 46 | else: 47 | prefix = f"Prefix: " 48 | for p in self.prefix: 49 | prefix += f"`{p}`, " 50 | prefix = prefix[:-2] # remove trailing comma and whitespace 51 | lines.append(prefix) 52 | lines.append("") 53 | 54 | # add python snippet 55 | lines.append("```python") 56 | for line in self.body: 57 | if line == "$0": 58 | continue 59 | lines.append(line) 60 | lines.append("```") 61 | 62 | return lines 63 | 64 | @staticmethod 65 | def _convert_to_json(body): 66 | json_body = [] 67 | for line in body[:-1]: 68 | line = '"' + line + '",' 69 | json_body.append(line) 70 | line = '"' + body[-1] + '"' 71 | json_body.append(line) 72 | return json_body 73 | 74 | 75 | frontmatter = """--- 76 | title: VS Code Snippets 77 | description: A collection of VS Code Snippets for working with Azure ML. 78 | --- 79 | 80 | We have compiled a collection of useful templates in the form of 81 | [VS code snippets](https://code.visualstudio.com/docs/editor/userdefinedsnippets). 82 | 83 | ![VS Code Snippets](vs-code-snippets-demo.gif) 84 | 85 | To add these snippets to your VS Code: `ctrl+shift+p` > Type 'Configure user 86 | snippets' > Select `python.json`. All of these snippets are available here: 87 | [python.json](https://github.com/Azure/azureml-web/blob/main/website/docs/vs-code-snippets/python.json) 88 | 89 | """ 90 | 91 | if __name__ == "__main__": 92 | 93 | # parse snippets 94 | with open("python.json") as f: 95 | snippets_file = json.load(f) 96 | 97 | snippets = [] 98 | for name, snippet_json in snippets_file.items(): 99 | snippet = Snippet(name, snippet_json) 100 | snippets.append(snippet) 101 | 102 | # create file and write frontmatter 103 | md_filename = "snippets.md" 104 | with open(md_filename, "w") as f: 105 | # write frontmatter 106 | f.writelines(frontmatter) 107 | 108 | # write each snippet 109 | for snippet in snippets: 110 | lines = snippet.to_markdown() 111 | for line in lines: 112 | f.write(line + "\n") 113 | -------------------------------------------------------------------------------- /website/docs/vs-code-snippets/vs-code-snippets-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/docs/vs-code-snippets/vs-code-snippets-demo.gif -------------------------------------------------------------------------------- /website/docusaurus.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | i18n: { 5 | defaultLocale: 'en', 6 | locales: ['en', 'ja'], 7 | }, 8 | title: 'Azure Machine Learning', 9 | tagline: 'Open source cheat sheets for Azure ML', 10 | url: 'https://github.com/Azure/', 11 | baseUrl: '/azureml-cheatsheets/', 12 | onBrokenLinks: 'ignore', 13 | favicon: 'img/logo.svg', 14 | organizationName: 'Azure', // Usually your GitHub org/user name. 15 | projectName: 'azureml-cheatsheets', // Usually your repo name. 16 | themeConfig: { 17 | agolia: { 18 | contextualSearch: true 19 | }, 20 | navbar: { 21 | title: 'Azure Machine Learning', 22 | logo: { 23 | alt: 'Logo', 24 | src: 'img/logo.svg', 25 | }, 26 | items: [ 27 | {to: '/docs/cheatsheets/python/v1/cheatsheet', label: 'Python SDK', position: 'left'}, 28 | { 29 | type: 'localeDropdown', 30 | position: 'left', 31 | }, 32 | ], 33 | }, 34 | footer: { 35 | style: 'dark', 36 | links: [ 37 | { 38 | title: 'Resources', 39 | items: [ 40 | { 41 | label: 'Azure ML - Microsoft Docs', 42 | href: 'https://docs.microsoft.com/azure/machine-learning', 43 | }, 44 | { 45 | label: 'Azure ML - Python API', 46 | href: 'https://docs.microsoft.com/python/api/overview/azure/ml/?view=azure-ml-py' 47 | } 48 | ], 49 | }, 50 | { 51 | title: 'Support', 52 | items: [ 53 | { 54 | label: 'GitHub issues', 55 | href: 'https://github.com/Azure/azureml-cheatsheets/issues', 56 | }, 57 | { 58 | label: 'Stack Overflow', 59 | href: 'https://stackoverflow.microsoft.com/questions/tagged/10888', 60 | } 61 | ], 62 | }, 63 | { 64 | title: 'GitHub', 65 | items: [ 66 | { 67 | label: 'Cheat sheets', 68 | href: 'https://github.com/Azure/azureml-cheatsheets', 69 | }, 70 | { 71 | label: 'Azure ML Examples', 72 | href: 'https://github.com/Azure/azureml-examples', 73 | }, 74 | { 75 | label: 'Contribution', 76 | href: '/docs/misc/contributing', 77 | }, 78 | ], 79 | }, 80 | ], 81 | copyright: `Copyright © ${new Date().getFullYear()} Microsoft Corporation`, 82 | }, 83 | algolia: { 84 | // search only api key: https://docsearch.algolia.com/docs/faq/#can-i-share-the-apikey-in-my-repo 85 | apiKey: '1e4db0054fd7da8ce9923b63900fa842', 86 | indexName: 'azureml-cheatsheets', 87 | searchParameters: {}, 88 | placeholder: 'Search cheat sheets' 89 | } 90 | }, 91 | plugins: [path.resolve(__dirname, 'plugins/appinsights')], // uncomment for appinsights 92 | presets: [ 93 | [ 94 | '@docusaurus/preset-classic', 95 | { 96 | docs: { 97 | sidebarPath: require.resolve('./sidebars.js'), 98 | // Please change this to your repo. 99 | editUrl: 100 | 'https://github.com/Azure/azureml-cheatsheets/tree/main/website/', 101 | }, 102 | theme: { 103 | customCss: require.resolve('./src/css/custom.css'), 104 | }, 105 | }, 106 | ], 107 | ], 108 | }; 109 | -------------------------------------------------------------------------------- /website/i18n/ja/code.json: -------------------------------------------------------------------------------- 1 | { 2 | "theme.NotFound.title": { 3 | "message": "ページが見つかりません", 4 | "description": "The title of the 404 page" 5 | }, 6 | "theme.NotFound.p1": { 7 | "message": "お探しのページが見つかりませんでした。", 8 | "description": "The first paragraph of the 404 page" 9 | }, 10 | "theme.NotFound.p2": { 11 | "message": "このページにリンクしているサイトの所有者に連絡をしてリンクが壊れていることを伝えてください。", 12 | "description": "The 2nd paragraph of the 404 page" 13 | }, 14 | "theme.AnnouncementBar.closeButtonAriaLabel": { 15 | "message": "閉じる", 16 | "description": "The ARIA label for close button of announcement bar" 17 | }, 18 | "theme.blog.paginator.navAriaLabel": { 19 | "message": "ブログ記事一覧のナビゲーション", 20 | "description": "The ARIA label for the blog pagination" 21 | }, 22 | "theme.blog.paginator.newerEntries": { 23 | "message": "新しい記事", 24 | "description": "The label used to navigate to the newer blog posts page (previous page)" 25 | }, 26 | "theme.blog.paginator.olderEntries": { 27 | "message": "過去の記事", 28 | "description": "The label used to navigate to the older blog posts page (next page)" 29 | }, 30 | "theme.blog.post.readingTime.plurals": { 31 | "message": "約{readingTime}分", 32 | "description": "Pluralized label for \"{readingTime} min read\". Use as much plural forms (separated by \"|\") as your language support (see https://www.unicode.org/cldr/cldr-aux/charts/34/supplemental/language_plural_rules.html)" 33 | }, 34 | "theme.tags.tagsListLabel": { 35 | "message": "タグ:", 36 | "description": "The label alongside a tag list" 37 | }, 38 | "theme.blog.post.readMore": { 39 | "message": "もっと見る", 40 | "description": "The label used in blog post item excerpts to link to full blog posts" 41 | }, 42 | "theme.blog.post.paginator.navAriaLabel": { 43 | "message": "ブログ記事のナビゲーション", 44 | "description": "The ARIA label for the blog posts pagination" 45 | }, 46 | "theme.blog.post.paginator.newerPost": { 47 | "message": "新しい記事", 48 | "description": "The blog post button label to navigate to the newer/previous post" 49 | }, 50 | "theme.blog.post.paginator.olderPost": { 51 | "message": "過去の記事", 52 | "description": "The blog post button label to navigate to the older/next post" 53 | }, 54 | "theme.tags.tagsPageTitle": { 55 | "message": "タグ", 56 | "description": "The title of the tag list page" 57 | }, 58 | "theme.blog.post.plurals": { 59 | "message": "{count}件", 60 | "description": "Pluralized label for \"{count} posts\". Use as much plural forms (separated by \"|\") as your language support (see https://www.unicode.org/cldr/cldr-aux/charts/34/supplemental/language_plural_rules.html)" 61 | }, 62 | "theme.blog.tagTitle": { 63 | "message": "「{tagName}」タグの記事が{nPosts}あります", 64 | "description": "The title of the page for a blog tag" 65 | }, 66 | "theme.tags.tagsPageLink": { 67 | "message": "全てのタグを見る", 68 | "description": "The label of the link targeting the tag list page" 69 | }, 70 | "theme.CodeBlock.copyButtonAriaLabel": { 71 | "message": "クリップボードにコードをコピー", 72 | "description": "The ARIA label for copy code blocks button" 73 | }, 74 | "theme.CodeBlock.copied": { 75 | "message": "コピーしました", 76 | "description": "The copied button label on code blocks" 77 | }, 78 | "theme.CodeBlock.copy": { 79 | "message": "コピー", 80 | "description": "The copy button label on code blocks" 81 | }, 82 | "theme.docs.sidebar.expandButtonTitle": { 83 | "message": "サイドバーを開く", 84 | "description": "The ARIA label and title attribute for expand button of doc sidebar" 85 | }, 86 | "theme.docs.sidebar.expandButtonAriaLabel": { 87 | "message": "サイドバーを開く", 88 | "description": "The ARIA label and title attribute for expand button of doc sidebar" 89 | }, 90 | "theme.docs.sidebar.collapseButtonTitle": { 91 | "message": "サイドバーを隠す", 92 | "description": "The title attribute for collapse button of doc sidebar" 93 | }, 94 | "theme.docs.sidebar.collapseButtonAriaLabel": { 95 | "message": "サイドバーを隠す", 96 | "description": "The title attribute for collapse button of doc sidebar" 97 | }, 98 | "theme.docs.sidebar.responsiveCloseButtonLabel": { 99 | "message": "メニューを閉じる", 100 | "description": "The ARIA label for close button of mobile doc sidebar" 101 | }, 102 | "theme.docs.sidebar.responsiveOpenButtonLabel": { 103 | "message": "メニューを開く", 104 | "description": "The ARIA label for open button of mobile doc sidebar" 105 | }, 106 | "theme.docs.paginator.navAriaLabel": { 107 | "message": "ドキュメントのナビゲーション", 108 | "description": "The ARIA label for the docs pagination" 109 | }, 110 | "theme.docs.paginator.previous": { 111 | "message": "前へ", 112 | "description": "The label used to navigate to the previous doc" 113 | }, 114 | "theme.docs.paginator.next": { 115 | "message": "次へ", 116 | "description": "The label used to navigate to the next doc" 117 | }, 118 | "theme.docs.versions.unreleasedVersionLabel": { 119 | "message": "これはリリース前の{siteTitle} {versionLabel}のドキュメントです。", 120 | "description": "The label used to tell the user that he's browsing an unreleased doc version" 121 | }, 122 | "theme.docs.versions.unmaintainedVersionLabel": { 123 | "message": "これは{siteTitle} {versionLabel}のドキュメントで現在はアクティブにメンテナンスされていません。", 124 | "description": "The label used to tell the user that he's browsing an unmaintained doc version" 125 | }, 126 | "theme.docs.versions.latestVersionSuggestionLabel": { 127 | "message": "最新のドキュメントは{latestVersionLink} ({versionLabel}) を見てください。", 128 | "description": "The label userd to tell the user that he's browsing an unmaintained doc version" 129 | }, 130 | "theme.docs.versions.latestVersionLinkLabel": { 131 | "message": "最新バージョン", 132 | "description": "The label used for the latest version suggestion link label" 133 | }, 134 | "theme.common.editThisPage": { 135 | "message": "このページを編集", 136 | "description": "The link label to edit the current page" 137 | }, 138 | "theme.common.headingLinkTitle": { 139 | "message": "見出しへの直接リンク", 140 | "description": "Title for link to heading" 141 | }, 142 | "theme.lastUpdated.atDate": { 143 | "message": "{date}に", 144 | "description": "The words used to describe on which date a page has been last updated" 145 | }, 146 | "theme.lastUpdated.byUser": { 147 | "message": "{user}が", 148 | "description": "The words used to describe by who the page has been last updated" 149 | }, 150 | "theme.lastUpdated.lastUpdatedAtBy": { 151 | "message": "{atDate}{byUser}最終更新", 152 | "description": "The sentence used to display when a page has been last updated, and by who" 153 | }, 154 | "theme.common.skipToMainContent": { 155 | "message": "メインコンテンツまでスキップ", 156 | "description": "The skip to content label used for accessibility, allowing to rapidly navigate to main content with keyboard tab/enter navigation" 157 | }, 158 | "theme.SearchPage.documentsFound.plurals": { 159 | "message": "{count}件のドキュメントが見つかりました", 160 | "description": "Pluralized label for \"{count} documents found\". Use as much plural forms (separated by \"|\") as your language support (see https://www.unicode.org/cldr/cldr-aux/charts/34/supplemental/language_plural_rules.html)" 161 | }, 162 | "theme.SearchPage.existingResultsTitle": { 163 | "message": "『{query}』の検索結果", 164 | "description": "The search page title for non-empty query" 165 | }, 166 | "theme.SearchPage.emptyResultsTitle": { 167 | "message": "ドキュメントを検索", 168 | "description": "The search page title for empty query" 169 | }, 170 | "theme.SearchPage.inputPlaceholder": { 171 | "message": "ここに検索するキーワードを入力してください", 172 | "description": "The placeholder for search page input" 173 | }, 174 | "theme.SearchPage.inputLabel": { 175 | "message": "検索", 176 | "description": "The ARIA label for search page input" 177 | }, 178 | "theme.SearchPage.algoliaLabel": { 179 | "message": "Algoliaで検索", 180 | "description": "The ARIA label for Algolia mention" 181 | }, 182 | "theme.SearchPage.noResultsText": { 183 | "message": "検索結果が見つかりませんでした", 184 | "description": "The paragraph for empty search result" 185 | }, 186 | "theme.SearchPage.fetchingNewResults": { 187 | "message": "新しい検索結果を取得しています...", 188 | "description": "The paragraph for fetching new search results" 189 | }, 190 | "theme.SearchBar.label": { 191 | "message": "検索", 192 | "description": "The ARIA label and placeholder for search button" 193 | }, 194 | "index.title": { 195 | "message": "Azure Machine Learning 日本語版" 196 | }, 197 | "index.tagline": { 198 | "message": "オープンソースの Azure Machine Learning チートシート" 199 | }, 200 | "section1": { 201 | "message": "GPU 分散学習" 202 | }, 203 | "section0": { 204 | "message": "チートシート" 205 | }, 206 | "section2": { 207 | "message": "環境" 208 | }, 209 | "section0.desc": { 210 | "message": "Azure ML で頻出するコードに関するチートシートです。" 211 | }, 212 | "section1.desc": { 213 | "message": "Azure ML で分散学習をするためのガイドです。" 214 | }, 215 | "section2.desc": { 216 | "message": "Azure ML で Python パッケージと Docker イメージを構築・管理します。" 217 | } 218 | } -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current.json: -------------------------------------------------------------------------------- 1 | { 2 | "version.label": { 3 | "message": "Next", 4 | "description": "The label for version current" 5 | }, 6 | "sidebar.pythonSidebar.category.Python": { 7 | "message": "Python", 8 | "description": "The label for category Python in sidebar pythonSidebar" 9 | }, 10 | "sidebar.pythonSidebar.category.Getting Started": { 11 | "message": "作業の開始", 12 | "description": "The label for category Getting Started in sidebar pythonSidebar" 13 | }, 14 | "sidebar.pythonSidebar.category.Azure ML Resources": { 15 | "message": "Azure ML リソース", 16 | "description": "The label for category Azure ML Resources in sidebar pythonSidebar" 17 | }, 18 | "sidebar.pythonSidebar.category.Guides": { 19 | "message": "ガイド", 20 | "description": "The label for category Guides in sidebar pythonSidebar" 21 | }, 22 | "sidebar.cliSidebar.category.CLI (preview)": { 23 | "message": "CLI (preview)", 24 | "description": "The label for category CLI (preview) in sidebar cliSidebar" 25 | } 26 | } -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/cli/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/cheatsheet.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: チートシート 3 | id: cheatsheet 4 | description: A cheat sheet for Azure ML. 5 | keywords: 6 | - azure machine learning 7 | - aml 8 | - cheatsheet 9 | - overview 10 | --- 11 | 12 | 13 | ## 基本セットアップ 14 | 15 | ### ワークスペースへの接続 16 | 17 | ```python 18 | from azureml.core import Workspace 19 | ws = Workspace.from_config() 20 | ``` 21 | 22 | この Workspace オブジェクトは Azure ML 操作における基本的なオブジェクトで、一連のコードを通して共有されます。(`ws`という変数名で参照されることが多いです。) 23 | 24 | ワークスペースの詳細: [ワークスペース](./workspace.md) 25 | 26 | ### コンピューティングターゲットへの接続 27 | 28 | ```python 29 | compute_target = ws.compute_targets[''] 30 | ``` 31 | 32 | **使用例** 33 | 34 | ```python 35 | compute_target = ws.compute_targets['powerful-gpu'] 36 | 37 | config = ScriptRunConfig( 38 | compute_target=compute_target, # train.py スクリプトを実行するために使用されるコンピューティングターゲット 39 | source_directory='.', 40 | script='train.py', 41 | ) 42 | ``` 43 | 44 | コンピューティングターゲットの詳細: [コンピューティングターゲット](./compute-targets.md) 45 | 46 | ### Python 環境の準備 47 | 48 | pip の`requirements.txt`ファイルや Conda の`env.yml`ファイルを使い、コンピューティング環境の Python 環境を Environment オブジェクトとして定義することができます。 49 | 50 | ```python 51 | from azureml.core import Environment 52 | # 選択肢 1: pip 53 | environment = Environment.from_pip_requirements('', '') 54 | # 選択肢 2: Conda 55 | environment = Environment.from_conda_specification('', '') 56 | ``` 57 | 58 | docker イメージを使って環境を準備することもできます。 59 | 60 | **使用例** 61 | 62 | ```python 63 | environment = Environment.from_pip_requirements('', '') 64 | 65 | config = ScriptRunConfig( 66 | environment=environment, # Python 環境を設定する 67 | source_directory='.', 68 | script='train.py', 69 | ) 70 | ``` 71 | 72 | 環境の詳細: [環境](./environment.md) 73 | 74 | 75 | ## コードをサブミットする 76 | 77 | Azure ML 上でコードを実行するためには: 78 | 79 | 1. エントリーポイントとなるコードのパス、コードを実行するコンピューティングターゲット、そしてコードを実行する Python 環境の**設定情報を作成**します。 80 | 2. Azure ML の実験を新規作成または再利用して**サブミット**します。 81 | 82 | ### ScriptRunConfig 83 | 84 | 典型的なディレクトリ構成例: 85 | 86 | ```bash 87 | source_directory/ 88 | script.py # エントリーポイントとなるコード 89 | module1.py # script.py により呼ばれるモジュール 90 | ... 91 | ``` 92 | 93 | リモートコンピューティングクラスター`target: ComputeTarget`上の、Python 環境`env: Environment`で、`$ (env) python /script.py [arguments]`を実行するには、 `ScriptRunConfig`クラスを使用します。 94 | 95 | ```python 96 | from azureml.core import ScriptRunConfig 97 | 98 | config = ScriptRunConfig( 99 | source_directory='', # 相対パスでも OK 100 | script='script.py', 101 | compute_target=compute_target, 102 | environment=environment, 103 | arguments=arguments, 104 | ) 105 | ``` 106 | 107 | ScriptRunConfig の引数の詳細: [コマンドライン引数](./script-run-config.md#コマンドライン引数) 108 | 109 | :::info 110 | - `compute_target`: もし引数が与えられなかった場合は、スクリプトはローカルマシン上で実行されます。 111 | - `environment`: もし引数が与えられなかった場合、Azure ML のデフォルトPython 環境が使用されます。環境の詳細: [Environment](./environment.md) 112 | ::: 113 | 114 | #### コマンド 115 | 116 | もしも明示的なコマンドを与える場合。 117 | 118 | ```python 119 | command = 'echo cool && python script.py'.split() 120 | 121 | config = ScriptRunConfig( 122 | source_directory='', # 相対パスも OK 123 | command=command, 124 | compute_target=compute_target, 125 | environment=environment, 126 | arguments=arguments, 127 | ) 128 | ``` 129 | 130 | コマンドの詳細: [コマンド](./script-run-config.md#コマンド) 131 | 132 | ### 実験 133 | 134 | コードをサブミットするには`実験`を作成します。実験は、サブミットされた一連のコードをグルーピングしてコードの実行履歴を追跡する軽量のコンテナです。 (参照: [Run History](./run-history.md)). 135 | 136 | 137 | ```python 138 | exp = Experiment(ws, '') 139 | run = exp.submit(config) 140 | print(run.get_portal_url()) 141 | ``` 142 | 143 | 上記コードで返される Azure ML Studio へのリンクにより、実験の実行をモニタリングすることができます。 144 | 145 | 詳細: [クラウド上でコードを実行する](./script-run-config.md) 146 | 147 | ### 使用例 148 | 149 | 以下はコマンドラインから Conda 環境を使ってトレーニングスクリプト`train.py`をローカルマシン上で実行する典型的な例です。 150 | 151 | ```bash 152 | $ conda env create -f env.yml # pythorch という名前の conda env を作成 153 | $ conda activate pytorch 154 | (pytorch) $ cd 155 | (pytorch) $ python train.py --learning_rate 0.001 --momentum 0.9 156 | ``` 157 | 158 | このスクリプトを Azure 上の GPU を使って実行したいと仮定します。 159 | 160 | ```python 161 | ws = Workspace.from_config() 162 | compute_target = ws.compute_targets['powerful-gpu'] 163 | environment = Environment.from_conda_specification('pytorch', 'env.yml') 164 | 165 | config = ScriptRunConfig( 166 | source_directory='', 167 | script='train.py', 168 | environment=environment, 169 | arguments=['--learning_rate', 0.001, '--momentum', 0.9], 170 | ) 171 | 172 | run = Experiment(ws, 'PyTorch model training').submit(config) 173 | ``` 174 | 175 | ## 分散 GPU 学習 176 | 177 | 分散 GPU 学習を有効にするために`ScriptRunConfig`を変更します。 178 | 179 | ```python {3,8-9,12,19} 180 | from azureml.core import Workspace, Experiment, ScriptRunConfig 181 | from azureml.core import Environment 182 | from azureml.core.runconfig import MpiConfiguration 183 | 184 | ws = Workspace.from_config() 185 | compute_target = ws.compute_targets['powerful-gpu'] 186 | environment = Environment.from_conda_specification('pytorch', 'env.yml') 187 | environment.docker.enabled = True 188 | environment.docker.base_image = 'mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.1-cudnn7-ubuntu18.04' 189 | 190 | # それぞれ 4 つの GPU を搭載した 2 つのノード上でトレーニングを行う 191 | mpiconfig = MpiConfiguration(process_count_per_node=4, node_count=2) 192 | 193 | config = ScriptRunConfig( 194 | source_directory='', # train.py が含まれるディレクトリ 195 | script='train.py', 196 | environment=environment, 197 | arguments=['--learning_rate', 0.001, '--momentum', 0.9], 198 | distributed_job_config=mpiconfig, # 分散学習のための設定を追加 199 | ) 200 | 201 | run = Experiment(ws, 'PyTorch model training').submit(config) 202 | ``` 203 | 204 | :::info 205 | - `mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.1-cudnn7-ubuntu18.04`は OpenMPI の docker イメージです。このイメージは Azure ML 上で分散学習を実行する際に必要となります。 206 | - `MpiConfiguration`はトレーニングを行うノード数とノードあたりの GPU 数を指定するために使います。 207 | ::: 208 | 209 | 詳細: [分散 GPU トレーニング](./distributed-training.md) 210 | 211 | ## データへの接続 212 | 213 | ワークスペース`ws`のデフォルトデータストアにあるデータをトレーニングスクリプトから扱うためには: 214 | 215 | ```python 216 | datastore = ws.get_default_datastore() 217 | dataset = Dataset.File.from_files(path=(datastore, '')) 218 | ``` 219 | 詳細: [データ](./data.md) 220 | 221 | コマンドライン引数に以下を渡すことで上記の`dataset`を使用できます。 222 | 223 | ```python 224 | arguments=['--data', dataset.as_mount()] 225 | ``` 226 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/ci-dev.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Azure ML 上での開発 3 | description: Guide to developing your code on Azure ML. 4 | keywords: 5 | - ssh 6 | - development 7 | - compute 8 | --- 9 | 10 | 本ガイドでは、Azure ML 上でのコード開発をするためのポイントを紹介します。典型的なシナリオとしては、分散学習のコードのテストやローカルの開発環境での再現が難しいものを対象にします。 11 | 12 | これらのシナリオの共通のペインポイントは、特に VM 上での開発と比較して、Azure ML での繰り返し作業が遅く感じられる点です。 13 | 14 | 15 | **本ガイドの目的.** Azure ML 上での開発体験をベアメタルの VM と同等もしくはそれ以上に改善することこと。 16 | 17 | ## 🚧 ハードル 18 | 19 | Azure ML での開発が遅く感じられる主な理由は 2 つあります。 20 | - Python 環境の変更には Docker イメージの再構築が必須となり、通常 5 分以上かかる。 21 | - 計算リソースはイテレーションの間に _解放される_ ために、(Docker イメージの Pull などの) ウォームアップの待機時間が発生する。 22 | 23 | 下記にてこれらの問題に対応するためのいくつかのテクニックと、Azure ML Compute を直接利用するメリットを紹介します。またこれらのテクニックを応用した [例](#例) もいくつか提供します。 24 | 25 | 26 | ## 🕰️ 開発用計算環境の準備 27 | 28 | _コンピューティングインスタンス / コンピューティングクラスター_ を作成するときに、何点か設定事項がありますj。 : 29 | 30 | 31 | 1. **SSH の有効化** 32 | 33 | SSH は、_コンピューティングインスタンス_ と _コンピューティングクラスター_ の両方でサポートされています。VM のように操作することが出来るようになります。 34 | 35 | :::tip VS Code Remote Extension. 36 | VS Code の [remote extension](https://code.visualstudio.com/docs/remote/ssh) 37 | は SSH 経由で Azure ML への計算リソースへの接続ができます。クラウド上で直接開発できるようになります。 38 | ::: 39 | 40 | 2. **"スケールダウンする前のアイドル時間 (秒)" の増加** 41 | 42 | コンピューティングクラスターはこのパラメータを例えば 30 分に増やすことができます。これは開発のイテレーションをしている間に計算環境がリリースされないようにするためです。 43 | 44 | :::warning 45 | 開発のイテレーション終了後に元に戻すことを忘れないようにしてください。 46 | ::: 47 | 48 | ## 🏃‍♀️ コマンド 49 | 50 | 通常 コードは以下のような `ScriptRunConfig` を利用して Azure ML へ送信されます : 51 | 52 | ```python 53 | config = ScriptRunConfig( 54 | source_directory='', 55 | script='script.py', 56 | compute_target=target, 57 | environment=env, 58 | ... 59 | ) 60 | ``` 61 | 62 | :::info 63 | コードを送信するのに利用する `ScriptRunConfig` に関するより詳細な情報は [クラウド上でコードを実行する](script-run-config) を参照ください。 64 | ::: 65 | [`コマンド`](script-run-config#commands) の引数を用いることでアジリティを向上することができます。以下のように、コマンドを用いて複数のステップを連結されることができます : 66 | 67 | ```python 68 | command = "pip install torch && python script.py --learning_rate 2e-5".split() 69 | ``` 70 | 71 | 他の例として下記のようなセットアップスクリプトを含めることもできます : 72 | 73 | ```bash title="setup.sh" 74 | echo "Running setup script" 75 | pip install torch 76 | pip install -r requirements.txt 77 | export PYTHONPATH=$PWD 78 | ``` 79 | 80 | を作成し、コマンドで呼び出します。 81 | 82 | ```python 83 | command = "bash setup.sh && python script.py --learning_rate 2e-5".split() 84 | ``` 85 | 86 | この方法であれば、Azure ML は追加分について Docker Image をリビルドする必要がありません。 87 | 88 | ## メリット 89 | 90 | VM 上での開発ができるだけでなく、Azure ML の計算環境を直接利用することによるメリットもあります。 91 | 92 | - **本番にすぐに展開可能.** Azure ML 上で直接開発することで、VM で開発したコードを Azure ML に移植する手間を削減できます。これは本番コードを Azure ML 上で稼働される場合に該当します。 93 | - **データアクセス.** 学習スクリプトが Azure 上のデータを利用する際、Azure ML Python SDK を用いることができます (例としては [データ](data) を参照のこと)。それ以外の方法となるとユーザ自身で、開発している VM 上でデータが取得できる方法を探す必要があります。 94 | - **ノートブック.** Azure ML の _コンピューティングインスタンス_ は Jupyter notebook を提供しており、クイックにデバッグすることできます。加えて、ノートブックは異なる計算基盤に対して実行できますし、コラボレーションの機能も提供しています。 95 | 96 | 97 | ## 例 98 | 99 | ここでは、簡易的なデモンストレーションで上記の仕組みを説明します。次のような設定を考えます。: 100 | 101 | ```bash 102 | src/ 103 | .azureml/ 104 | config.json # workspace への接続設定 105 | train.py # 開発している Python スクリプト 106 | setup.sh # train.py の前に実行するスクリプト 107 | azureml_run.py # azure への job 実行 108 | ``` 109 | 110 | ```bash title="setup.sh" 111 | echo "Running setup script" 112 | pip install numpy 113 | ``` 114 | 115 | ```python title="train.py" 116 | import numpy as np 117 | print(np.random.rand()) 118 | ``` 119 | 120 | Azure ML Python SDK を用いて、ローカル端末からクラウド上でコマンドを実行できます。 121 | 122 | ```python title="azureml_run.py" 123 | from azureml.core import Workspace, Experiment, ScriptRunConfig 124 | 125 | # workspace の取得 126 | ws = Workspace.from_config() 127 | target = ws.compute_targets['cpucluster'] 128 | exp = Experiment(ws, 'dev-example') 129 | 130 | command = "bash setup.sh && python script.py".split() 131 | 132 | # script run 構成設定 133 | config = ScriptRunConfig( 134 | source_directory='.', 135 | command=command, 136 | compute_target=target, 137 | ) 138 | 139 | # AML への script 送信 140 | run = exp.submit(config) 141 | print(run.get_portal_url()) # link to ml.azure.com 142 | run.wait_for_completion(show_output=True) 143 | ``` 144 | 145 | Python 環境の更新が必要な場合は、`setup.sh` にコマンドを追加するだけです。: 146 | 147 | ```bash title="setup.sh" 148 | echo "Running setup script" 149 | pip install numpy 150 | pip install pandas # 追加のライブラリ 151 | export CUDA_VISIBLE_DEVICES="0,1" # 環境変数の設定 152 | nvidia-smi # 便利なコマンドラインツールの実行 153 | ``` 154 | 155 | Docker イメージを再構築する必要はありません。 156 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/compute-targets.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: コンピューティングターゲット 3 | description: Guide to setting up and using Azure compute resources in Azure ML. 4 | keywords: 5 | - compute 6 | - cpu 7 | - gpu 8 | --- 9 | 10 | Compute Target (コンピューティングターゲット) は AML の計算環境の概念を抽象化したものです。対象はローカルマシンから Azure VM で構成されるクラスターまで様々です。 11 | 12 | 13 | ### Compute Target の取得 14 | 15 | ワークスペース `ws` にある既存の Compute Target の取得: 16 | 17 | ```python 18 | from azureml.core import ComputeTarget 19 | target = ComputeTarget(ws, '') 20 | ``` 21 | 22 | ### 既存 Compute Target のリスト 23 | 24 | ワークスペース `ws` にある Compute Target のリストの取得: 25 | 26 | ```python 27 | ComputeTarget.list(ws): List[ComputeTarget] 28 | ``` 29 | 30 | ### 空き状況の確認 31 | 32 | ワークスペースをチームで共有するときには、ジョブを実行する前にワークスペース `ws` の計算環境が利用可能か確認することがよくあります。 33 | 34 | [studio](https://ml.azure.com) から簡単に確認することができます。 35 | 36 | ![](img/compute-target.png) 37 | 38 | ## Compute Target の作成 39 | 40 | [studio](https://ml.azure.com) から簡単に新しい Compute Target が作成できます。 41 | 42 | "コンピューティング" のメニュー選択> "コンピューティングクラスタ" のタブを選択 > "+ 新規作成" ボタンを選択: 43 | 44 | ![](img/create-compute.png) 45 | 46 | 作成時に次の情報を入力します。: 47 | 48 | - **コンピューティング名**: 後に studio や Python SDK から参照するのに利用されます。入力必須です。名前は 名前の長さは 2 から 16 文字の間でなければなりません。有効な文字は英字、数字、- 文字です。 49 | - **仮想マシンの種類**: "CPU" or "GPU" 50 | - **仮想マシンの優先度**: "専用" もしくは "低優先度" 51 | > 低優先度の仮想マシンは安く使えますが、計算環境の確保を保証していません。ジョブが途中で中断される場合があります。 52 | - **仮想マシンのサイズ**: ドロップダウンリストから選択します。利用可能な仮想マシンのサイズの一覧は[こちら](https://azure.microsoft.com/global-infrastructure/services/?products=virtual-machines)です。 53 | - **最小 / 最大ノード数**: Compute Target は実行されたジョブの数に依って最小ノード数と最大ノード数の間でオートスケースします。最小ノード数を 0 に設定することで計算環境上でのジョブが完了すると自動で 0 台に縮小するためコストを節約できます。 54 | - **スケールダウンする前のアイドル時間 (秒)**: 計算環境をスケールダウンする前のアイドル時間を指定します。 55 | 56 | 備考: 計算環境を常に Azure Machine Learning Workspace と同じリージョンに作成されます。 57 | 58 | ### SSH の利用 59 | 60 | 管理者ユーザ名とパスワード・SSH キーを設定することで、Compute Target に対して SSH で接続できます。 61 | 62 | ![](img/create-compute-ssh.png) 63 | 64 | ### 低優先度 の Compute Target 65 | 66 | 低優先度の仮想マシンは安く使えますが、計算環境の確保を保証していません。ジョブが途中で中断される場合があります。 67 | 68 | ![](img/create-compute-lp.png) 69 | 70 | ### SDK 経由での作成 71 | 72 | SDK 経由での Compute Target の作成: 73 | 74 | ```python 75 | from azureml.core import Workspace 76 | from azureml.core.compute import ComputeTarget, AmlCompute 77 | from azureml.core.compute_target import ComputeTargetException 78 | 79 | ws = Workspace.from_config() # .azureml フォルダのファイルから接続情報を参照 80 | 81 | # CPU クラスターの名前を選択 82 | cpu_cluster_name = "cpu-cluster" 83 | 84 | # 既存のクラスターが無いことを確認 85 | try: 86 | cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name) 87 | print('Found existing cluster, use it.') 88 | except ComputeTargetException: 89 | compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', 90 | max_nodes=4, 91 | idle_seconds_before_scaledown=2400) 92 | cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, compute_config) 93 | 94 | cpu_cluster.wait_for_completion(show_output=True) 95 | ``` -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/data.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: データ 3 | description: Guide to working with data in Azure ML. 4 | keywords: 5 | - data 6 | - dataset 7 | - datastore 8 | --- 9 | 10 | ## 基本的な概念 11 | 12 | Azure ML にはデータを扱うための 2 つの概念が存在します: 13 | 14 | - データストア 15 | - データセット 16 | 17 | ### データストア 18 | 19 | Azure ML から多数のストレージアカウントへのインターフェースを提供します。 20 | 21 | 各 Azure ML ワークスペースには 1 つのデフォルトデータストアが付属しています: 22 | 23 | ```python 24 | from azureml.core import Workspace 25 | ws = Workspace.from_config() 26 | datastore = ws.get_default_datastore() 27 | ``` 28 | 29 | このデータストアには [Azure Portal](https://portal.azure.com) からもアクセスすることができます。 (Azure ML ワークスペースと同じリソースグループに存在します) 30 | 31 | データストアはワークスペースに追加され、Azure ストレージサービスとの接続情報を保持するために使用されます。そのため、ユーザーはストレージサービスの接続情報やシークレットを覚えることなく、データストアの名前で参照をすることができます。 32 | 33 | 上記コードで取得した Datastore クラスからは、データストアの登録、リスト、取得、削除などの管理を行うことができます。 34 | 35 | 36 | ### データセット 37 | 38 | データセットはデータへの参照です。これはデータストア、パブリックURL上の両方のデータを含みます。 39 | 40 | データセットはデータ管理を強化する機能を提供します (データセットのバージョン管理)。 41 | 42 | ## データストアの取得 43 | 44 | ### デフォルトデータストア 45 | 46 | 各ワークスペースにはデフォルトデータストアが付属しています。 47 | 48 | ```python 49 | datastore = ws.get_default_datastore() 50 | ``` 51 | 52 | ### データストアの登録 53 | 54 | Azure が提供する様々なデータストアに対して、データストアの作成と接続が行えます。例として: 55 | 56 | - Azure Blob Container 57 | - Azure Data Lake (Gen1 or Gen2) 58 | - Azure File Share 59 | - Azure MySQL 60 | - Azure PostgreSQL 61 | - Azure SQL 62 | - Azure Databricks File System 63 | 64 | データストアの種類のリストと認証オプションの包括的な情報は [Datastores (SDK)](https://docs.microsoft.com/python/api/azureml-core/azureml.core.datastore(class)?view=azure-ml-py) を参照してください。 65 | 66 | #### 新たなデータストアを登録する 67 | 68 | - **アカウントキー** でデータストアを登録する: 69 | 70 | ```python 71 | datastores = Datastore.register_azure_blob_container( 72 | workspace=ws, 73 | datastore_name='', 74 | container_name='', 75 | account_name='', 76 | account_key='', 77 | ) 78 | ``` 79 | 80 | - **SAS トークン**でデータストアを登録する: 81 | 82 | ```python 83 | datastores = Datastore.register_azure_blob_container( 84 | workspace=ws, 85 | datastore_name='', 86 | container_name='', 87 | account_name='', 88 | sas_token='', 89 | ) 90 | ``` 91 | 92 | ### データストアへの接続 93 | 94 | ワークスペースオブジェクト`ws`からは下記のようにデータストアのリストを取得することができます: 95 | 96 | ```python 97 | ws.datastores: Dict[str, Datastore] 98 | ``` 99 | 100 | ワークスペースに登録済みのデータストアに対しては名前でアクセスすることができます: 101 | 102 | ```python 103 | datastore = ws.datastores[''] 104 | ``` 105 | 106 | ### Azure Storage Explorer とデータストアを接続する 107 | 108 | ワークスペースオブジェクト`ws`はワークスペースがアクセス可能なデータアセットを管理する際の強力なハンドルです。例えば、Azure Storage Explorer を使ってデータストアに接続する際にもワークスペースを使うことができます。 109 | 110 | ```python 111 | from azureml.core import Workspace 112 | ws = Workspace.from_config() 113 | datastore = ws.datastores[''] 114 | ``` 115 | 116 | - **アカウントキー**を使って作成したデータストアの場合: 117 | 118 | ```python 119 | account_name, account_key = datastore.account_name, datastore.account_key 120 | ``` 121 | 122 | - **SAS トークン**を使って作成したデータストアの場合: 123 | 124 | ```python 125 | sas_token = datastore.sas_token 126 | ``` 127 | 128 | この account_name と account_key は Azure Storage Explorer を使ってデータストアに接続する際に使用できます。 129 | 130 | ## Blob データストア 131 | 132 | [AzureBlobDatastore](https://docs.microsoft.com/python/api/azureml-core/azureml.data.azure_storage_datastore.azureblobdatastore?view=azure-ml-py) データストアを使ったデータのダウンロードとアップロード。 133 | 134 | ### Blob データストアへのアップロード 135 | 136 | AzureBlobDatastore はデータをアップロードするためのAPIを提供します: 137 | 138 | ```python 139 | datastore.upload( 140 | src_dir='./data', 141 | target_path='', 142 | overwrite=True, 143 | ) 144 | ``` 145 | 146 | 別々の場所に存在する複数のファイルをアップロードする場合: 147 | 148 | ```python 149 | datastore.upload_files( 150 | files, # アップロードするファイルの絶対パスのリスト (List[str] 形式) 151 | target_path='', 152 | overwrite=False, 153 | ) 154 | ``` 155 | 156 | ### Blob データストアからのダウンロード 157 | 158 | Blob コンテナからローカルファイルシステムへデータをダウンロードする: 159 | 160 | ```python 161 | datastore.download( 162 | target_path, # str: ダウンロードするローカルディレクトリ 163 | prefix='', 164 | overwrite=False, 165 | ) 166 | ``` 167 | 168 | ### Storage Explorer 経由 169 | 170 | Azure Storage Explorer は Azure クラウドストレージ上のデータを簡単に扱うことができる Windows、macOS、Linux 向けのツールです。[こちら](https://azure.microsoft.com/features/storage-explorer/)からダウンロードすることができます。 171 | 172 | Azure Storage Explorer は GUI のファイルエクスプローラです。そのため、ファイルをドラッグアンドドロップすることでダウンロードやアップロードを行うことができます。 173 | 174 | 詳細は [Azure Storage Explorer とデータストアを接続する](#azure-storage-explorer-とデータストアを接続する) のセクションを参照してください。 175 | 176 | ## データストアからの読み込み 177 | 178 | コードの中から`Datastore`のデータを参照します。例えばリモート設定を使用する場合です。 179 | 180 | ### DataReference 181 | 182 | 最初に、基本的なオブジェクトである`Workspace`、`ComputeTarget`、`Datastore`に接続します。 183 | 184 | ```python 185 | from azureml.core import Workspace 186 | ws: Workspace = Workspace.from_config() 187 | compute_target: ComputeTarget = ws.compute_targets[''] 188 | ds: Datastore = ws.get_default_datastore() 189 | ``` 190 | 191 | `DataReference`を作成してマウントする場合: 192 | 193 | ```python 194 | data_ref = ds.path('').as_mount() 195 | ``` 196 | 197 | ダウンロードする場合: 198 | 199 | ```python 200 | data_ref = ds.path('').as_download() 201 | ``` 202 | :::info 203 | データストアを`as_mount`するためには、ワークスペースは対象のストレージに対する read と write アクセスが必要になります。readonly のデータストアの場合は`as_download`が唯一のオプションです。 204 | ::: 205 | 206 | #### ScriptRunConfig から DataReference を使用する 207 | 208 | 以下のようにして ScriptRunConfig に DataReference を追加します。 209 | 210 | ```python 211 | config = ScriptRunConfig( 212 | source_directory='.', 213 | script='script.py', 214 | arguments=[str(data_ref)], # 環境変数 $AZUREML_DATAREFERENCE_example_data を返します 215 | compute_target=compute_target, 216 | ) 217 | 218 | config.run_config.data_references[data_ref.data_reference_name] = data_ref.to_config() 219 | ``` 220 | 221 | コマンドライン引数`str(data_ref)`は、環境変数`$AZUREML_DATAREFERENCE_example_data`を返します。 222 | 最終的に`data_ref.to_config()`は、Azure ML の実行に対してデータをコンピューティングターゲットにマウントすることと、上記の環境変数を適切に設定することの指示を出します。 223 | 224 | 225 | #### 引数を指定しないとき 226 | 227 | コマンドライン引数を指定せずに、`path_on_compute`を指定してデータを参照する場合: 228 | 229 | ```python 230 | data_ref = ds.path('').as_mount() 231 | data_ref.path_on_compute = '/tmp/data' 232 | 233 | config = ScriptRunConfig( 234 | source_directory='.', 235 | script='script.py', 236 | compute_target=compute_target, 237 | ) 238 | 239 | config.run_config.data_references[data_ref.data_reference_name] = data_ref.to_config() 240 | ``` 241 | 242 | ## データセットの作成 243 | 244 | ### ローカルデータから 245 | 246 | #### データストアへのアップロード 247 | 248 | ローカルディレクトリ`./data/`をアップロードする場合: 249 | 250 | ```python 251 | datastore = ws.get_default_datastore() 252 | datastore.upload(src_dir='./data', target_path='', overwrite=True) 253 | ``` 254 | 255 | このコードはローカルディレクトリ`./data`をまるごとワークスペース`ws`に関連付けられたデフォルトデータストアにアップロードします。 256 | 257 | #### データストア内のファイルからデータセットを作成する 258 | 259 | データストアの``ディレクトリに存在するデータからデータセットを作成する場合: 260 | 261 | ```python 262 | datastore = ws.get_default_datastore() 263 | dataset = Dataset.File.from_files(path=(datastore, '')) 264 | ``` 265 | 266 | ## データセットの使用 267 | 268 | ### ScriptRunConfig 269 | 270 | 271 | データセットを ScriptRunConfig から参照してマウントしたりダウンロードしたい時、下記のようにして行えます: 272 | 273 | - `dataset.as_mount(path_on_compute)` : リモート実行時にデータセットをマウントする 274 | - `dataset.as_download(path_on_compute)` : リモート実行時にデータセットをダウンロードする 275 | 276 | **Path on compute**: `as_mount`と`as_download`の両方がオプションパラメータの`path_on_compute`を受け取ります。このパラメータはコンピューティングターゲット上で利用できるデータセットのパスを定義します。 277 | 278 | - `指定しない`場合、データは一時ディレクトリにダウンロードされます。 279 | - `path_on_compute`が`/`始まる場合は**絶対パス**として扱われます。(もしも絶対パスを指定した場合は実行するジョブがそのディレクトリに対する書き込み権限を持っている必要があります) 280 | - それ以外は、ワーキングディレクトリからの相対パスとして扱われます。 281 | 282 | マウントモードでリモート実行時にデータを参照する例: 283 | 284 | ```python title="run.py" 285 | arguments=[dataset.as_mount()] 286 | config = ScriptRunConfig(source_directory='.', script='train.py', arguments=arguments) 287 | experiment.submit(config) 288 | ``` 289 | 290 | `train.py`から参照する例: 291 | 292 | ```python title="train.py" 293 | import sys 294 | data_dir = sys.argv[1] 295 | 296 | print("===== DATA =====") 297 | print("DATA PATH: " + data_dir) 298 | print("LIST FILES IN DATA DIR...") 299 | print(os.listdir(data_dir)) 300 | print("================") 301 | ``` 302 | 303 | より詳細は [クラウド上でコードを実行する](script-run-config) を参照してください。 304 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/debugging.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: デバッグ 3 | description: Guide to debugging in Azure ML. 4 | keywords: 5 | - debug 6 | - log files 7 | --- 8 | 9 | ## Azure ML ログファイル 10 | 11 | Azure ML のログファイルは Azure ML ワークロード のデバッグを行う上で必須の情報です。 12 | 13 | | ログファイル | 説明 | 14 | | - | - | 15 | | `20_image_build_log*.txt` | Docker ビルドログ。環境を更新した時のみ該当。それ以外の場合 Azure ML はキャッシュされたイメージを再利用する。

もしも成功した場合、関連するイメージのレジストリ詳細情報を含む。 | 16 | | `55_azureml-execution*.txt` | コンピューティングターゲットに対するイメージのプル。このログはコンピューティングリソースが確保された時点で一度だけ作成される。 | 17 | | `65_job_prep*.txt` | ジョブ準備: コンピューティングターゲットやデータストアへのコードのダウンロード。(もしも操作が行われた場合) | 18 | | **`70_driver_log.txt`** | **スクリプトの標準出力。(e.g. コード中の print ステートメント)**

多くの場合、ユーザーはこのログを確認する。 | 19 | | `75_job_post*.txt` | ジョブのリリース: コンピューティングリソースが Azure へ返された際のログ。 | 20 | 21 | :::info 22 | 実行のたびにすべてのログファイルを確認する必要はありません。例えば、`20_image_build_log*.txt`は新しいイメージがビルドされた時のみ作成されます。(e.g. 環境変更時) 23 | ::: 24 | 25 | ### Studio からログを確認する 26 | 27 | これらのログファイルは Studio UI (https://ml.azure.com) の ワークスペース > 実験 > 実行 > 出力とログ から確認できます。 28 | 29 | ![](img/log-files.png) 30 | 31 | ### ストリーミングログ 32 | 33 | `Run`オブジェクトを使うことで、ストリームログをローカルターミナルに直接流し込むことができます。以下がその例です: 34 | 35 | ```python 36 | from azureml.core import Workspace, Experiment, ScriptRunConfig 37 | ws = Workspace.from_config() 38 | config = ScriptRunConfig(...) 39 | run = Experiment(ws, 'my-amazing-experiment').submit(config) 40 | run.wait_for_completion(show_output=True) 41 | ``` 42 | 43 | ## SSH 44 | 45 | デバッグの一環として、使用中のコンピューティングに対して SSH 接続することも有用な場合があります。 46 | 47 | :::warning コンピューティング作成時の SSH 有効化 48 | SSH はコンピューティングインスタンス / コンピューティングターゲット 作成時に有効にする必要があります。詳細は [Compute Targets](compute-targets#with-ssh) を参照してください。 49 | ::: 50 | 51 | 1. コンピューティングの**パブリック IP アドレス**と**ポート番号**を取得します。 52 | 53 | Studio [ml.azure.com](https://ml.azure.com/) の コンピューティングタブから、コンピューティングインスタンスもしくはコンピューティングターゲットを選択します。 54 | 55 | **Note** 接続時にコンピューティングは実行中である必要があります。 56 | - コンピューティングインスタンスの場合は単に実行中であれば良いです。 57 | - コンピューティングターゲットの場合は何かのジョブがクラスター上で実行中されている必要があります。このとき、クラスターのノードタブから、各ノードのパブリック IP アドレスとポート番号を確認することができます。([ml.azure.com](https://ml.azure.com/) > コンピューティング > _対象のコンピューティングターゲット_ > ノード) 58 | 59 | 2. 任意のシェルで下記を実行します: 60 | 61 | ```bash 62 | ssh azureuser@ -p 63 | ``` 64 | 65 | 66 | :::info RSA を使った SSH キーペア 67 | SSH パブリック-プライベートキーペアが推奨されています。詳細は[こちら](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys)を参照してください。 68 | ::: 69 | 70 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/docker-build.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Azure ML コンテナー' 3 | description: Guide to containers in Azure ML. 4 | keywords: 5 | - containers 6 | - dockerfile 7 | - docker 8 | - environment 9 | --- 10 | 11 | 本ガイドでは Azure ML でコードを実行するのに利用されるコンテナをビルドする方法を説明します。 12 | 13 | 14 | ## Dockerfile 15 | 16 | Azure ML の各ジョブは `環境` に紐づいて実行されます。実質、各環境は Docker Image に該当します。 17 | 18 | 環境を設定する方法はさまざまで、Python パッケージのセットを指定する方法から、カスタム Docker イメージを直接提供する方法などがあります。いずれの場合も、関連する Dockerfile の内容は環境のオブジェクトから直接利用できます。 19 | 20 | 詳細はこちら : [環境](environment) 21 | 22 | 23 | #### 例 24 | 25 | 環境を作成します。この例では Conda を使っていきます。: 26 | 27 | ```yml title="env.yml" 28 | name: pytorch 29 | channels: 30 | - defaults 31 | - pytorch 32 | dependencies: 33 | - python=3.7 34 | - pytorch 35 | - torchvision 36 | ``` 37 | 38 | 次のようにワークスペース `ws` に `環境` を作成し登録することができます。 39 | 40 | ```python 41 | from azureml.core import Environment 42 | env = Environment.from_conda_specification('pytorch', 'env.yml') 43 | env.register(ws) 44 | ``` 45 | 46 | この環境をリモート実行時に利用するためには、Azure ML は対応する python 環境の docker image をビルドします。 47 | 48 | ビルドで利用された Dockerfile は環境のオブジェクトから直接利用できます。 49 | 50 | ```python 51 | details = env.get_image_details(ws) 52 | print(details['ingredients']['dockerfile']) 53 | ``` 54 | 55 | Dockerfile の中身を見てみます。 : 56 | 57 | ```docker title="Dockerfile" {1,7-12} 58 | FROM mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1@sha256:8cee6f674276dddb23068d2710da7f7f95b119412cc482675ac79ba45a4acf99 59 | USER root 60 | RUN mkdir -p $HOME/.cache 61 | WORKDIR / 62 | COPY azureml-environment-setup/99brokenproxy /etc/apt/apt.conf.d/ 63 | RUN if dpkg --compare-versions `conda --version | grep -oE '[^ ]+$'` lt 4.4.11; then conda install conda==4.4.11; fi 64 | COPY azureml-environment-setup/mutated_conda_dependencies.yml azureml-environment-setup/mutated_conda_dependencies.yml 65 | RUN ldconfig /usr/local/cuda/lib64/stubs && conda env create -p /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6 - 66 | f azureml-environment-setup/mutated_conda_dependencies.yml && rm -rf "$HOME/.cache/pip" && conda clean -aqy && CONDA_ROO 67 | T_DIR=$(conda info --root) && rm -rf "$CONDA_ROOT_DIR/pkgs" && find "$CONDA_ROOT_DIR" -type d -name __pycache__ -exec rm 68 | -rf {} + && ldconfig 69 | # AzureML Conda environment name: azureml_7459a71437df47401c6a369f49fbbdb6 70 | ENV PATH /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6/bin:$PATH 71 | ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6 72 | ENV LD_LIBRARY_PATH /azureml-envs/azureml_7459a71437df47401c6a369f49fbbdb6/lib:$LD_LIBRARY_PATH 73 | COPY azureml-environment-setup/spark_cache.py azureml-environment-setup/log4j.properties /azureml-environment-setup/ 74 | RUN if [ $SPARK_HOME ]; then /bin/bash -c '$SPARK_HOME/bin/spark-submit /azureml-environment-setup/spark_cache.py'; fi 75 | ENV AZUREML_ENVIRONMENT_IMAGE True 76 | CMD ["bash"] 77 | ``` 78 | 79 | 注意: 80 | 81 | - ベースイメージは Azure ML で管理されている標準的なイメージです。全ベースイメージの Dockerfile は github (https://github.com/Azure/AzureML-Containers) で利用可能です。 82 | - Dockerfile は `mutated_conda_dependencies.yml` を参照し Conda 経由で Python 環境を構築します。 83 | 84 | `mutated_conda_dependencies.yml` の内容は環境から取得できます。: 85 | 86 | ```python 87 | print(env.python.conda_dependencies.serialize_to_string()) 88 | ``` 89 | 90 | 以下のようになっています。 91 | 92 | ```bash title="mutated_conda_dependencies.yml" 93 | channels: 94 | - defaults 95 | - pytorch 96 | dependencies: 97 | - python=3.7 98 | - pytorch 99 | - torchvision 100 | name: azureml_7459a71437df47401c6a369f49fbbdb6 101 | ``` -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/environment.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 環境 3 | description: Guide to working with Python environments in Azure ML. 4 | keywords: 5 | - environment 6 | - python 7 | - conda 8 | - pip 9 | - docker 10 | - environment variables 11 | --- 12 | 13 | Azure ML 環境 (Environments) は、コードを実行するコンテナを定義するために用いられます。最もシンプルなケースとしては、pip、Conda、または Azure ML Python SDK 経由で直接、カスタムの Python ライブラリを追加することができます。もっとカスタムが必要であれば、カスタムの Docker イメージを使うことができます。 14 | 15 | このページでは、環境の作成について例示します: 16 | 17 | - pip の `requirements.txt` ファイルから作成 18 | - Conda の `env.yml` ファイルから作成 19 | - Azure ML Python SDK 経由で直接作成 20 | - カスタム Docker イメージから作成 21 | 22 | 23 | ## Azure ML が管理する Python 環境 24 | 25 | ### pip から作成 26 | 27 | pip の `requirements.txt` ファイルから環境を作成します。 28 | 29 | ```python 30 | from azureml.core import Environment 31 | env = Environment.from_pip_requirements('', '') 32 | ``` 33 | 34 | ### Conda から作成 35 | 36 | Conda の `env.yml` ファイルから環境を作成します。 37 | 38 | ```python 39 | from azureml.core import Environment 40 | env = Environment.from_conda_specification('', '') 41 | ``` 42 | 43 | ### Azure ML SDK で作成 44 | 45 | Azure ML Python SDK を使って直接 Python 環境を作成するために、 `CondaDependencies` クラスを使います: 46 | 47 | ```python 48 | from azureml.core import Environment 49 | from azureml.core.conda_dependencies import CondaDependencies 50 | 51 | conda = CondaDependencies() 52 | 53 | # add channels 54 | conda.add_channel('pytorch') 55 | 56 | # add conda packages 57 | conda.add_conda_package('python=3.7') 58 | conda.add_conda_package('pytorch') 59 | conda.add_conda_package('torchvision') 60 | 61 | # add pip packages 62 | conda.add_pip_package('pyyaml') 63 | conda.add_pip_package('mpi4py') 64 | conda.add_pip_package('deepspeed') 65 | 66 | # create environment 67 | env = Environment('pytorch') 68 | env.python.conda_dependencies = conda 69 | ``` 70 | 71 | ## カスタム Docker イメージ または Dockerfile で作成 72 | 73 | カスタム Docker イメージから `環境` を作成するには、以下のように定義します: 74 | 75 | ```python 76 | env = Environment('') 77 | env.docker.base_image = '' 78 | env.docker.base_image_registry.address = '' 79 | env.docker.base_image_registry.username = '' 80 | env.docker.base_image_registry.password = os.environ.get("CONTAINER_PASSWORD") 81 | # optional 82 | env.python.user_managed_dependencies = True 83 | env.python.interpreter_path = '/opt/miniconda/envs/example/bin/python' 84 | ``` 85 | 86 | 例えば、 Azure Container Registry のアドレスは、 `".azurecr.io"` という形式です。 87 | 88 | **パスワードは絶対に入力しないでください。** この例では、環境変数を経由してパスワードを渡しています。 89 | 90 | Dockerfile から `環境` を作成するには、以下のように定義します: 91 | 92 | ```python 93 | env = Environment('') 94 | env.docker.base_dockerfile = './Dockerfile' # path to your dockerfile 95 | # optional 96 | env.python.user_managed_dependencies = True 97 | env.python.interpreter_path = '/opt/miniconda/envs/example/bin/python' 98 | ``` 99 | **備考** 100 | 101 | - `user_managed_dependencies = True`: 必要となるすべての Python ライブラリをインストールする必要があります。 Docker イメージの中で実行するのが典型です。 102 | - `interpreter_path`: `user_managed_dependencies=True` のときにのみ使用され、 Python インタプリタのパスを設定します。 (例: `which python`) 103 | 104 | 105 | カスタムのベースイメージを用いることで、Azure ML に Python のインストールを管理させることができます。例えば、 pip の `requirements.txt` を使って、以下のように定義します: 106 | 107 | ```python 108 | env = Environment.from_pip_requirements('', '') 109 | env.docker.base_dockerfile = './Dockerfile' 110 | ``` 111 | 112 | **メモ:** このケースでは、 `Dockerfile` でインストールされた Python ライブラリは、利用 **できなく** なります。 113 | 114 | ### Azure ML 用にカスタムの Docker イメージをビルドする 115 | 116 | こちらで入手可能な Azure ML ベースイメージ から Docker イメージをビルドすることを **強く** 推奨します: [AzureML-Containers GitHub Repo](https://github.com/Azure/AzureML-Containers) - このように定義してください: 117 | 118 | ```dockerfile title="Dockerfile" 119 | FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04 120 | ... 121 | ``` 122 | 123 | これらのイメージには、 Azure ML 上で動作するために必要となるすべての設定がなされています。 124 | 125 | スクラッチでビルドしたい方は、留意すべき要件と推奨のリストを以下に示しますので、ご覧ください: 126 | - **Conda**: デフォルトでは、 Azure ML は Python 環境を管理するために Conda を使用します。 Azure ML に Python 環境の管理をさせるつもりであれば、 Conda が必須となります。 127 | - **libfuse**: `Dataset` を使う際に必須となります。 128 | - **Openmpi**: 分散実行をする際に必須となります。 129 | - **nvidia/cuda**: (推奨) GPU ベースの学習をするためには、 [nvidia/cuda](https://hub.docker.com/r/nvidia/cuda) からイメージをビルドしてください。 130 | - **Mellanox OFED user space drivers**: (推奨) Infiniband のある SKU が対象です。 131 | 132 | 参考として [dockerfiles of Azure ML base images](https://github.com/Azure/AzureML-Containers) をご覧ください。 133 | 134 | ### プライベート・レジストリにあるカスタムイメージを使用する 135 | 136 | ログイン情報があれば、プライベート・レジストリにあるカスタムイメージを Azure ML で利用することができます。 137 | 138 | ```python 139 | env = Environment('') 140 | env.docker.base_image = "/my/private/img:tag", # image repository path 141 | env.docker.base_image_registry.address = "myprivateacr.azurecr.io" # private registry 142 | 143 | # Retrieve username and password from the workspace key vault 144 | env.docker.base_image_registry.username = ws.get_default_keyvault().get_secret("username") 145 | env.docker.base_image_registry.password = ws.get_default_keyvault().get_secret("password") 146 | ``` 147 | 148 | ## 環境の管理 149 | 150 | ### 登録された環境 151 | 152 | チームで再利用したり共有したりするために、 環境 `env: Environment` をワークスペース `ws` に登録します。 153 | 154 | ```python 155 | env.register(ws) 156 | ``` 157 | 158 | 登録された環境は、ワークスペースのハンドラ `ws` から直接取得することができます: 159 | 160 | ```python 161 | envs: Dict[str, Environment] = ws.environments 162 | ``` 163 | 164 | このディクショナリには、ワークスペースに登録されているカスタムの環境が含まれています。これは、 Azure ML によって管理される _curated environments_ のコレクションと同様です。 165 | 166 | #### 例 167 | 168 | ```python 169 | # create / update, register environment 170 | env = Environment.from_pip_requirements('my-env', 'requirements.txt') 171 | env.register(ws) 172 | 173 | # use later 174 | env = ws.environments['my-env'] 175 | 176 | # get a specific version 177 | env = Environment.get(ws, 'my-env', version=6) 178 | ``` 179 | 180 | ### 環境の保存と読み出し 181 | 182 | ローカルのディレクトリに環境を保存します: 183 | 184 | ```python 185 | env.save_to_directory('', overwrite=True) 186 | ``` 187 | 188 | これによって、(人間が理解し編集することができる) 2つのファイルを含むディレクトリが生成されます: 189 | 190 | - `azureml_environment.json` : 名前、バージョン、環境変数、 Python と Docker の設定を含むメタデータです。 191 | - `conda_dependencies.yml` : Conda で標準の、依存関係を表す YAML です。 (詳細は以下をご覧ください。 [Conda docs](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file)) 192 | 193 | この環境を後で以下の要領で読み出します。 194 | 195 | ```python 196 | env = Environment.load_from_directory('') 197 | ``` 198 | 199 | ### 環境変数 200 | 201 | 環境変数をセットするためには、 `environment_variables: Dict[str, str]` アトリビュートを使用してください。環境変数は、スクリプトが実行されるプロセスにてセットされます。 202 | 203 | ```python 204 | env = Environment('example') 205 | env.environment_variables['EXAMPLE_ENV_VAR'] = 'EXAMPLE_VALUE' 206 | ``` 207 | 208 | ## ヒントと Tips 209 | 210 | Azure ML が Conda の依存関係を管理する場合 (デフォルトでは `user_managed_dependencies=False`)、Azure ML ワークスペースと関連付けられた Azure Container Registry にある Docker イメージにて同じ環境が構成されているかどうかを、 Azure ML がチェックします。新規の環境の場合、Azure ML には、新しい環境用の新しい Docker イメージを構築するためのジョブ準備段階があります。 logs にあるイメージ・ビルドのログファイルをみて、その進捗を監視しましょう。このジョブは、イメージがビルドされ、コンテナ・レジストリに push されるまで開始しません。 211 | 212 | イメージをビルドするプロセスには少し時間がかかり、ジョブの開始が遅れます。不必要なビルドを避けるために、以下を検討してください: 213 | 214 | 1. 必要となる多くのパッケージを含む環境を登録し、可能であれば再利用してください。 215 | 2. 既存の環境に対して少しの追加パッケージを上乗せする必要があるというだけであれば、 216 | 1. 既存の環境が Docker イメージである場合、この Docker イメージの Dockerfile を使ってください。追加パッケージをインストールするためのレイヤーを1つ追加するだけで済みます。 217 | 2. ユーザースクリプトにて追加の Python パッケージをインストールするようにしてください。スクリプト内で発生するパッケージのインストールは、あなたのコードの一部として実行されます。これは新規の環境の一部として Azure ML に取り扱いを依頼することの代わりとなります。[setup script](#advanced-shell-initialization-script) を使うことを検討してください。 218 | 219 | Python パッケージの依存関係が複雑で、バージョンが競合する可能性があるため、カスタムの Docker イメージと Dockerfile (Azure ML のベースイメージをベースにしたもの) を使用することをお勧めします。この方法により、環境の透明性が完全なものとなるだけでなく、アジャイル開発段階でイメージをビルドする時間を節約することができます。 220 | 221 | ### Docker イメージをローカルでビルドし、 Azure Container Registry にプッシュする 222 | 223 | Docker をローカルにインストールしている場合、 Workspace の ACR に直接イメージをプッシュするオプションを利用して、 Azure ML の環境からの Docker イメージをローカルでビルドすることができます。ローカル・ビルドではキャッシュされたレイヤーを利用できるため、 Dockerfile を反復処理する場合にお勧めです。 224 | 225 | ```python 226 | from azureml.core import Environment 227 | myenv = Environment(name='') 228 | registered_env = myenv.register(ws) 229 | registered_env.build_local(ws, useDocker=True, pushImageToWorkspaceAcr=True) 230 | ``` 231 | 232 | ### ブートストラップ・スクリプト 233 | 234 | 開発を高速化するには、 `bootstrap.sh` を呼び出すと便利です。イメージの再ビルドが頻繁に発生することを避けるために、 _実行時に_ Python をインストールするよう変更するのが1つの典型例です。 235 | 236 | _コマンド_ を使うことでとてもシンプルに実行できます。まずは `bootstrap.sh` スクリプトをセットアップしてください。 237 | 238 | ```bash title="bootstrap.sh" 239 | echo "Running bootstrap.sh" 240 | pip install torch==1.8.0+cu111 241 | ... 242 | ``` 243 | 244 | 学習用スクリプトである `train.py` の前にこれが実行されるよう、次のコマンドを使用します: 245 | 246 | ```python 247 | cmd = "bash bootstrap.sh && python train.py --learning_rate 1e-5".split() 248 | 249 | config = ScriptRunConfig( 250 | source_directory='', 251 | command=command, 252 | compute_target=compute_target, 253 | environment=environment, 254 | ) 255 | ``` 256 | 257 | `ScriptRunConfig` についてより詳しく知りたい場合は、[クラウド上でコードを実行する](script-run-config) をご覧ください。 258 | 259 | ### 分散ブートストラッピング 260 | 261 | 場合によっては、 `bootstrap.sh` スクリプトの特定の部分を、分散セットアップの特定のランクで実行したいことがあり得ます。これは、次に示すように少し注意をすることで実現可能です: 262 | 263 | ```bash title="bootstrap.sh" 264 | MARKER="/tmp/.azureml_bootstrap_complete" 265 | 266 | if [[ $AZ_BATCHAI_TASK_INDEX = 0 ]] ; then 267 | echo "Running bootstrap.sh" 268 | echo "Installing transformers from source" 269 | pip install git+https://github.com/huggingface/transformers 270 | python -c "from transformers import pipeline; print(pipeline('sentiment-analysis')('we love you'))" 271 | pip install datasets 272 | pip install tensorflow 273 | echo "Installation complete" 274 | touch $MARKER 275 | fi 276 | echo "Barrier..." 277 | while [[ ! -f $MARKER ]] 278 | do 279 | sleep 1 280 | done 281 | echo "Bootstrap complete!" 282 | ``` 283 | 284 | このスクリプトは、他のプロセスが続行するよりも前に、ローカルランク0 (`$AZ_BATCHAI_TASK_INDEX`) が `MARKER` ファイルを作成するのを待ちます。 285 | 286 | ### シークレットを渡すために Keyvault を使う 287 | 288 | #### Workspace のデフォルトの Keyvault 289 | 290 | Azure の各 Workspace には Keyvault が付属しています。(これは Azure Portal 内で Workspace と同じリソースグループ配下にあります。) 291 | 292 | ```python 293 | from azureml.core import Workspace 294 | 295 | ws = Workspace.from_config() 296 | kv = ws.get_default_keyvault() 297 | ``` 298 | 299 | これは、シークレットの get と set の両方に使うことができます。 300 | 301 | ```python 302 | import os 303 | from azureml.core import Keyvault 304 | 305 | # add a secret to keyvault 306 | kv.set_secret(name="", value=os.environ.get("MY_SECRET")) 307 | 308 | # get a secret from the keyvault 309 | secret = kv.get_secret(name="") 310 | 311 | # equivalently 312 | secret = run.get_secret(name="") 313 | ``` 314 | 315 | #### 汎用の Azure Keyvault 316 | 317 | もちろん、Azure 内にある他の Keyvault を使うこともできます。 318 | 319 | ```python 320 | from azure.identity import DefaultAzureCredential 321 | from azure.keyvault.secrets import SecretClient 322 | 323 | credential = DefaultAzureCredential() 324 | client = SecretClient(vault_url=kv_url, credential=credential) 325 | my_secret = client.get_secret(secret_name).value 326 | 327 | env = 環境('example') 328 | env.environment_variables['POWERFUL_SECRET'] = my_secret 329 | ``` 330 | 331 | この場合、 `azure-identity` と `azure-keyvault` をプロジェクトの requirements に追加してください。 332 | 333 | ```bash 334 | pip install azure-identity azure-keyvault 335 | ``` -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/compute-target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/compute-target.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-1.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-lp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-lp.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-lp.pngZone.Identifier: -------------------------------------------------------------------------------- 1 | [ZoneTransfer] 2 | LastWriterPackageFamilyName=Microsoft.ScreenSketch_8wekyb3d8bbwe 3 | ZoneId=3 4 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-ssh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-ssh.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute-ssh.pngZone.Identifier: -------------------------------------------------------------------------------- 1 | [ZoneTransfer] 2 | LastWriterPackageFamilyName=Microsoft.ScreenSketch_8wekyb3d8bbwe 3 | ZoneId=3 4 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/create-compute.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/log-files.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/log-files.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/logging-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/logging-metrics.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/run-ex-sine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/img/run-ex-sine.png -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/installation.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: インストール 3 | description: Guide to installing Azure ML Python SDK and setting up key resources. 4 | keywords: 5 | - azureml-sdk 6 | - installation 7 | - workspace 8 | - compute 9 | - cpu 10 | - gpu 11 | --- 12 | 13 | Azure ML Python SDK のインストール: 14 | 15 | ```console 16 | pip install azureml-sdk 17 | ``` 18 | 19 | ### ワークスペースの作成 20 | 21 | Azure MLの`Workspace`作成のため、Azureサブスクリプション、リソースグループをご用意頂き、`Workspace`の名前を予め決めておいて下さい。 22 | もしAzureサブスクリプションをお持ちでない場合は[当サイトから無料でご利用頂けます](https://azure.microsoft.com/ja-jp/free/)。 23 | 24 | ```python 25 | from azureml.core import Workspace 26 | 27 | ws = Workspace.create(name='', # 任意のワークスペース名 28 | subscription_id='', # サブスクリプションID 29 | resource_group='', # 任意のリソースグループ名 30 | create_resource_group=True, 31 | location='') # リソースを作成するリージョン e.g. 'japaneast' 32 | 33 | # ワークスペースの情報を設定ファイルに書き出し: azureml/config.json 34 | ws.write_config(path='.azureml') 35 | ``` 36 | 37 | :::info 38 | 次回からは以下のように簡単にワークスペースにアクセスすることができます。 39 | ```python 40 | from azureml.core import Workspace 41 | ws = Workspace.from_config() 42 | ``` 43 | ::: 44 | 45 | ### コンピューティングターゲットの作成 46 | 47 | 以下の例はワークスペースにコンピューティングターゲットを作成します。 48 | 49 | - VM の種類: CPU 50 | - VM のサイズ: STANDARD_D2_V2 51 | - VM クラスターの最大ノード数: 4 52 | - VM クラスターのノードが自動的にスケールインするまでのアイドル時間: 2400秒 53 | 54 | GPU を使用したり VM のサイズを変更する場合は以下のコードを変更してください。 55 | 56 | ```python 57 | from azureml.core import Workspace 58 | from azureml.core.compute import ComputeTarget, AmlCompute 59 | from azureml.core.compute_target import ComputeTargetException 60 | 61 | ws = Workspace.from_config() # 自動的に .azureml/ ディレクトリを参照 62 | 63 | # 任意のクラスター名 64 | cpu_cluster_name = "cpu-cluster" 65 | 66 | try: 67 | # クラスターが既に存在するかどうかのチェック 68 | cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name) 69 | print('Found existing cluster, use it.') 70 | except ComputeTargetException: 71 | # もし無ければ作成する 72 | compute_config = AmlCompute.provisioning_configuration( 73 | vm_size='STANDARD_D2_V2', 74 | max_nodes=4, 75 | idle_seconds_before_scaledown=2400,) 76 | cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, compute_config) 77 | cpu_cluster.wait_for_completion(show_output=True) 78 | ``` 79 | 80 | :::info 81 | 次回からは以下のように簡単にコンピューティングターゲットにアクセスすることができます。 82 | 83 | ```python 84 | from azureml.core import ComputeTarget 85 | cpu_cluster = ComputeTarget(ws, 'cpu-cluster') 86 | ``` 87 | ::: -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/logging.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: メトリック 3 | description: Guide to metric logging in Azure ML. 4 | keywords: 5 | - metric 6 | - logging 7 | --- 8 | 9 | ## メトリックの記録 10 | 11 | メトリックは Azure ML の各実行に紐付けて記録され、複数の実行は一つの実験に紐付けられて記録されます。 12 | メトリックの履歴の保存と可視化を行います。 13 | 14 | ### `log` 15 | 16 | あるメトリックの 1 つの値を実行に記録します。 17 | 18 | ```python 19 | from azureml.core import Run 20 | run = Run.get_context() 21 | run.log('metric-name', metric_value) 22 | ``` 23 | 24 | あるメトリックを同一の実行に対して複数回記録することもできます。その場合、記録されたメトリックはチャートで表示されます。 25 | 26 | ### `log_row` 27 | 28 | あるメトリックを複数の列として記録します。 29 | 30 | ```python 31 | from azureml.core import Run 32 | run = Run.get_context() 33 | run.log_row("Y over X", x=1, y=0.4) 34 | ``` 35 | 36 | :::info その他の記録オプション 37 | メトリックの記録に使われる一般的な API は含まれていますが、完全なリストについては[こちら](https://docs.microsoft.com/azure/machine-learning/how-to-log-view-metrics#data-types)を参照してください。 38 | ::: 39 | 40 | ## メトリックを表示する 41 | 42 | メトリックは Azure ML Studio の中で自動的に表示可能になります。[こちら](https://ml.azure.com)のリンク先か、SDK から見ることができます: 43 | 44 | ``` 45 | run.get_workspace_url() 46 | ``` 47 | 48 | "メトリック"タブを選択し、表示したいメトリックを選択します。 49 | 50 | ![](/img/view-metrics.png) 51 | 52 | 53 | また、サマリページではメトリック間の比較をすることも可能です。 54 | `Experimets`タブから比較したい図表を選択して下さい。 55 | 56 | 57 | ![](/img/custom-view.png) 58 | 59 | 60 | :::info Custom views 61 | 図表の追加後、実行結果を選択してテーブル内のカラムを更新して下さい。 62 | 該当の図表は保存、複数作成に加え、他の方との共有もできます！ 63 | ::: 64 | 65 | 66 | ### SDK からメトリックを表示する 67 | 68 | 実行に記録されたメトリックを確認します。(詳細: [実験と実行](run)) 69 | 70 | 71 | ```python 72 | metrics = run.get_metrics() 73 | # メトリックは Dict[str, List[float]] 形式になっており、 74 | # メトリック名と list 形式の値がマッピングされて実行に保存されています。 75 | 76 | metrics.get('metric-name') 77 | # 記録された順に並んだメトリックのリスト 78 | ``` 79 | 80 | 実験`my-experiment`のメトリック`my-metric`のすべてのレコードを表示する: 81 | 82 | ```python 83 | experiments = ws.experiments 84 | # 実験名と実験オブジェクトのリスト 85 | 86 | exp = experiments['my-experiment'] 87 | for run in exp.get_runs(): 88 | metrics = run.get_metrics() 89 | 90 | my_metric = metrics.get('my-metric') 91 | if my_metric: 92 | print(my_metric) 93 | ``` 94 | 95 | ## 例 96 | 97 | ### MLFlow を使って記録する 98 | 99 | [MLFlow](https://mlflow.org/) を使って Azure ML にメトリックを記録します。 100 | 101 | ```python 102 | from azureml.core import Run 103 | 104 | # コードから実行中の実験や実行が含まれるワークスペースに接続する 105 | run = Run.get_context() 106 | ws = run.experiment.workspace 107 | 108 | # ワークスペースを ml-flow-tracking-uri に関連付ける 109 | mlflow_url = ws.get_mlflow_tracking_uri() 110 | ``` 111 | 112 | ### PyTorch Lightning を使って記録する 113 | 114 | この例は: 115 | - Lightning の`TensorBoardLogger`を含みます。 116 | - Azure ML の`Run.get_context()`を使って Lightning の`MLFlowLogger`を設定します。 117 | - Azure ML の実行の一部として使うときはこのロガーを追加するだけです。 118 | 119 | ```python 120 | import pytorch_lightning as pl 121 | 122 | run = None 123 | try: 124 | from azureml.core.run import Run, _OfflineRun 125 | run = Run.get_context() 126 | if isinstance(run, _OfflineRun): 127 | run = None 128 | except ImportError: 129 | print("Couldn't import azureml.core.run.Run") 130 | 131 | def get_logger(): 132 | tb_logger = pl.loggers.TensorBoardLogger('logs/') 133 | logger = [tb_logger] 134 | 135 | if run is not None: 136 | mlflow_url = run.experiment.workspace.get_mlflow_tracking_uri() 137 | mlf_logger = pl.loggers.MLFlowLogger( 138 | experiment_name=run.experiment.name, 139 | tracking_uri=mlflow_url, 140 | ) 141 | mlf_logger._run_id = run.id 142 | logger.append(mlf_logger) 143 | 144 | return logger 145 | ``` 146 | 147 | ここでこのロガーを lightning の`Trainer`クラスに含めます: 148 | 149 | ```python 150 | logger = get_logger() 151 | 152 | trainer = pl.Trainer.from_argparse_args( 153 | args=args, 154 | logger=logger, 155 | ) 156 | trainer.fit(model) 157 | ``` -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/run-history.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Run History 3 | --- 4 | 5 | :::note 6 | このコンテンツはお使いの言語では利用できません。 7 | ::: 8 | 9 | 10 | Azure ML can supercharge your ML workloads in (at least!) two ways: 11 | 12 | - AML Compute: Providing powerful compute resoures to train larger models 13 | - Run history: Best-in-class lineage and reproducability 14 | 15 | In this article we focus on Run History - and why you need it in your life! 16 | 17 | As teams progress to running dozens, and eventually hundreds of experiments, having 18 | some way to organize them is essential. Run History is a service that provides a number 19 | features that quickly become essential to your ML-model builders: 20 | 21 | ### Experiments and Runs 22 | 23 | When you are running dozens of experiments in multiple different projects, having a clear 24 | way to organize and search though the results is key. Azure ML provides two concepts to help 25 | with this: `Run`s and `Experiment`s. 26 | 27 | #### Runs 28 | A run is a single execution of your code - usually a training script. The run has a life-cycle: 29 | the code is prepared to be submited to Azure ML (e.g. via a ScriptRunConfig), then the code is 30 | submitted 31 | 32 | Once the code is submitted to Azure ML (for example, via a `ScriptRunConfig`) a `Run` object is 33 | created. This compute target is prepared (nodes are provisioned, containers hosting your Python 34 | environment are fired up), the entry point script is called (`$ python run.py [args]`) and logs 35 | start being generated: 36 | 37 | ```console 38 | Files already downloaded and verified 39 | epoch=1, batch= 2000: loss 2.19 40 | epoch=1, batch= 4000: loss 1.82 41 | epoch=1, batch= 6000: loss 1.66 42 | ... 43 | ``` 44 | 45 | You may log metrics to Azure ML with `run.log('', metric_value)` and monitor them in the studio: 46 | 47 | ![](img/logging-metrics.png) 48 | 49 | The training concludes, usually some model files are saved, and the nodes are 50 | released. 51 | 52 | But the story doesn't end there. The run persists even after the nodes are returned 53 | to Azure. You can always return, either in code or via the studio, to see a history 54 | of your runs, all their outputs and metrics, and the exact code that was used to generate them. 55 | 56 | #### Experiments 57 | 58 | An Experiment is a collection of runs. All runs belongs to an Experiment. Usually 59 | an Experiment is tied to a specific work item, for example, "Finetune Bert-Large", 60 | and will possess a number of runs as you iterate toward this goal. 61 | 62 | ### Snapshot 63 | 64 | When you submit your code to run in Azure ML, a _snapshot_ is taken. This is a copy of the exact 65 | code that ran. Think of this as version control for your experiments. Want to reproduce the 66 | results from that experiment 2-months ago even though you've iterated on the model and the 67 | training script in the meantime? No problem, snapshot has you covered! 68 | 69 | You have total control of what goes into the snapshot with the `.amlignore` file. This plays 70 | the same role as a `.gitignore` so you can efficiently manage what to include in the snapshot. 71 | 72 | ### Metrics 73 | 74 | As you run experiments, you track metrics - from validation loss through to GPU load. Analysing these metrics is essential to determining your best model. With Run History, these metrics are stored for all your runs. -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/run.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Experiment and Run 3 | description: Guide to running code with Azure ML 4 | keywords: 5 | - run 6 | - experiment 7 | - submit 8 | - remote 9 | - ScriptRunConfig 10 | --- 11 | 12 | :::note 13 | このコンテンツはお使いの言語では利用できません。 14 | ::: 15 | 16 | ## Concepts 17 | 18 | ### Run 19 | 20 | A run represents a single execution of your code. 21 | 22 | Azure ML is a machine-learning service that facilitates running your code in 23 | the cloud. A `Run` is an abstraction layer around each such submission, and is used to 24 | monitor the job in real time as well as keep a history of your results. 25 | 26 | ### Experiments 27 | 28 | An experiment is a light-weight container for `Run`. Use experiments to submit 29 | and track runs. 30 | 31 | Create an experiment in your workspace `ws`. 32 | 33 | ```python 34 | from azureml.core import Experiment 35 | exp = Experiment(ws, '') 36 | ``` 37 | 38 | ## Create Run 39 | 40 | ### Via ScriptRunConfig 41 | 42 | Usually a run is created by submitting a ScriptRunConfig. 43 | 44 | ```python 45 | from azureml.core import Workspace, Experiment, ScriptRunConfig 46 | ws = Workspace.from_config() 47 | exp = Experiment(ws, '') 48 | 49 | config = ScriptRunConfig(source_directory=<''>, script='train.py', ...) 50 | run = exp.submit(config) 51 | ``` 52 | 53 | For more details: [ScriptRunConfig](script-run-config) 54 | 55 | ### Get Context 56 | 57 | Code that is running within Azure ML is associated to a `Run`. The submitted code 58 | can access its own run. 59 | 60 | ```py 61 | from azureml.core import Run 62 | run = Run.get_context() 63 | ``` 64 | 65 | #### Example: Logging metrics to current run context 66 | 67 | A common use-case is logging metrics in a training script. 68 | 69 | ```py title="train.py" 70 | from azureml.core import Run 71 | 72 | run = Run.get_context() 73 | 74 | # training code 75 | for epoch in range(n_epochs): 76 | model.train() 77 | ... 78 | val = model.evaluate() 79 | run.log('validation', val) 80 | ``` 81 | 82 | When this code is submitted to Azure ML (e.g. via ScriptRunConfig) it will log metrics to its assocaited run. 83 | 84 | For more details: [Logging Metrics](logging) 85 | 86 | ### Interactive 87 | 88 | In an interactive setting e.g. a Jupyter notebook 89 | 90 | ```python 91 | run = exp.start_logging() 92 | ``` 93 | 94 | #### Example: Jupyter notebook 95 | 96 | A common use case for interacive logging is to train a model in a notebook. 97 | 98 | ```py 99 | from azureml.core import Workspace 100 | from azureml.core import Experiment 101 | ws = Workspace.from_config() 102 | exp = Experiment(ws, 'example') 103 | 104 | run = exp.start_logging() # start interactive run 105 | print(run.get_portal_url()) # get link to studio 106 | 107 | # toy example in place of e.g. model 108 | # training or exploratory data analysis 109 | import numpy as np 110 | for x in np.linspace(0, 10): 111 | y = np.sin(x) 112 | run.log_row('sine', x=x, y=y) # log metrics 113 | 114 | run.complete() # stop interactive run 115 | ``` 116 | 117 | Follow the link to the run to see the metric logging in real time. 118 | 119 | ![](img/run-ex-sine.png) -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/script-run-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: クラウド上でコードを実行する 3 | description: Guide to running code with Azure ML 4 | keywords: 5 | - run 6 | - experiment 7 | - submit 8 | - remote 9 | - ScriptRunConfig 10 | --- 11 | 12 | ## 実験と実行 13 | 14 | Azure ML は機械学習コードのクラウド上での実行を支援するサービスです。`実行`は Azure ML にサブミットされたジョブの履歴をただ保存するだけではなく、リアルタイムで監視することもできる抽象レイヤーです。 15 | 16 | - 実行: 一度のコード実行を表します。詳細: [実行](#実行) 17 | - 実験: 実験は`実行`の軽量なコンテナです。実験は実行の Azure ML へのサブミットと追跡に使われます。 18 | 19 | ワークスペース`ws`に実験を作成します。 20 | 21 | ```python 22 | from azureml.core import Experiment 23 | exp = Experiment(ws, '') 24 | ``` 25 | 26 | ## ScriptRunConfig 27 | 28 | 一般的に Azure ML では`ScriptRunConfig`を使って実行するコードの情報と実行のための設定情報をパッケージ化して、クラウド上にサブミットして実行します。 29 | 30 | 実行しようとしているコードが以下のディレクトリ構成だとします。 31 | 32 | ```bash 33 | source_directory/ 34 | script.py # コードのエントリーポイント 35 | module1.py # script.py によって呼ばれるモジュール 36 | ... 37 | ``` 38 | 39 | `ScriptRunConfig`を使って`script.py`をクラウド上で実行するための設定: 40 | 41 | ```python 42 | config = ScriptRunConfig( 43 | source_directory='', 44 | script='script.py', 45 | compute_target=target, 46 | environment=env, 47 | arguments = [ 48 | '--learning_rate', 0.001, 49 | '--momentum', 0.9, 50 | ] 51 | ) 52 | ``` 53 | 54 | ここで: 55 | 56 | - `source_directory='source_directory'` : 実行するコードが存在するローカルディレクトリ。 57 | - `script='script.py'` : 実行する Python スクリプト。必ずしも`source_directory`のルートにある必要はない。 58 | - `compute_taget=target` : 参照 [コンピューティングターゲット](compute-targets) 59 | - `environment` : 参照 [環境](environment) 60 | - `arguments` : 参照 [コマンドライン引数](#コマンドライン引数) 61 | 62 | このコードを Azure ML にサブミットする: 63 | 64 | ```python 65 | exp = Experiment(ws, '') 66 | run = exp.submit(config) 67 | print(run) 68 | run.wait_for_completion(show_output=True) 69 | ``` 70 | 71 | このコードはターミナル上にログストリームを出力するだけではなく、サブミットされた実行を Web 上で監視するためのリンクを出力します。(https://ml.azure.com) 72 | 73 | ## コマンドライン引数 74 | 75 | スクリプトにコマンドライン引数を渡すには`ScriptRunConfig`の中にある`arguments`パラメータを使います。 76 | 引数は list 形式で指定します: 77 | 78 | ```python 79 | arguments = [first, second, third, ...] 80 | ``` 81 | 82 | このとき引数は下記のコマンドライン引数のように渡されます: 83 | 84 | ```console 85 | $ python script.py first second third ... 86 | ``` 87 | 88 | 名前付きの引数もサポートされます: 89 | 90 | ```python 91 | arguments = ['--first_arg', first_val, '--second_arg', second_val, ...] 92 | ``` 93 | 94 | 引数には`int`、`float`、`str`などのデータ型に加えて他の参照型のデータも使えます。 95 | 96 | コマンドライン引数の詳細: [データセットの使用](data#データセットの使用) 97 | 98 | ### 引数の例 1: `sys.argv` 99 | 100 | この例では 2 つの引数をスクリプトに渡します。コンソールから実行する場合: 101 | 102 | ```console title="console" 103 | $ python script.py 0.001 0.9 104 | ``` 105 | 106 | これを`ScriptRunConfig`の中の`arguments`を使って表現する場合: 107 | 108 | ```python title="run.py" 109 | arguments = [0.001, 0.9] 110 | 111 | config = ScriptRunConfig( 112 | source_directory='.', 113 | script='script.py', 114 | arguments=arguments, 115 | ) 116 | ``` 117 | 118 | これらの引数はスクリプトの中で通常のコマンドライン引数と同じように使えます: 119 | 120 | ```python title="script.py" 121 | import sys 122 | learning_rate = sys.argv[1] # 0.001 を受け取る 123 | momentum = sys.argv[2] # 0.9 を受け取る 124 | ``` 125 | 126 | ### 引数の例 2: `argparse` 127 | 128 | この例では 2 つの名前付きの引数をスクリプトに渡します。コンソールから実行する場合: 129 | 130 | ```console title="console" 131 | $ python script.py --learning_rate 0.001 --momentum 0.9 132 | ``` 133 | 134 | これを`ScriptRunConfig`の中の`arguments`を使って表現する場合: 135 | 136 | ```python title="run.py" 137 | arguments = [ 138 | '--learning_rate', 0.001, 139 | '--momentum', 0.9, 140 | ] 141 | 142 | config = ScriptRunConfig( 143 | source_directory='.', 144 | script='script.py', 145 | arguments=arguments, 146 | ) 147 | ``` 148 | 149 | これらの引数はスクリプトの中で通常のコマンドライン引数と同じように使えます: 150 | 151 | ```python title="script.py" 152 | import argparse 153 | parser = argparse.Argparser() 154 | parser.add_argument('--learning_rate', type=float) 155 | parser.add_argument('--momentum', type=float) 156 | args = parser.parse_args() 157 | 158 | learning_rate = args.learning_rate # 0.001 を受け取る 159 | momentum = args.momentum # 0.9 を受け取る 160 | ``` 161 | 162 | ## コマンド 163 | 164 | 明示的に実行するコマンドを与えることもできます。 165 | 166 | ```python 167 | command = 'python script.py'.split() 168 | 169 | config = ScriptRunConfig( 170 | source_directory='', 171 | command=command, 172 | compute_target=compute_target, 173 | environment=environment, 174 | ) 175 | ``` 176 | 177 | この例は`command`引数の代わりに`script='script.py'`という引数をするということと同じです。 178 | 179 | このオプションは多くの柔軟性を与えます。例えば: 180 | 181 | - **環境変数の設定**: よくある例: 182 | 183 | ```python 184 | command = 'export PYTHONPATH=$PWD && python script.py'.split() 185 | ``` 186 | 187 | ```python 188 | command = f'export RANK={rank} && python script.py'.split() 189 | ``` 190 | 191 | - **セットアップスクリプトの実行**: データのダウンロードや環境変数の設定を行うセットアップスクリプトの実行。 192 | 193 | ```python 194 | command = 'python setup.py && python script.py'.split() 195 | ``` 196 | 197 | ## データセットの使用 198 | 199 | ### 引数から 200 | 201 | `ScriptRunConfig`に引数としてデータセットを渡します。 202 | 203 | ```py 204 | # データセットを作成する 205 | datastore = ws.get_default_datastore() 206 | dataset = Dataset.File.from_files(path=(datastore, '')) 207 | 208 | arguments = ['--dataset', dataset.as_mount()] 209 | 210 | config = ScriptRunConfig( 211 | source_directory='.', 212 | script='script.py', 213 | arguments=arguments, 214 | ) 215 | ``` 216 | 217 | この例では`script.py`から参照可能な実行に対してデータセットがマウントされます。 218 | 219 | ## 実行 220 | 221 | ### インタラクティブ 222 | 223 | Jupyter Notebookなどを使う場合のインタラクティブ設定 224 | 225 | ```python 226 | run = exp.start_logging() 227 | ``` 228 | 229 | #### 例 : Jupyter Notebook 230 | 231 | よくあるユースケースはノートブック内で学習中のモデルのログをインタラクティブに表示する場合です。 232 | 233 | ```py 234 | from azureml.core import Workspace 235 | from azureml.core import Experiment 236 | ws = Workspace.from_config() 237 | exp = Experiment(ws, 'example') 238 | 239 | run = exp.start_logging() # インタラクティブ実行の開始 240 | print(run.get_portal_url()) # Azure ML Studio へのリンクの取得 241 | 242 | # モデル学習コードのダミー 243 | # 実際は学習やEDAなど 244 | import numpy as np 245 | for x in np.linspace(0, 10): 246 | y = np.sin(x) 247 | run.log_row('sine', x=x, y=y) # メトリックのロギング 248 | 249 | run.complete() # インタラクティブ実行の終了 250 | ``` 251 | 252 | Azure ML Studio へのリンクからリアルタイムでメトリックのログや実行を確認できます。 253 | 254 | ![](img/run-ex-sine.png) 255 | 256 | ### コンテキストの取得 257 | 258 | Azrue ML 上で実行されているコードは`実行`に関連付けられます。サブミットされたコードは`実行`からアクセスすることができます。 259 | 260 | ```py 261 | from azureml.core import Run 262 | run = Run.get_context() 263 | ``` 264 | 265 | #### 例: 実行中のコードのメトリックをログに保存する 266 | 267 | よくあるユースケースはトレーニングスクリプト内でのメトリックのログです。 268 | 269 | ```py title="train.py" 270 | from azureml.core import Run 271 | 272 | run = Run.get_context() 273 | 274 | # training code 275 | for epoch in range(n_epochs): 276 | model.train() 277 | ... 278 | val = model.evaluate() 279 | run.log('validation', val) 280 | ``` 281 | 282 | このコードが Azrue ML (例えば`ScriptRunConfig`により) にサブミットされた時、このコードは関連付けられている`実行`にメトリックのログを保存します。 283 | 284 | 詳細: [メトリック](logging) -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/templates.md: -------------------------------------------------------------------------------- 1 | # Templates 2 | 3 | ## Introduction 4 | 5 | Cookiecutter is a simple command-line tool that allows you to quickly create 6 | new projects from pre-defined templates. Let's see it in action! 7 | 8 | First go ahead and get cookiecutter using your environment manager of choice, 9 | for example: 10 | 11 | ```bash 12 | pip install cookiecutter 13 | ``` 14 | 15 | Then give this repo a home 16 | 17 | ```bash 18 | cd ~/repos # or wherever your repos call home :-) 19 | git clone 20 | ``` 21 | 22 | To create a new project from the `ScriptRunConfig` template for example, simply 23 | run 24 | 25 | ```bash 26 | cookiecutter path/to/cheatsheet/repo/templates/ScriptRunConfig 27 | ``` 28 | 29 | See [ScriptRunConfig](#ScriptRunConfig) for more details for this template. 30 | 31 | ## Project Templates 32 | 33 | - ScriptRunConfig: Create a project to run a script in AML making use of the 34 | ScriptRunConfig class. This template is well suited for smaller projects and 35 | is especially handy for testing. 36 | 37 | ### ScriptRunConfig 38 | 39 | [Cookiecutter](https://cookiecutter.readthedocs.io/en/1.7.2/README.html) 40 | template for setting up an AML 41 | [ScriptRunConfig](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.scriptrunconfig?view=azure-ml-py) 42 | used to run your script in Azure. 43 | 44 | #### Usage 45 | 46 | Run the cookiecutter command 47 | 48 | ```bash 49 | cookiecutter /ScriptRunConfig 50 | ``` 51 | 52 | to create a new `ScriptRunConfig` project. 53 | 54 | **Note.** Install with `pip install cookiecutter` (see 55 | [cookiecutter docs](https://cookiecutter.readthedocs.io/en/1.7.2/installation.html) 56 | for more installation options) 57 | 58 | You will be prompted for the following: 59 | 60 | - `directory_name`: The desired name of the directory (default: 61 | "aml-src-script") 62 | - `script_name`: The name of the python script to be run in Azure (default: 63 | "script") 64 | - `subscription_id`: Your Azure Subscription ID 65 | - `resource_group`: Your Azure resource group name 66 | - `workspace_name`: Your Azure ML workspace name 67 | - `compute_target_name`: The name of the Azure ML compute target to run the 68 | script on (default: "local", will run on your box) 69 | 70 | Cookiecutter creates a new project with the following layout. 71 | 72 | ```bash 73 | {directory_name}/ 74 | {script_name}.py # the script you want to run in the cloud 75 | run.py # wraps your script in ScriptRunConfig to send to Azure 76 | config.json # your Azure ML metadata 77 | readme.md # this readme file! 78 | ``` 79 | 80 | See 81 | [ScriptRunConfig](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.scriptrunconfig?view=azure-ml-py) 82 | for more options and details on configuring runs. 83 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/troubleshooting.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: トラブルシューティング 3 | id: troubleshooting 4 | description: A cheat sheet for Azure ML. 5 | keywords: 6 | - azure machine learning 7 | - aml 8 | - troubleshooting 9 | --- 10 | 11 | ### エラー: az acr login- APIVersion 2020-11-01-preview is not available. 12 | **説明**: 13 | az acr を使ってイメージをビルドすると NotImplementedError が発生する。 14 | ```bash 15 | az acr build --image $image_name --subscription $ws.subscription_id --registry $cr --file docker/Dockerfile docker/ 16 | ``` 17 | エラー表示: 18 | ```text 19 | NotImplementedError: APIVersion 2020-11-01-preview is not available. 20 | ``` 21 | 22 | **解決策**: 23 | az cli のバージョンに依存する問題です。下記コマンドを実行して az cli の更新を行ってください。 24 | ```bash 25 | az upgrade --yes 26 | ``` 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v1/workspace.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: ワークスペース 3 | description: Azure ML ワークスペースの概要 4 | keywords: 5 | - workspace 6 | --- 7 | 8 | ワークスペースは、Azure ML で用いられる基本的なオブジェクトであり、他の多くのクラスのコンストラクタの中で使用されます。 9 | このドキュメントを通して、私たちは頻繁にワークスペース・オブジェクトのインスタンス化を省略し、単純に `ws` を参照します。 10 | 11 | 新規ワークスペースの作成についての説明が必要でしたら、[インストール](installation)を見てください。 12 | 13 | ## ワークスペースを取得する 14 | 15 | AMLアセットへの接続に用いられる `Workspace` オブジェクトをインスタンス化します。 16 | 17 | ```python title="run.py" 18 | from azureml.core import Workspace 19 | ws = Workspace( 20 | subscription_id="", 21 | resource_group="", 22 | workspace_name="", 23 | ) 24 | ``` 25 | 26 | 利便性のために、ワークスペースのメタデータを `config.json` 内に保存します。 27 | 28 | ```json title=".azureml/config.json" 29 | { 30 | "subscription_id": , 31 | "resource_group": , 32 | "workspace_name": 33 | } 34 | ``` 35 | 36 | ### 役立つメソッド 37 | 38 | - `ws.write_config(path, file_name)` : あなたの代わりに `config.json` を書き出します。 `path` はデフォルトでカレントワーキングディレクトリ内の `.azureml/` 、 `file_name` はデフォルトで `config.json` です。 39 | - `Workspace.from_config(path, _file_name)`: コンフィグからワークスペースの設定を読み込みます。そのパラメーターは、カレントディレクトリで検索を開始するのがデフォルトです。 40 | 41 | :::info 42 | これらを `.azureml/` ディレクトリに格納するのが推奨されます。 `Workspace.from_config` メソッドでは _デフォルトで_ このパスが検索されるためです。 43 | ::: 44 | 45 | ## ワークスペースのアセットを取得する 46 | 47 | ワークスペースは、以下の Azure ML アセットへのハンドラを提供します。 48 | 49 | ### Compute Targets 50 | 51 | ワークスペースにアタッチされた全ての Compute ターゲットを取得します。 52 | 53 | ```python 54 | ws.compute_targets: Dict[str, ComputeTarget] 55 | ``` 56 | 57 | ### Datastores 58 | 59 | ワークスペースに登録された全てのデータストアを取得します。 60 | 61 | ```python 62 | ws.datastores: Dict[str, Datastore] 63 | ``` 64 | 65 | ワークスペースのデフォルトのデータストアを取得します。 66 | 67 | ```python 68 | ws.get_default_datastore(): Datastore 69 | ``` 70 | 71 | ### Keyvault 72 | 73 | ワークスペースのデフォルトの Keyvault を取得します。 74 | 75 | ```python 76 | ws.get_default_keyvault(): Keyvault 77 | ``` 78 | 79 | ### Environments 80 | 81 | ワークスペースに登録された Environments を取得します。 82 | 83 | ```python 84 | ws.environments: Dict[str, Environment] 85 | ``` 86 | 87 | ### MLFlow 88 | 89 | MLFlow の tracking URI を取得します。 90 | 91 | ```python 92 | ws.get_mlflow_tracking_uri(): str 93 | ``` -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/cheatsheets/python/v2/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/misc/contributing.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Contributing 3 | description: Guide to contributing. 4 | --- 5 | 6 | ## Issues 7 | 8 | All forms of feedback are welcome through [issues](https://github.com/Azure/azureml-cheatsheets/issues) - please follow the pre-defined templates where applicable. 9 | 10 | ## Pull requests 11 | 12 | Pull requests (PRs) to this repo require review and approval by the Azure Machine Learning team to merge. Please follow the pre-defined template and read all relevant sections below. 13 | 14 | Make PR's against the `main` branch. 15 | 16 | ```bash 17 | git clone git@github.com:Azure/azureml-cheatsheets.git 18 | cd azureml-cheatsheets 19 | git checkout -b user/contrib 20 | ... 21 | gh pr create 22 | ``` 23 | 24 | - When a PR arrives against `main` GitHub actions (deploy) will test the build is successful 25 | - When the PR is merged the change will be automatically deployed to `gh-pages` branch (and the webpage will be updated) 26 | 27 | 99% of contributions should only need the following: 28 | 29 | - Add markdown files to the `website/docs/cheatsheets` folder 30 | - Update the `sidebar.js` file to add a page to the sidebar 31 | - Put any images in `website/docs/cheatsheets//img/` and refer to them like this: `![](img/.png)` 32 | 33 | If you need to do anything more than adding a new page to the sidebar (e.g. 34 | modify the nav bar) then please refer to the [Docusaurus 2 documentation](https://v2.docusaurus.io/). 35 | 36 | ## Previewing changes locally 37 | 38 | - Install [npm](https://nodejs.org/en/download/) and [yarn](https://classic.yarnpkg.com/en/docs/install#windows-stable). 39 | 40 | - Initial Docusaurus installation: (**First time only**) 41 | 42 | ```bash 43 | cd website 44 | npm install 45 | ``` 46 | 47 | 48 | - Run local server while developing: 49 | 50 | ```bash 51 | cd website 52 | yarn start 53 | ``` 54 | 55 | See Docusaurus instructions [here](https://v2.docusaurus.io/docs/installation) for more details. -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/misc/deployment.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Deployment 3 | description: One-time website deployment setup. 4 | --- 5 | 6 | ## Deployment 7 | 8 | This article describes the one-time process for deploying the cheat sheeets as a GitHub Pages website. 9 | 10 | This repo has GitHub actions in place that automate deployment by watching the `main` branch. 11 | If you are interested in how deployment works then read on! 12 | 13 | ### GitHub Actions 14 | 15 | We use GitHub actions to automate deployment. Set up was as follows: 16 | 17 | - Generated new SSH key 18 | - NB. Since there was an existing ssh key tied the repo a new key was generated (in a different location) `/tmp/.ssh/id_rsa` 19 | - Add public key to repo's [deploy key](https://developer.github.com/v3/guides/managing-deploy-keys/) 20 | - NB. Allow write access 21 | - Add private key as [GitHub secret](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets) 22 | - We use repo-level (not org level) secret 23 | - Secret is named `GH_PAGES_DEPLOY` 24 | - `xclip -sel clip < /tmp/.ssh/id_rsa` 25 | 26 | ### Manual 27 | 28 | It is possible to make manual deployments without use of the GitHub action above. 29 | 30 | ```console 31 | GIT_USER= USE_SSH=true yarn deploy 32 | ``` 33 | 34 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 35 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/misc/issues.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Issue Triage Process 3 | id: issues 4 | description: GitHub issue triage process for Azure Machine Learning. 5 | keywords: 6 | - azure machine learning 7 | - aml 8 | - azure 9 | --- 10 | 11 | ## Overview 12 | 13 | This page defines the triage process for Azure Machine Learning (AML) repositories. 14 | 15 | ## Repositories 16 | 17 | AML examples: 18 | 19 | - https://github.com/Azure/MachineLearningNotebooks 20 | - https://github.com/Azure/azureml-examples 21 | 22 | Azure core: 23 | 24 | - https://github.com/Azure/azure-cli 25 | - https://github.com/Azure/azure-cli-extensions 26 | - https://github.com/Azure/azure-powershell 27 | - https://github.com/Azure/azure-rest-api-specs 28 | - https://github.com/Azure/azure-sdk-for-js 29 | - https://github.com/Azure/azure-sdk-for-python 30 | 31 | > To request a repository to be added, [open an issue](https://github.com/Azure/azureml-web/issues) 32 | 33 | ## Code of Conduct 34 | 35 | All interactions on GitHub must follow the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 36 | 37 | ## Priority 38 | 39 | GitHub user experience. 40 | 41 | ## Metrics 42 | 43 | - FQR: first quality response 44 | - TTC: time to close 45 | 46 | ## Goals 47 | 48 | - triage issue area and type in <3 hrs 49 | - FQR <8 hrs 50 | - TTC for questions <5 days 51 | - TTC for bugs <30 days 52 | 53 | ## SLA 54 | 55 | - triage <1 day 56 | - FQR <3 days 57 | 58 | ## Labels 59 | 60 | ### Areas 61 | 62 | #### Foundations 63 | 64 | - `Foundations/Data` 65 | - `Foundations/Compute` 66 | - `Foundations/Infrastructure` 67 | - `Foundations/Admin` 68 | 69 | #### Experiences 70 | 71 | - `Experiences/UI` 72 | - `Experiences/Lifecycle` 73 | - `Experiences/Intelligence` 74 | - `Experiences/Inference` 75 | 76 | #### Pipelines 77 | 78 | - `Pipelines/UI` 79 | - `Pipelines/Aether` 80 | 81 | ### Issue types 82 | 83 | - `bug` 84 | - `question` 85 | - `feature-request` 86 | 87 | ### Other 88 | 89 | - `needs-details`: additional details needed from author 90 | - `v2`: planned for AMLv2 91 | 92 | ## Process 93 | 94 | ### Triage 95 | 96 | Initial triage will be performed by the GitHub v-team. On initial triage, assign the correct area label and issue type. 97 | 98 | If the issue needs obvious clarification before this can be done, kindly ask the user. If the issue has no path to closing without user response, mark it as `needs-details`. 99 | 100 | After initial triage, it is up to each area (Experiences, Foundations, Pipelines) to further triage as necessary to the correct engineering team members. 101 | 102 | One type of issue may be changed to another, i.e. for an issue like “can I do X” could end up as a feature request for X. Simply change the issue labels as appropriate. In some cases, it might make sense to open a new issue and close the original instead of changing the label. 103 | 104 | Once the issue is understood, it is up to each area to appropriately route through internal tools such as ADO, maintaining the GitHub issue as the point of communication with the user. Major developments should be communicated back to the user. 105 | 106 | ### Closing 107 | 108 | Issues may be closed by their creator at anytime, which is preferred, **especially for questions**. 109 | 110 | Additionally, issues may be closed once: 111 | 112 | - `needs-details`: user/author has not responded for 5+ days with no other path to closure 113 | - `question`: the question has been thoroughly answered with relevant links, documentation, and examples and has no follow-up questions from user(s) in 48 hrs 114 | - `bug`: the bug fix has been released, tested, and the user confirms the solution or does not respond for 48 hrs after being made aware of the fix 115 | - `feature-request`: the feature has been released, tested, and the user confirms the solution or does not respond for 48 hrs after being made aware of the release 116 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/vs-code-snippets/readme.md: -------------------------------------------------------------------------------- 1 | # VS Code Snippets 2 | 3 | Notes for contributing Azure ML Snippets. 4 | 5 | For using snippets see https://azure.github.io/azureml-web/docs/vs-code-snippets/snippets. 6 | 7 | 1. Add snippets to `python.json`. For more details on VS Code snippets: [vs-code-docs](https://code.visualstudio.com/docs/editor/userdefinedsnippets) 8 | 2. Run `python snippets-parser.py` to automatically update the `snippets.md` (which will document your changes) 9 | 3. Make a PR to the `main` branch and request a review. 10 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/vs-code-snippets/snippets-parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List 3 | 4 | 5 | class Snippet: 6 | """Handle json snippets 7 | 8 | Parse json (VS Code) snippets file and generate markdown summary. 9 | """ 10 | 11 | def __init__(self, name, snippet_json): 12 | self.name = name 13 | self.description = snippet_json.get("description") 14 | self.prefix = self._read_prefix(snippet_json.get("prefix")) 15 | self.body = snippet_json.get("body") 16 | 17 | def __repr__(self): 18 | return f"Snippet({self.name})" 19 | 20 | @staticmethod 21 | def _read_prefix(prefix): 22 | """Guarentee prefix is of type List.""" 23 | if type(prefix) == list: 24 | return prefix 25 | else: 26 | assert type(prefix) == str 27 | return [prefix] 28 | 29 | def to_markdown(self) -> List[str]: 30 | """Convert snippet to markdown (as list of lines).""" 31 | lines = [] 32 | 33 | # add heading 34 | heading = f"### {self.name}" 35 | lines.append(heading) 36 | lines.append("") 37 | 38 | # add description 39 | description = f"Description: {self.description}" 40 | lines.append(description) 41 | lines.append("") 42 | 43 | # add prefix(es) 44 | if len(self.prefix) > 1: 45 | prefix = f"Prefixes: " 46 | else: 47 | prefix = f"Prefix: " 48 | for p in self.prefix: 49 | prefix += f"`{p}`, " 50 | prefix = prefix[:-2] # remove trailing comma and whitespace 51 | lines.append(prefix) 52 | lines.append("") 53 | 54 | # add python snippet 55 | lines.append("```python") 56 | for line in self.body: 57 | if line == "$0": 58 | continue 59 | lines.append(line) 60 | lines.append("```") 61 | 62 | return lines 63 | 64 | @staticmethod 65 | def _convert_to_json(body): 66 | json_body = [] 67 | for line in body[:-1]: 68 | line = '"' + line + '",' 69 | json_body.append(line) 70 | line = '"' + body[-1] + '"' 71 | json_body.append(line) 72 | return json_body 73 | 74 | 75 | frontmatter = """--- 76 | title: VS Code Snippets 77 | description: A collection of VS Code Snippets for working with Azure ML. 78 | --- 79 | 80 | We have compiled a collection of useful templates in the form of 81 | [VS code snippets](https://code.visualstudio.com/docs/editor/userdefinedsnippets). 82 | 83 | ![VS Code Snippets](vs-code-snippets-demo.gif) 84 | 85 | To add these snippets to your VS Code: `ctrl+shift+p` > Type 'Configure user 86 | snippets' > Select `python.json`. All of these snippets are available here: 87 | [python.json](https://github.com/Azure/azureml-web/blob/main/website/docs/vs-code-snippets/python.json) 88 | 89 | """ 90 | 91 | if __name__ == "__main__": 92 | 93 | # parse snippets 94 | with open("python.json") as f: 95 | snippets_file = json.load(f) 96 | 97 | snippets = [] 98 | for name, snippet_json in snippets_file.items(): 99 | snippet = Snippet(name, snippet_json) 100 | snippets.append(snippet) 101 | 102 | # create file and write frontmatter 103 | md_filename = "snippets.md" 104 | with open(md_filename, "w") as f: 105 | # write frontmatter 106 | f.writelines(frontmatter) 107 | 108 | # write each snippet 109 | for snippet in snippets: 110 | lines = snippet.to_markdown() 111 | for line in lines: 112 | f.write(line + "\n") 113 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-plugin-content-docs/current/vs-code-snippets/vs-code-snippets-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/i18n/ja/docusaurus-plugin-content-docs/current/vs-code-snippets/vs-code-snippets-demo.gif -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-theme-classic/footer.json: -------------------------------------------------------------------------------- 1 | { 2 | "link.title.Resources": { 3 | "message": "Resources", 4 | "description": "The title of the footer links column with title=Resources in the footer" 5 | }, 6 | "link.title.Support": { 7 | "message": "Support", 8 | "description": "The title of the footer links column with title=Support in the footer" 9 | }, 10 | "link.title.GitHub": { 11 | "message": "GitHub", 12 | "description": "The title of the footer links column with title=GitHub in the footer" 13 | }, 14 | "link.item.label.Azure ML - Microsoft Docs": { 15 | "message": "Azure ML - Microsoft Docs", 16 | "description": "The label of footer link with label=Azure ML - Microsoft Docs linking to https://docs.microsoft.com/azure/machine-learning" 17 | }, 18 | "link.item.label.Azure ML - Python API": { 19 | "message": "Azure ML - Python API", 20 | "description": "The label of footer link with label=Azure ML - Python API linking to https://docs.microsoft.com/python/api/overview/azure/ml/?view=azure-ml-py" 21 | }, 22 | "link.item.label.GitHub issues": { 23 | "message": "GitHub issues", 24 | "description": "The label of footer link with label=GitHub issues linking to https://github.com/Azure/azureml-cheatsheets/issues" 25 | }, 26 | "link.item.label.Stack Overflow": { 27 | "message": "Stack Overflow", 28 | "description": "The label of footer link with label=Stack Overflow linking to https://stackoverflow.microsoft.com/questions/tagged/10888" 29 | }, 30 | "link.item.label.Cheat sheets": { 31 | "message": "Cheat sheets", 32 | "description": "The label of footer link with label=Cheat sheets linking to https://github.com/Azure/azureml-cheatsheets" 33 | }, 34 | "link.item.label.Azure ML Examples": { 35 | "message": "Azure ML Examples", 36 | "description": "The label of footer link with label=Azure ML Examples linking to https://github.com/Azure/azureml-examples" 37 | }, 38 | "link.item.label.Contribution": { 39 | "message": "Contribution", 40 | "description": "The label of footer link with label=Contribution linking to /docs/misc/contributing" 41 | }, 42 | "copyright": { 43 | "message": "Copyright © 2022 Microsoft Corporation", 44 | "description": "The footer copyright" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /website/i18n/ja/docusaurus-theme-classic/navbar.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": { 3 | "message": "Azure Machine Learning", 4 | "description": "The title in the navbar" 5 | }, 6 | "item.label.Python SDK": { 7 | "message": "Python SDK", 8 | "description": "Navbar item with label Python SDK" 9 | } 10 | } -------------------------------------------------------------------------------- /website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aml-cheatsheet", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "docusaurus": "docusaurus", 7 | "start": "docusaurus start", 8 | "build": "docusaurus build", 9 | "swizzle": "docusaurus swizzle", 10 | "deploy": "docusaurus deploy", 11 | "serve": "docusaurus serve" 12 | }, 13 | "dependencies": { 14 | "@docusaurus/core": "2.0.0-beta.0", 15 | "@docusaurus/preset-classic": "2.0.0-beta.0", 16 | "@mdx-js/react": "^1.6.21", 17 | "@svgr/webpack": "^5.5.0", 18 | "clsx": "^1.1.1", 19 | "file-loader": "^6.2.0", 20 | "react": "^17.0.1", 21 | "react-dom": "^17.0.1", 22 | "url-loader": "^4.1.1" 23 | }, 24 | "browserslist": { 25 | "production": [ 26 | ">0.2%", 27 | "not dead", 28 | "not op_mini all" 29 | ], 30 | "development": [ 31 | "last 1 chrome version", 32 | "last 1 firefox version", 33 | "last 1 safari version" 34 | ] 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /website/plugins/appinsights/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function (context) { 2 | 3 | return { 4 | name: 'docusaurus-plugin-appinsights', 5 | 6 | injectHtmlTags() { 7 | return { 8 | headTags: [ 9 | { 10 | tagName: 'script', 11 | innerHTML: ` 12 | !function(T,l,y){var S=T.location,u="script",k="instrumentationKey",D="ingestionendpoint",C="disableExceptionTracking",E="ai.device.",I="toLowerCase",b="crossOrigin",w="POST",e="appInsightsSDK",t=y.name||"appInsights";(y.name||T[e])&&(T[e]=t);var n=T[t]||function(d){var g=!1,f=!1,m={initialize:!0,queue:[],sv:"4",version:2,config:d};function v(e,t){var n={},a="Browser";return n[E+"id"]=a[I](),n[E+"type"]=a,n["ai.operation.name"]=S&&S.pathname||"_unknown_",n["ai.internal.sdkVersion"]="javascript:snippet_"+(m.sv||m.version),{time:function(){var e=new Date;function t(e){var t=""+e;return 1===t.length&&(t="0"+t),t}return e.getUTCFullYear()+"-"+t(1+e.getUTCMonth())+"-"+t(e.getUTCDate())+"T"+t(e.getUTCHours())+":"+t(e.getUTCMinutes())+":"+t(e.getUTCSeconds())+"."+((e.getUTCMilliseconds()/1e3).toFixed(3)+"").slice(2,5)+"Z"}(),iKey:e,name:"Microsoft.ApplicationInsights."+e.replace(/-/g,"")+"."+t,sampleRate:100,tags:n,data:{baseData:{ver:2}}}}var h=d.url||y.src;if(h){function a(e){var t,n,a,i,r,o,s,c,p,l,u;g=!0,m.queue=[],f||(f=!0,t=h,s=function(){var e={},t=d.connectionString;if(t)for(var n=t.split(";"),a=0;a Sources, saw a format error 14 | - Guessed and removed this line, and it worked! 15 | 16 | ```diff 17 | ... 18 | o.baseData.exceptions=[ 19 | { 20 | typeName : "SDKLoadFailed", 21 | message : n.replace(/\./g,"-"), 22 | hasFullStack : !1, 23 | - stack:n+"\nSnippet failed to load ["+a+"] -- Telemetry is disabled\nHelp Link: https://go.microsoft.com/fwlink/?linkid=2128109\nHost: "+(S&&S.pathname||"_unknown_")+"\nEndpoint: "+i 24 | + stack : n, 25 | parsedStack:[] 26 | } 27 | ] 28 | ... 29 | ``` -------------------------------------------------------------------------------- /website/sidebars.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | pythonSidebar: { 3 | 'Python': [ 4 | { 5 | type: 'doc', 6 | id: 'cheatsheets/python/v1/cheatsheet' 7 | }, 8 | { 9 | type: 'category', 10 | label: 'Getting Started', 11 | collapsed: false, 12 | items: ['cheatsheets/python/v1/installation'], 13 | }, 14 | { 15 | type: 'category', 16 | label: 'Azure ML Resources', 17 | collapsed: false, 18 | items: ['cheatsheets/python/v1/workspace', 'cheatsheets/python/v1/compute-targets', 'cheatsheets/python/v1/environment', 'cheatsheets/python/v1/data'], 19 | }, 20 | { 21 | type: 'category', 22 | label: 'Guides', 23 | collapsed: false, 24 | items: ['cheatsheets/python/v1/script-run-config', 'cheatsheets/python/v1/logging', 'cheatsheets/python/v1/distributed-training', 'cheatsheets/python/v1/docker-build', 'cheatsheets/python/v1/debugging', 'cheatsheets/python/v1/ci-dev','cheatsheets/python/v1/troubleshooting'] 25 | } 26 | ], 27 | }, 28 | cliSidebar: { 29 | 'CLI (preview)': [ 30 | { 31 | }, 32 | ], 33 | } 34 | }; 35 | -------------------------------------------------------------------------------- /website/src/css/custom.css: -------------------------------------------------------------------------------- 1 | /* stylelint-disable docusaurus/copyright-header */ 2 | /** 3 | * Any CSS included here will be global. The classic template 4 | * bundles Infima by default. Infima is a CSS framework designed to 5 | * work well for content-centric websites. 6 | */ 7 | 8 | /* You can override the default Infima variables here. */ 9 | :root { 10 | --ifm-color-primary: #9933ff; 11 | --ifm-color-primary-dark: rgb(33, 175, 144); 12 | --ifm-color-primary-darker: rgb(31, 165, 136); 13 | --ifm-color-primary-darkest: rgb(26, 136, 112); 14 | --ifm-color-primary-light: rgb(70, 203, 174); 15 | --ifm-color-primary-lighter: rgb(102, 212, 189); 16 | --ifm-color-primary-lightest: rgb(146, 224, 208); 17 | --ifm-code-font-size: 95%; 18 | } 19 | 20 | .docusaurus-highlight-code-line { 21 | background-color: rgb(72, 77, 91); 22 | display: block; 23 | margin: 0 calc(-1 * var(--ifm-pre-padding)); 24 | padding: 0 var(--ifm-pre-padding); 25 | } 26 | -------------------------------------------------------------------------------- /website/src/pages/index.backup: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import clsx from 'clsx'; 3 | import Layout from '@theme/Layout'; 4 | import Link from '@docusaurus/Link'; 5 | import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; 6 | import useBaseUrl from '@docusaurus/useBaseUrl'; 7 | import styles from './styles.module.css'; 8 | 9 | const features = [ 10 | { 11 | title: 'Cloud compute', 12 | //pageUrl: 'docs/cheatsheet', 13 | imageUrl: 'img/undraw_docusaurus_mountain.svg', 14 | description: ( 15 | <> 16 | Scale up and out on a variety of Azure compute targets, 17 | including high-performance GPUs for hardware acceleration. 18 | 19 | ), 20 | }, 21 | { 22 | title: 'Central tracking', 23 | //pageUrl: 'docs/cheatsheet/distributed-training', 24 | imageUrl: 'img/undraw_docusaurus_react.svg', 25 | description: ( 26 | <> 27 | Use mlflow in your jobs to log parameters, metrics, code, and artifacts 28 | in the cloud with little to no code changes. 29 | 30 | ), 31 | }, 32 | { 33 | title: 'Asset management', 34 | //pageUrl: 'docs/cheatsheet/environment', 35 | imageUrl: 'img/undraw_docusaurus_tree.svg', 36 | description: ( 37 | <> 38 | Manage models, environments, datasets, compute targets, 39 | and other assets for reuse and collaboration. 40 | 41 | ), 42 | }, 43 | ]; 44 | 45 | function Feature({imageUrl, title, description, pageUrl}) { 46 | const imgUrl = useBaseUrl(imageUrl); 47 | return ( 48 |
49 | {imgUrl && ( 50 |
51 | {title} 52 |
53 | )} 54 |

{title}

55 |

{description}

56 |
57 | ); 58 | } 59 | 60 | function Home() { 61 | const context = useDocusaurusContext(); 62 | const {siteConfig = {}} = context; 63 | return ( 64 | 67 |
68 |
69 |

{siteConfig.title}

70 |

{siteConfig.tagline}

71 |
72 | 78 | Get started! 79 | 80 |
81 |
82 |
83 |
84 | {features && features.length > 0 && ( 85 |
86 |
87 |
88 | {features.map((props, idx) => ( 89 | 90 | ))} 91 |
92 |
93 |
94 | )} 95 |
96 | 97 | ); 98 | } 99 | 100 | export default Home; 101 | -------------------------------------------------------------------------------- /website/src/pages/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import clsx from 'clsx'; 3 | import Layout from '@theme/Layout'; 4 | import Link from '@docusaurus/Link'; 5 | import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; 6 | import useBaseUrl from '@docusaurus/useBaseUrl'; 7 | import styles from './styles.module.css'; 8 | import Translate, {translate} from '@docusaurus/Translate'; 9 | 10 | const features = [ 11 | { 12 | title: Cheat Sheet, 13 | pageUrl: 'docs/cheatsheets/python/v1/cheatsheet', 14 | imageUrl: 'img/undraw_docusaurus_mountain.svg', 15 | description: ( 16 | <> 17 | 18 | A cheat sheet for common use cases with AML. 19 | Get 80% of what you need in 20% of the documentation. 20 | 21 | 22 | ), 23 | }, 24 | { 25 | title: Distributed GPU Training, 26 | pageUrl: 'docs/cheatsheets/python/v1/distributed-training', 27 | imageUrl: 'img/undraw_docusaurus_react.svg', 28 | description: ( 29 | <> 30 | 31 | Guide to getting your distributed training code running in Azure ML. 32 | 33 | 34 | ), 35 | }, 36 | { 37 | title: Environments, 38 | pageUrl: 'docs/cheatsheets/python/v1/environment', 39 | imageUrl: 'img/undraw_docusaurus_tree.svg', 40 | description: ( 41 | <> 42 | 43 | Set up and manage your Python environments and docker images 44 | in Azure ML. 45 | 46 | 47 | ), 48 | }, 49 | ]; 50 | 51 | function Feature({imageUrl, title, description, pageUrl}) { 52 | const imgUrl = useBaseUrl(imageUrl); 53 | return ( 54 |
55 | {imgUrl && ( 56 |
57 | {title} 58 |
59 | )} 60 |

{title}

61 |

{description}

62 |
63 | ); 64 | } 65 | 66 | function Home() { 67 | const context = useDocusaurusContext(); 68 | const {siteConfig = {}} = context; 69 | return ( 70 | 73 |
74 |
75 |

{siteConfig.title}

76 |

{siteConfig.tagline}

77 |
78 | 84 | Get started! 85 | 86 |
87 |
88 |
89 |
90 | {features && features.length > 0 && ( 91 |
92 |
93 |
94 | {features.map((props, idx) => ( 95 | 96 | ))} 97 |
98 |
99 |
100 | )} 101 |
102 | 103 | ); 104 | } 105 | 106 | export default Home; 107 | -------------------------------------------------------------------------------- /website/src/pages/styles.module.css: -------------------------------------------------------------------------------- 1 | /* stylelint-disable docusaurus/copyright-header */ 2 | 3 | /** 4 | * CSS files with the .module.css suffix will be treated as CSS modules 5 | * and scoped locally. 6 | */ 7 | 8 | .heroBanner { 9 | padding: 4rem 0; 10 | text-align: center; 11 | position: relative; 12 | overflow: hidden; 13 | } 14 | 15 | @media screen and (max-width: 966px) { 16 | .heroBanner { 17 | padding: 2rem; 18 | } 19 | } 20 | 21 | .buttons { 22 | display: flex; 23 | align-items: center; 24 | justify-content: center; 25 | } 26 | 27 | .features { 28 | display: flex; 29 | align-items: center; 30 | padding: 2rem 0; 31 | width: 100%; 32 | } 33 | 34 | .featureImage { 35 | height: 200px; 36 | width: 200px; 37 | } 38 | -------------------------------------------------------------------------------- /website/static/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/static/.nojekyll -------------------------------------------------------------------------------- /website/static/img/aml-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/static/img/aml-logo.png -------------------------------------------------------------------------------- /website/static/img/custom-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/static/img/custom-view.png -------------------------------------------------------------------------------- /website/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/static/img/favicon.ico -------------------------------------------------------------------------------- /website/static/img/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /website/static/img/undraw_docusaurus_tree.svg: -------------------------------------------------------------------------------- 1 | docu_tree -------------------------------------------------------------------------------- /website/static/img/view-metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/static/img/view-metrics.png -------------------------------------------------------------------------------- /website/static/img/vs-code-snippets-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azureml-cheatsheets/8a9d2d6c3d30640e48b36551138e4d2c4607b197/website/static/img/vs-code-snippets-demo.gif --------------------------------------------------------------------------------