├── .gitattributes ├── lab-6 ├── pycharm │ └── images │ │ ├── 01.png │ │ ├── 02.png │ │ ├── 03.png │ │ ├── 04.png │ │ └── 05.png ├── visual-studio │ └── images │ │ ├── 01.png │ │ ├── 02.png │ │ ├── 03.png │ │ ├── 04.png │ │ └── 05.png ├── README.md └── databricks │ └── README.md ├── starter-artifacts ├── databricks │ └── Azure ML Labs.dbc ├── pycharm │ └── azure-ml-labs │ │ ├── model.pkl │ │ ├── 01-model-training │ │ └── training │ │ │ ├── .amlignore │ │ │ ├── aml_config │ │ │ ├── project.json │ │ │ ├── conda_dependencies.yml │ │ │ ├── local.runconfig │ │ │ └── docker.runconfig │ │ │ └── train.py │ │ ├── 02-model-management │ │ ├── training │ │ │ ├── .amlignore │ │ │ ├── aml_config │ │ │ │ ├── project.json │ │ │ │ ├── conda_dependencies.yml │ │ │ │ ├── docker.runconfig │ │ │ │ └── local.runconfig │ │ │ └── train.py │ │ └── 02_model_management.py │ │ ├── .idea │ │ ├── misc.xml │ │ ├── modules.xml │ │ ├── azure-ml-labs.iml │ │ └── workspace.xml │ │ ├── 04-automl │ │ ├── outputs │ │ │ └── aml_config │ │ │ │ ├── conda_dependencies.yml │ │ │ │ └── local.runconfig │ │ └── 04_automl.py │ │ ├── myenv.yml │ │ ├── iot_score.py │ │ ├── 03-model-deployment-score.py │ │ ├── 06-deploy-to-iot-edge │ │ └── 06_deploy_to_iot_edge.py │ │ └── 05-deep-learning │ │ └── 05_deep_learning.py └── visual-studio │ ├── 04-automl │ ├── model.pkl │ ├── 04-automl.pyproj.user │ ├── outputs │ │ └── aml_config │ │ │ ├── conda_dependencies.yml │ │ │ └── local.runconfig │ ├── 04-automl.pyproj │ └── _04_automl.py │ ├── 01-model-training │ ├── training │ │ ├── .amlignore │ │ ├── aml_config │ │ │ ├── project.json │ │ │ ├── conda_dependencies.yml │ │ │ ├── docker.runconfig │ │ │ └── local.runconfig │ │ └── train.py │ ├── 01-model-training.pyproj.user │ └── 01-model-training.pyproj │ ├── 02-model-management │ ├── training │ │ ├── .amlignore │ │ ├── aml_config │ │ │ ├── project.json │ │ │ ├── conda_dependencies.yml │ │ │ ├── docker.runconfig │ │ │ └── local.runconfig │ │ └── train.py │ ├── outputs │ │ ├── Experiment-01.pkl │ │ ├── Experiment-02.pkl │ │ ├── Experiment-03.pkl │ │ ├── Experiment-02-01.pkl │ │ ├── Experiment-02-02.pkl │ │ └── Experiment-02-03.pkl │ ├── 02-model-management.pyproj │ └── _02_model_management.py │ ├── .vs │ └── azure-ml-labs │ │ └── v15 │ │ └── .suo │ ├── 06-deploy-to-iot-edge │ ├── model.pkl │ ├── myenv.yml │ ├── iot_score.py │ ├── _06_deploy_to_iot_edge.py │ └── 06-deploy-to-iot-edge.pyproj │ ├── 03-model-deployment │ ├── outputs │ │ ├── scaler.pkl │ │ └── usedcarsmodel.pkl │ ├── azureml-models │ │ └── usedcarsmodel │ │ │ ├── 21 │ │ │ └── outputs │ │ │ │ ├── scaler.pkl │ │ │ │ └── usedcarsmodel.pkl │ │ │ └── 28 │ │ │ ├── outputs │ │ │ ├── scaler.pkl │ │ │ └── usedcarsmodel.pkl │ │ │ └── usedcarsmodel.tar.gz │ ├── 03-model-deployment.pyproj.user │ ├── mydeployenv.yml │ ├── score.py │ ├── 03-model-deployment.pyproj │ └── _03_model_deployment.py │ ├── 05-deep-learning │ ├── 05-deep-learning.pyproj │ └── _05_deep_learning.py │ └── azure-ml-labs.sln ├── lab-1 ├── README.md ├── databricks │ └── README.md ├── pycharm │ └── README.md └── visual-studio │ └── README.md ├── lab-2 ├── README.md ├── databricks │ └── README.md ├── pycharm │ └── README.md └── visual-studio │ └── README.md ├── lab-3 ├── README.md ├── databricks │ └── README.md ├── pycharm │ └── README.md └── visual-studio │ └── README.md ├── lab-4 ├── README.md ├── databricks │ └── README.md ├── pycharm │ └── README.md └── visual-studio │ └── README.md ├── lab-5 ├── README.md ├── databricks │ └── README.md ├── pycharm │ └── README.md └── visual-studio │ └── README.md ├── README.md └── lab-0 └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /lab-6/pycharm/images/01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/pycharm/images/01.png -------------------------------------------------------------------------------- /lab-6/pycharm/images/02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/pycharm/images/02.png -------------------------------------------------------------------------------- /lab-6/pycharm/images/03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/pycharm/images/03.png -------------------------------------------------------------------------------- /lab-6/pycharm/images/04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/pycharm/images/04.png -------------------------------------------------------------------------------- /lab-6/pycharm/images/05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/pycharm/images/05.png -------------------------------------------------------------------------------- /lab-6/visual-studio/images/01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/visual-studio/images/01.png -------------------------------------------------------------------------------- /lab-6/visual-studio/images/02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/visual-studio/images/02.png -------------------------------------------------------------------------------- /lab-6/visual-studio/images/03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/visual-studio/images/03.png -------------------------------------------------------------------------------- /lab-6/visual-studio/images/04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/visual-studio/images/04.png -------------------------------------------------------------------------------- /lab-6/visual-studio/images/05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/lab-6/visual-studio/images/05.png -------------------------------------------------------------------------------- /starter-artifacts/databricks/Azure ML Labs.dbc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/databricks/Azure ML Labs.dbc -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/pycharm/azure-ml-labs/model.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/04-automl/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/04-automl/model.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/training/.amlignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | azureml-logs 3 | .azureml 4 | .git 5 | outputs 6 | azureml-setup 7 | docs 8 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/training/.amlignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | azureml-logs 3 | .azureml 4 | .git 5 | outputs 6 | azureml-setup 7 | docs 8 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/01-model-training/training/.amlignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | azureml-logs 3 | .azureml 4 | .git 5 | outputs 6 | azureml-setup 7 | docs 8 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/02-model-management/training/.amlignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | azureml-logs 3 | .azureml 4 | .git 5 | outputs 6 | azureml-setup 7 | docs 8 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/.vs/azure-ml-labs/v15/.suo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/.vs/azure-ml-labs/v15/.suo -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/06-deploy-to-iot-edge/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/06-deploy-to-iot-edge/model.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/outputs/scaler.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/03-model-deployment/outputs/scaler.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/outputs/Experiment-01.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/02-model-management/outputs/Experiment-01.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/outputs/Experiment-03.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/02-model-management/outputs/Experiment-03.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/outputs/usedcarsmodel.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/03-model-deployment/outputs/usedcarsmodel.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02-01.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02-01.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02-02.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02-02.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02-03.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/02-model-management/outputs/Experiment-02-03.pkl -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/21/outputs/scaler.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/21/outputs/scaler.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/28/outputs/scaler.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/28/outputs/scaler.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/28/usedcarsmodel.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/28/usedcarsmodel.tar.gz -------------------------------------------------------------------------------- /lab-1/README.md: -------------------------------------------------------------------------------- 1 | This lab can be perfomed using multiple environments, choose the environment in which you want to perform the lab below: 2 | 3 | - [Visual Studio](./visual-studio/README.md) 4 | - [PyCharm](./pycharm/README.md) 5 | - [Azure Databricks](./databricks/README.md) -------------------------------------------------------------------------------- /lab-2/README.md: -------------------------------------------------------------------------------- 1 | This lab can be perfomed using multiple environments, choose the environment in which you want to perform the lab below: 2 | 3 | - [Visual Studio](./visual-studio/README.md) 4 | - [PyCharm](./pycharm/README.md) 5 | - [Azure Databricks](./databricks/README.md) -------------------------------------------------------------------------------- /lab-3/README.md: -------------------------------------------------------------------------------- 1 | This lab can be perfomed using multiple environments, choose the environment in which you want to perform the lab below: 2 | 3 | - [Visual Studio](./visual-studio/README.md) 4 | - [PyCharm](./pycharm/README.md) 5 | - [Azure Databricks](./databricks/README.md) -------------------------------------------------------------------------------- /lab-4/README.md: -------------------------------------------------------------------------------- 1 | This lab can be perfomed using multiple environments, choose the environment in which you want to perform the lab below: 2 | 3 | - [Visual Studio](./visual-studio/README.md) 4 | - [PyCharm](./pycharm/README.md) 5 | - [Azure Databricks](./databricks/README.md) -------------------------------------------------------------------------------- /lab-5/README.md: -------------------------------------------------------------------------------- 1 | This lab can be perfomed using multiple environments, choose the environment in which you want to perform the lab below: 2 | 3 | - [Visual Studio](./visual-studio/README.md) 4 | - [PyCharm](./pycharm/README.md) 5 | - [Azure Databricks](./databricks/README.md) -------------------------------------------------------------------------------- /lab-6/README.md: -------------------------------------------------------------------------------- 1 | This lab can be perfomed using multiple environments, choose the environment in which you want to perform the lab below: 2 | 3 | - [Visual Studio](./visual-studio/README.md) 4 | - [PyCharm](./pycharm/README.md) 5 | - [Azure Databricks](./databricks/README.md) -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/21/outputs/usedcarsmodel.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/21/outputs/usedcarsmodel.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/28/outputs/usedcarsmodel.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nthacker/AML-service-labs/master/starter-artifacts/visual-studio/03-model-deployment/azureml-models/usedcarsmodel/28/outputs/usedcarsmodel.pkl -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/04-automl/04-automl.pyproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ShowAllFiles 5 | 6 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/training/aml_config/project.json: -------------------------------------------------------------------------------- 1 | {"Id": "UsedCars_Batch_02", "Scope": "/subscriptions/e223f1b3-d19b-4cfa-98e9-bc9be62717bc/resourceGroups/aml-workspace-z/providers/Microsoft.MachineLearningServices/workspaces/aml-workspace-z/projects/UsedCars_Batch_02"} -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/02-model-management/training/aml_config/project.json: -------------------------------------------------------------------------------- 1 | {"Id": "UsedCars_Batch_02", "Scope": "/subscriptions/e223f1b3-d19b-4cfa-98e9-bc9be62717bc/resourceGroups/aml-workspace-z/providers/Microsoft.MachineLearningServices/workspaces/aml-workspace-z/projects/UsedCars_Batch_02"} -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/01-model-training.pyproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ShowAllFiles 5 | 6 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/training/aml_config/project.json: -------------------------------------------------------------------------------- 1 | {"Id": "UsedCars_ManagedCompute_01", "Scope": "/subscriptions/e223f1b3-d19b-4cfa-98e9-bc9be62717bc/resourceGroups/aml-workspace-z/providers/Microsoft.MachineLearningServices/workspaces/aml-workspace-z/projects/UsedCars_ManagedCompute_01"} -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/03-model-deployment.pyproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ShowAllFiles 5 | 6 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/01-model-training/training/aml_config/project.json: -------------------------------------------------------------------------------- 1 | {"Id": "UsedCars_ManagedCompute_01", "Scope": "/subscriptions/e223f1b3-d19b-4cfa-98e9-bc9be62717bc/resourceGroups/aml-workspace-z/providers/Microsoft.MachineLearningServices/workspaces/aml-workspace-z/projects/UsedCars_ManagedCompute_01"} -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/.idea/azure-ml-labs.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/mydeployenv.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | - azureml-defaults==0.1.80 15 | - scikit-learn 16 | - numpy 17 | - pandas 18 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/04-automl/outputs/aml_config/conda_dependencies.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | # Required packages for AzureML execution, history, and data preparation. 15 | - azureml-defaults 16 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/04-automl/outputs/aml_config/conda_dependencies.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | # Required packages for AzureML execution, history, and data preparation. 15 | - azureml-defaults 16 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/training/aml_config/conda_dependencies.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | # Required packages for AzureML execution, history, and data preparation. 15 | - azureml-defaults 16 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/training/aml_config/conda_dependencies.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | # Required packages for AzureML execution, history, and data preparation. 15 | - azureml-defaults 16 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/01-model-training/training/aml_config/conda_dependencies.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | # Required packages for AzureML execution, history, and data preparation. 15 | - azureml-defaults 16 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/02-model-management/training/aml_config/conda_dependencies.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | 13 | - pip: 14 | # Required packages for AzureML execution, history, and data preparation. 15 | - azureml-defaults 16 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/myenv.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | - pip: 13 | # Required packages for AzureML execution, history, and data preparation. 14 | - --index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1 15 | - --extra-index-url https://pypi.python.org/simple 16 | - azureml-defaults==0.1.50 17 | - pandas 18 | - scikit-learn 19 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/06-deploy-to-iot-edge/myenv.yml: -------------------------------------------------------------------------------- 1 | # Conda environment specification. The dependencies defined in this file will 2 | # be automatically provisioned for runs with userManagedDependencies=False. 3 | 4 | # Details about the Conda environment file format: 5 | # https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually 6 | 7 | name: project_environment 8 | dependencies: 9 | # The python interpreter version. 10 | # Currently Azure ML only supports 3.5.2 and later. 11 | - python=3.6.2 12 | - pip: 13 | # Required packages for AzureML execution, history, and data preparation. 14 | - --index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E7501C02541B433786111FE8E140CAA1 15 | - --extra-index-url https://pypi.python.org/simple 16 | - azureml-defaults==0.1.50 17 | - pandas 18 | - scikit-learn 19 | -------------------------------------------------------------------------------- /lab-6/databricks/README.md: -------------------------------------------------------------------------------- 1 | # Lab 6 - Model Deployment to IoT Edge 2 | 3 | In this lab you deploy a trained model container to an IoT Edge device. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/databricks`. 7 | 8 | 1. From the Azure Portal, navigate to your deployed Azure Databricks workspace and select Launch Workspace. 9 | 2. Within the Workspace, using the command bar on the left, select Workspace, Users and select your username (the entry with house icon). 10 | 3. In the blade that appears, select the downwards pointing chevron next to your name, and select Import. 11 | 4. On the Import Notebooks dialog, select File. 12 | 5. Select browse and then navigate to `starter-artifacts/databricks` and select `Azure ML Labs.dbc` 13 | 5. Select Import. 14 | 6. A folder named after the archive should appear. Select that folder. 15 | 7. Navigate into the folder `06 Deploy to IoT Edge` 16 | 8. The folder will contain one or more notebooks. These are the notebooks you will use in completing this lab. Start with the first notebook and follow the instructions. 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /lab-5/databricks/README.md: -------------------------------------------------------------------------------- 1 | # Lab 5 - Deep Learning 2 | 3 | In this lab you train deep learning models built with Keras and a Tensorflow backend that utilize GPUs with the Azure Machine Learning service. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/databricks`. 7 | 8 | 1. From the Azure Portal, navigate to your deployed Azure Databricks workspace and select Launch Workspace. 9 | 2. Within the Workspace, using the command bar on the left, select Workspace, Users and select your username (the entry with house icon). 10 | 3. In the blade that appears, select the downwards pointing chevron next to your name, and select Import. 11 | 4. On the Import Notebooks dialog, select File. 12 | 5. Select browse and then navigate to `starter-artifacts/databricks` and select `Azure ML Labs.dbc` 13 | 5. Select Import. 14 | 6. A folder named after the archive should appear. Select that folder. 15 | 7. Navigate into the folder `05 Deep Learning` 16 | 8. The folder will contain one or more notebooks. These are the notebooks you will use in completing this lab. Start with the first notebook and follow the instructions. 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /lab-3/databricks/README.md: -------------------------------------------------------------------------------- 1 | # Lab 3 - Model Deployment using Azure Machine Learning service 2 | 3 | In this lab you will deploy a trained model to containers using an Azure Container Instance and and Azure Kubernetes Service using the Azure Machine Learning SDK. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/databricks`. 7 | 8 | 1. From the Azure Portal, navigate to your deployed Azure Databricks workspace and select Launch Workspace. 9 | 2. Within the Workspace, using the command bar on the left, select Workspace, Users and select your username (the entry with house icon). 10 | 3. In the blade that appears, select the downwards pointing chevron next to your name, and select Import. 11 | 4. On the Import Notebooks dialog, select File. 12 | 5. Select browse and then navigate to `starter-artifacts/databricks` and select `azure-ml-labs.dbc` 13 | 5. Select Import. 14 | 6. A folder named after the archive should appear. Select that folder. 15 | 7. Navigate into the folder `03 Model Deployment` 16 | 8. The folder will contain one or more notebooks. These are the notebooks you will use in completing this lab. Start with the first notebook and follow the instructions. 17 | 18 | 19 | -------------------------------------------------------------------------------- /lab-1/databricks/README.md: -------------------------------------------------------------------------------- 1 | # Lab 1 - Training a Machine Learning Model using Azure Machine Learning service 2 | 3 | In this lab you will setup the Azure Machine Learning service from code and create a classical machine learning model that logs metrics collected during model training. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/databricks`. 7 | 8 | 1. From the Azure Portal, navigate to your deployed Azure Databricks workspace and select Launch Workspace. 9 | 2. Within the Workspace, using the command bar on the left, select Workspace, Users and select your username (the entry with house icon). 10 | 3. In the blade that appears, select the downwards pointing chevron next to your name, and select Import. 11 | 4. On the Import Notebooks dialog, select File. 12 | 5. Select browse and then navigate to `starter-artifacts/databricks` and select `azure-ml-labs.dbc` 13 | 5. Select Import. 14 | 6. A folder named after the archive should appear. Select that folder. 15 | 7. Navigate into the folder `01 Model Training` 16 | 8. The folder will contain one or more notebooks. These are the notebooks you will use in completing this lab. Start with the first notebook and follow the instructions. 17 | 18 | 19 | -------------------------------------------------------------------------------- /lab-2/databricks/README.md: -------------------------------------------------------------------------------- 1 | # Lab 2 - Using Azure Machine Learning service Model Versioning and Run History 2 | 3 | In this lab you will use the capabilities of the Azure Machine Learning service to collect model performance metrics and to capture model version, as well as query the experimentation run history to retrieve captured metrics. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/databricks`. 7 | 8 | 1. From the Azure Portal, navigate to your deployed Azure Databricks workspace and select Launch Workspace. 9 | 2. Within the Workspace, using the command bar on the left, select Workspace, Users and select your username (the entry with house icon). 10 | 3. In the blade that appears, select the downwards pointing chevron next to your name, and select Import. 11 | 4. On the Import Notebooks dialog, select File. 12 | 5. Select browse and then navigate to `starter-artifacts/databricks` and select `azure-ml-labs.dbc` 13 | 5. Select Import. 14 | 6. A folder named after the archive should appear. Select that folder. 15 | 7. Navigate into the folder `02 Model Management` 16 | 8. The folder will contain one or more notebooks. These are the notebooks you will use in completing this lab. Start with the first notebook and follow the instructions. 17 | -------------------------------------------------------------------------------- /lab-4/databricks/README.md: -------------------------------------------------------------------------------- 1 | # Lab 4 - Model Training with AutoML 2 | 3 | In this lab you will us the automated machine learning (Auto ML) capabilities within the Azure Machine Learning service to automatically train multiple models with varying algorithms and hyperparameters, select the best performing model and register that model. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/databricks`. 7 | 8 | 1. From the Azure Portal, navigate to your deployed Azure Databricks workspace and select Launch Workspace. 9 | 2. Within the Workspace, using the command bar on the left, select Workspace, Users and select your username (the entry with house icon). 10 | 3. In the blade that appears, select the downwards pointing chevron next to your name, and select Import. 11 | 4. On the Import Notebooks dialog, select File. 12 | 5. Select browse and then navigate to `starter-artifacts/databricks` and select `azure-ml-labs.dbc` 13 | 5. Select Import. 14 | 6. A folder named after the archive should appear. Select that folder. 15 | 7. Navigate into the folder `04 AutoML` 16 | 8. The folder will contain one or more notebooks. These are the notebooks you will use in completing this lab. Start with the first notebook and follow the instructions. 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/iot_score.py: -------------------------------------------------------------------------------- 1 | # This script generates the scoring file 2 | # with the init and run functions needed to 3 | # operationalize the anomaly detection sample 4 | 5 | import pickle 6 | import json 7 | import pandas 8 | from sklearn.externals import joblib 9 | from sklearn.linear_model import Ridge 10 | from azureml.core.model import Model 11 | 12 | def init(): 13 | global model 14 | # note here "sklearn_regression_model.pkl" is the name of the model registered under 15 | # this is a different behavior than before when the code is run locally, even though the code is the same. 16 | model_path = Model.get_model_path('model.pkl') 17 | # deserialize the model file back into a sklearn model 18 | model = joblib.load(model_path) 19 | 20 | # note you can pass in multiple rows for scoring 21 | def run(input_str): 22 | try: 23 | input_json = json.loads(input_str) 24 | input_df = pandas.DataFrame([[input_json['machine']['temperature'],input_json['machine']['pressure'],input_json['ambient']['temperature'],input_json['ambient']['humidity']]]) 25 | pred = model.predict(input_df) 26 | print("Prediction is ", pred[0]) 27 | except Exception as e: 28 | result = str(e) 29 | 30 | if pred[0] == '1': 31 | input_json['anomaly']=True 32 | else: 33 | input_json['anomaly']=False 34 | 35 | return [json.dumps(input_json)] 36 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/06-deploy-to-iot-edge/iot_score.py: -------------------------------------------------------------------------------- 1 | # This script generates the scoring file 2 | # with the init and run functions needed to 3 | # operationalize the anomaly detection sample 4 | 5 | import pickle 6 | import json 7 | import pandas 8 | from sklearn.externals import joblib 9 | from sklearn.linear_model import Ridge 10 | from azureml.core.model import Model 11 | 12 | def init(): 13 | global model 14 | # note here "sklearn_regression_model.pkl" is the name of the model registered under 15 | # this is a different behavior than before when the code is run locally, even though the code is the same. 16 | model_path = Model.get_model_path('model.pkl') 17 | # deserialize the model file back into a sklearn model 18 | model = joblib.load(model_path) 19 | 20 | # note you can pass in multiple rows for scoring 21 | def run(input_str): 22 | try: 23 | input_json = json.loads(input_str) 24 | input_df = pandas.DataFrame([[input_json['machine']['temperature'],input_json['machine']['pressure'],input_json['ambient']['temperature'],input_json['ambient']['humidity']]]) 25 | pred = model.predict(input_df) 26 | print("Prediction is ", pred[0]) 27 | except Exception as e: 28 | result = str(e) 29 | 30 | if pred[0] == '1': 31 | input_json['anomaly']=True 32 | else: 33 | input_json['anomaly']=False 34 | 35 | return [json.dumps(input_json)] 36 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/score.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import numpy as np 4 | import pandas as pd 5 | from sklearn import linear_model 6 | from sklearn.externals import joblib 7 | from sklearn.preprocessing import StandardScaler 8 | from sklearn.metrics import accuracy_score 9 | from sklearn.model_selection import train_test_split 10 | from azureml.core import Run 11 | from azureml.core import Workspace 12 | from azureml.core.run import Run 13 | from azureml.core.experiment import Experiment 14 | import pickle 15 | from sklearn.externals import joblib 16 | 17 | def init(): 18 | try: 19 | # One-time initialization of predictive model and scaler 20 | from azureml.core.model import Model 21 | 22 | global trainedModel 23 | global scaler 24 | 25 | model_name = "usedcarsmodel" 26 | model_path = Model.get_model_path(model_name) 27 | print('Looking for models in: ', model_path) 28 | 29 | trainedModel = pickle.load(open(os.path.join(model_path,'usedcarsmodel.pkl'), 'rb')) 30 | 31 | scaler = pickle.load(open(os.path.join(model_path,'scaler.pkl'),'rb')) 32 | 33 | except Exception as e: 34 | print('Exception during init: ', str(e)) 35 | 36 | def run(input_json): 37 | try: 38 | inputs = json.loads(input_json) 39 | 40 | #Scale the input 41 | scaled_input = scaler.transform(inputs) 42 | 43 | #Get the scored result 44 | prediction = json.dumps(trainedModel.predict(scaled_input).tolist()) 45 | 46 | except Exception as e: 47 | prediction = str(e) 48 | return prediction -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/03-model-deployment-score.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import numpy as np 4 | import pandas as pd 5 | from sklearn import linear_model 6 | from sklearn.externals import joblib 7 | from sklearn.preprocessing import StandardScaler 8 | from sklearn.metrics import accuracy_score 9 | from sklearn.model_selection import train_test_split 10 | from azureml.core import Run 11 | from azureml.core import Workspace 12 | from azureml.core.run import Run 13 | from azureml.core.experiment import Experiment 14 | import pickle 15 | from sklearn.externals import joblib 16 | 17 | def init(): 18 | try: 19 | # One-time initialization of predictive model and scaler 20 | from azureml.core.model import Model 21 | 22 | global trainedModel 23 | global scaler 24 | 25 | model_name = "usedcarsmodel" 26 | model_path = Model.get_model_path(model_name) 27 | print('Looking for models in: ', model_path) 28 | 29 | trainedModel = pickle.load(open(os.path.join(model_path,'usedcarsmodel.pkl'), 'rb')) 30 | 31 | scaler = pickle.load(open(os.path.join(model_path,'scaler.pkl'),'rb')) 32 | 33 | except Exception as e: 34 | print('Exception during init: ', str(e)) 35 | 36 | def run(input_json): 37 | try: 38 | inputs = json.loads(input_json) 39 | 40 | #Scale the input 41 | scaled_input = scaler.transform(inputs) 42 | 43 | #Get the scored result 44 | prediction = json.dumps(trainedModel.predict(scaled_input).tolist()) 45 | 46 | except Exception as e: 47 | prediction = str(e) 48 | return prediction -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Machine Learning service Labs 2 | 3 | This repo contains labs that show how to use the Azure Machine Learning service with the Python SDK. These labs will teach you how to perform the training locally within a Deep Learning Virtual Machine (DLVM) as well scale out model training by using Azure Batch AI workspaces or Azure Databricks. Each lab provides instructions for you to peform them using the environment of your choice- Visual Studio (with the Visual Studio Tools for AI), PyCharm or Azure Databricks. 4 | 5 | The following labs are available: 6 | - [Lab 0:](./lab-0/README.md) Setting up your environment. If a lab environment has not be provided for you, this lab provides the instructions to get started in your own Azure Subscription. 7 | - [Lab 1:](./lab-1/README.md) Setup the Azure Machine Learning service from code and create a classical machine learning model that logs metrics collected during model training. 8 | - [Lab 2:](./lab-2/README.md) Use the capabilities of the Azure Machine Learning service to collect model performance metrics and to capture model version, as well as query the experimentation run history to retrieve captured metrics. 9 | - [Lab 3:](./lab-3/README.md) Deploying a trained model to containers using an Azure Container Instance and and Azure Kubernetes Service using Azure Machine Learning. 10 | - [Lab 4:](./lab-4/README.md) Using the automated machine learning (Auto ML) capabilities within the Azure Machine Learning service to automatically train multiple models with varying algorithms and hyperparameters and then select the best performing model. 11 | - [Lab 5:](./lab-5/README.md) Training deep learning models built with Keras and a Tensorflow backend that utilize GPUs with the Azure Machine Learning service. 12 | - [Lab 6:](./lab-6/README.md) Deploy a trained model container to an IoT Edge device via the Azure Machine Learning service. 13 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/06-deploy-to-iot-edge/_06_deploy_to_iot_edge.py: -------------------------------------------------------------------------------- 1 | # Step 1 - Create or retrieve your Azure ML Workspace 2 | ##################################################### 3 | from azureml.core import Workspace 4 | from azureml.core.model import Model 5 | #Provide the Subscription ID of your existing Azure subscription 6 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 7 | 8 | #Provide values for the Resource Group and Workspace that will be created 9 | resource_group = "aml-workspace-z" 10 | workspace_name = "aml-workspace-z" 11 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 12 | 13 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 14 | ws = Workspace.create( 15 | name = workspace_name, 16 | subscription_id = subscription_id, 17 | resource_group = resource_group, 18 | location = workspace_region, 19 | exist_ok = True) 20 | 21 | print("Workspace Provisioning complete.") 22 | 23 | 24 | # Step 2 - Build the ContainerImage for the IoT Edge Module 25 | ########################################################### 26 | from azureml.core.image import ContainerImage, Image 27 | 28 | runtime = "python" 29 | driver_file = "iot_score.py" 30 | conda_file = "myenv.yml" 31 | 32 | image_config = ContainerImage.image_configuration(execution_script = driver_file, 33 | runtime = runtime, 34 | conda_file = conda_file) 35 | 36 | model = Model.register(model_path = "model.pkl", 37 | model_name = "iot_model.pkl", 38 | workspace = ws) 39 | 40 | image = Image.create(name = "iotimage", 41 | # this is the model object 42 | models = [model], 43 | image_config = image_config, 44 | workspace = ws) 45 | image.wait_for_creation(show_output = True) 46 | 47 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/06-deploy-to-iot-edge/06_deploy_to_iot_edge.py: -------------------------------------------------------------------------------- 1 | # Step 1 - Create or retrieve your Azure ML Workspace 2 | ##################################################### 3 | from azureml.core import Workspace 4 | from azureml.core.model import Model 5 | #Provide the Subscription ID of your existing Azure subscription 6 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 7 | 8 | #Provide values for the Resource Group and Workspace that will be created 9 | resource_group = "aml-workspace-z" 10 | workspace_name = "aml-workspace-z" 11 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 12 | 13 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 14 | ws = Workspace.create( 15 | name = workspace_name, 16 | subscription_id = subscription_id, 17 | resource_group = resource_group, 18 | location = workspace_region, 19 | exist_ok = True) 20 | 21 | print("Workspace Provisioning complete.") 22 | 23 | 24 | # Step 2 - Build the ContainerImage for the IoT Edge Module 25 | ########################################################### 26 | from azureml.core.image import ContainerImage, Image 27 | 28 | runtime = "python" 29 | driver_file = "iot_score.py" 30 | conda_file = "myenv.yml" 31 | 32 | image_config = ContainerImage.image_configuration(execution_script = driver_file, 33 | runtime = runtime, 34 | conda_file = conda_file) 35 | 36 | model = Model.register(model_path = "model.pkl", 37 | model_name = "iot_model.pkl", 38 | workspace = ws) 39 | 40 | image = Image.create(name = "iotimage", 41 | # this is the model object 42 | models = [model], 43 | image_config = image_config, 44 | workspace = ws) 45 | image.wait_for_creation(show_output = True) 46 | 47 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/05-deep-learning/05-deep-learning.pyproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | 549c3865-479b-454a-9ea9-1a11500e1d7d 7 | . 8 | {8B0147FC-7A0C-481F-99C4-C90CCB52692E};{888888A0-9F3D-457C-B088-3A5042F75D52} 9 | _05_deep_learning.py 10 | 11 | 12 | . 13 | . 14 | --input_dir . --output_dir . 15 | 05-deep-learning 16 | 05-deep-learning 17 | CondaEnv|CondaEnv|AzureML 18 | 19 | 20 | true 21 | false 22 | 23 | 24 | true 25 | false 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/04-automl/04-automl.pyproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | 692e2964-b5b8-4f23-8c78-c1bdf024108c 7 | . 8 | {8B0147FC-7A0C-481F-99C4-C90CCB52692E};{888888A0-9F3D-457C-B088-3A5042F75D52} 9 | _04_automl.py 10 | 11 | 12 | . 13 | . 14 | --input_dir . --output_dir . 15 | 04-automl 16 | 04-automl 17 | CondaEnv|CondaEnv|AzureML 18 | 19 | 20 | true 21 | false 22 | 23 | 24 | true 25 | false 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/06-deploy-to-iot-edge/06-deploy-to-iot-edge.pyproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | 4cec97c0-1883-4240-b2e8-6ab72d1d6363 7 | . 8 | {8B0147FC-7A0C-481F-99C4-C90CCB52692E};{888888A0-9F3D-457C-B088-3A5042F75D52} 9 | _06_deploy_to_iot_edge.py 10 | 11 | 12 | . 13 | . 14 | --input_dir . --output_dir . 15 | 06-deploy-to-iot-edge 16 | 06-deploy-to-iot-edge 17 | CondaEnv|CondaEnv|AzureML 18 | 19 | 20 | true 21 | false 22 | 23 | 24 | true 25 | false 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/02-model-management.pyproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | 894d287b-11c2-4f93-b751-3a18facb6f62 7 | . 8 | {8B0147FC-7A0C-481F-99C4-C90CCB52692E};{888888A0-9F3D-457C-B088-3A5042F75D52} 9 | _02_model_management.py 10 | 11 | 12 | . 13 | . 14 | --input_dir . --output_dir . 15 | 02-model-management 16 | 02-model-management 17 | CondaEnv|CondaEnv|AzureML 18 | 19 | 20 | true 21 | false 22 | 23 | 24 | true 25 | false 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/03-model-deployment/03-model-deployment.pyproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | 87fa8233-9906-4555-8be8-d26de841fbd1 7 | . 8 | {8B0147FC-7A0C-481F-99C4-C90CCB52692E};{888888A0-9F3D-457C-B088-3A5042F75D52} 9 | _03_model_deployment.py 10 | 11 | 12 | . 13 | . 14 | --input_dir . --output_dir . 15 | 03-model-deployment 16 | 03-model-deployment 17 | CondaEnv|CondaEnv|AzureML 18 | 19 | 20 | true 21 | false 22 | 23 | 24 | true 25 | false 26 | 27 | 28 | 29 | Code 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /lab-5/pycharm/README.md: -------------------------------------------------------------------------------- 1 | # Lab 5 - Deep Learning 2 | 3 | In this lab you train deep learning models built with Keras and a Tensorflow backend that utilize GPUs with the Azure Machine Learning service. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/pycharm`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 1. Within PyCharm, select open Existing Project and navigate to the directory where you cloned the repo to open the project `azure-ml-labs`. 10 | 2. In the Project window, expand External Libraries. You should see one environment called `` where the path points to your AzureML Anaconda environment. This Anaconda environment will be used when you execute code. 11 | 3. In the Project tool window expand the folder `05-deep-learning`. 12 | 4. Open `05_deep_learning.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 13 | 14 | 15 | ## Exercise 2 - Train an autoencoder using GPU 16 | 1. Begin with Step 1 and read thru the code in Step 1. Here you will use Keras to define an autoencoder. Don't get hung up on the details of constructing the auto-encoder. The point of this lab is to show you how to train neural networks using GPU's. Select Step 1 and type `Alt + Shift + Enter` to execute the selected code in the Python Console window. In the output, verify that `K.tensorflow_backend._get_available_gpus()` returned an entry describing a GPU available in your environment. 17 | 2. Once you have your autoencoder model structured, you need to train the the underlying neural network. Training this model on regular CPU's will take hours. However, you can execute this same code in an environment with GPU's for better performance. Select and execute Step 2. How long did your training take? 18 | 3. With a trained auto-encoder in hand, try using the model by selecting and executing Step 3. 19 | 20 | ## Exercise 3 - Register the neural network model with Azure Machine Learning 21 | 1. In Step 4, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments to create or retrieve your workspace. Observe that you can register a neural network model with Azure Machine Learning in exactly the same way you would register a classical machine learning model. Execute Step 4 to register the model. 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/01-model-training.pyproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | 3dd87218-537c-45fb-b7db-9735f5694c83 7 | . 8 | {8B0147FC-7A0C-481F-99C4-C90CCB52692E};{888888A0-9F3D-457C-B088-3A5042F75D52} 9 | _01_model_training.py 10 | 11 | 12 | . 13 | . 14 | --input_dir . --output_dir . 15 | 01-model-training 16 | 01-model-training 17 | CondaEnv|CondaEnv|AzureML 18 | 19 | 20 | true 21 | false 22 | 23 | 24 | true 25 | false 26 | 27 | 28 | 29 | Code 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /lab-5/visual-studio/README.md: -------------------------------------------------------------------------------- 1 | # Lab 5 - Deep Learning 2 | 3 | In this lab you train deep learning models built with Keras and a Tensorflow backend that utilize GPUs with the Azure Machine Learning service. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/visual-studio`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 10 | 1. Navigate to the directory where you cloned the repo and then open the solution `azure-ml-labs.sln`. 11 | 2. With the solution open in Visual Studio, look at Solution Explorer and expand the project `05-deep-learning`. 12 | 3. Under that, expand Python Environments. You should see one environment called `AzureML (3.6, 64-bit)`. This Anaconda environment will be used when you execute code either by running the script using F5 or by executing Python code in the Python Interactive Window. 13 | 4. Open `_05_deep_learning.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 14 | 15 | 16 | ## Exercise 2 - Train an autoencoder using GPU 17 | 1. Begin with Step 1 and read thru the code in Step 1. Here you will use Keras to define an autoencoder. Don't get hung up on the details of constructing the auto-encoder. The point of this lab is to show you how to train neural networks using GPU's. Select Step 1 and type `Control + Enter` to execute the selected code in the Python Immediate Window. In the output, verify that `K.tensorflow_backend._get_available_gpus()` returned an entry describing a GPU available in your environment. 18 | 2. Once you have your autoencoder model structured, you need to train the the underlying neural network. Training this model on regular CPU's will take hours. However, you can execute this same code in an environment with GPU's for better performance. Select and execute Step 2. How long did your training take? 19 | 3. With a trained auto-encoder in hand, try using the model by selecting and executing Step 3. 20 | 21 | ## Exercise 3 - Register the neural network model with Azure Machine Learning 22 | 1. In Step 4, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments to create or retrieve your workspace. Observe that you can register a neural network model with Azure Machine Learning in exactly the same way you would register a classical machine learning model. Execute Step 4 to register the model. 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/training/train.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | 7 | from sklearn import linear_model 8 | from sklearn.externals import joblib 9 | from sklearn.preprocessing import StandardScaler 10 | from sklearn.metrics import accuracy_score 11 | 12 | from azureml.core import Run 13 | 14 | # let user feed in 2 parameters, the location of the data files (from datastore), and the training set percentage to use 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('--data-folder', type=str, dest='data_folder', help='data folder mounting point') 17 | parser.add_argument('--training-set-percentage', type=float, dest='training_set_percentage', default=0.25, help='percentage of dataset to use for training') 18 | args = parser.parse_args() 19 | 20 | data_folder = os.path.join(args.data_folder, 'used_cars') 21 | print('Data folder:', data_folder) 22 | data_csv_path = os.path.join(data_folder, 'UsedCars_Clean.csv') 23 | print('Path to CSV file dataset:' + data_csv_path) 24 | 25 | # Load the data 26 | #df = pd.read_csv('UsedCars_Clean.csv', delimiter=',') 27 | df = pd.read_csv(data_csv_path) 28 | df['Affordable'] = np.where(df['Price']<12000, 1, 0) 29 | df_affordability = df[["Age","KM", "Affordable"]] 30 | 31 | 32 | # Now experiment with different training subsets 33 | from sklearn.model_selection import train_test_split 34 | full_X = df_affordability[["Age", "KM"]] 35 | full_Y = df_affordability[["Affordable"]] 36 | 37 | def train_eval_model(full_X, full_Y,training_set_percentage): 38 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 39 | 40 | scaler = StandardScaler() 41 | X_scaled = scaler.fit_transform(train_X) 42 | clf = linear_model.LogisticRegression(C=1) 43 | clf.fit(X_scaled, train_Y) 44 | 45 | scaled_inputs = scaler.transform(test_X) 46 | predictions = clf.predict(scaled_inputs) 47 | score = accuracy_score(test_Y, predictions) 48 | 49 | return (clf, score) 50 | 51 | # Acquire the current run 52 | run = Run.get_context() 53 | 54 | 55 | training_set_percentage = args.training_set_percentage 56 | model, score = train_eval_model(full_X, full_Y, training_set_percentage) 57 | print("With %0.2f percent of data, model accuracy reached %0.4f." % (training_set_percentage, score)) 58 | run.log("Training_Set_Percentage", training_set_percentage) 59 | run.log("Accuracy", score) 60 | 61 | 62 | # note file saved in the outputs folder is automatically uploaded into experiment record 63 | joblib.dump(value=model, filename='outputs/model.pkl') 64 | 65 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/01-model-training/training/train.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | 7 | from sklearn import linear_model 8 | from sklearn.externals import joblib 9 | from sklearn.preprocessing import StandardScaler 10 | from sklearn.metrics import accuracy_score 11 | 12 | from azureml.core import Run 13 | 14 | # let user feed in 2 parameters, the location of the data files (from datastore), and the training set percentage to use 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('--data-folder', type=str, dest='data_folder', help='data folder mounting point') 17 | parser.add_argument('--training-set-percentage', type=float, dest='training_set_percentage', default=0.25, help='percentage of dataset to use for training') 18 | args = parser.parse_args() 19 | 20 | data_folder = os.path.join(args.data_folder, 'used_cars') 21 | print('Data folder:', data_folder) 22 | data_csv_path = os.path.join(data_folder, 'UsedCars_Clean.csv') 23 | print('Path to CSV file dataset:' + data_csv_path) 24 | 25 | # Load the data 26 | #df = pd.read_csv('UsedCars_Clean.csv', delimiter=',') 27 | df = pd.read_csv(data_csv_path) 28 | df['Affordable'] = np.where(df['Price']<12000, 1, 0) 29 | df_affordability = df[["Age","KM", "Affordable"]] 30 | 31 | 32 | # Now experiment with different training subsets 33 | from sklearn.model_selection import train_test_split 34 | full_X = df_affordability[["Age", "KM"]] 35 | full_Y = df_affordability[["Affordable"]] 36 | 37 | def train_eval_model(full_X, full_Y,training_set_percentage): 38 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 39 | 40 | scaler = StandardScaler() 41 | X_scaled = scaler.fit_transform(train_X) 42 | clf = linear_model.LogisticRegression(C=1) 43 | clf.fit(X_scaled, train_Y) 44 | 45 | scaled_inputs = scaler.transform(test_X) 46 | predictions = clf.predict(scaled_inputs) 47 | score = accuracy_score(test_Y, predictions) 48 | 49 | return (clf, score) 50 | 51 | # Acquire the current run 52 | run = Run.get_context() 53 | 54 | 55 | training_set_percentage = args.training_set_percentage 56 | model, score = train_eval_model(full_X, full_Y, training_set_percentage) 57 | print("With %0.2f percent of data, model accuracy reached %0.4f." % (training_set_percentage, score)) 58 | run.log("Training_Set_Percentage", training_set_percentage) 59 | run.log("Accuracy", score) 60 | 61 | 62 | # note file saved in the outputs folder is automatically uploaded into experiment record 63 | joblib.dump(value=model, filename='outputs/model.pkl') 64 | 65 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/azure-ml-labs.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28010.2026 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "01-model-training", "01-model-training\01-model-training.pyproj", "{3DD87218-537C-45FB-B7DB-9735F5694C83}" 7 | EndProject 8 | Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "02-model-management", "02-model-management\02-model-management.pyproj", "{894D287B-11C2-4F93-B751-3A18FACB6F62}" 9 | EndProject 10 | Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "03-model-deployment", "03-model-deployment\03-model-deployment.pyproj", "{87FA8233-9906-4555-8BE8-D26DE841FBD1}" 11 | EndProject 12 | Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "04-automl", "04-automl\04-automl.pyproj", "{692E2964-B5B8-4F23-8C78-C1BDF024108C}" 13 | EndProject 14 | Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "05-deep-learning", "05-deep-learning\05-deep-learning.pyproj", "{549C3865-479B-454A-9EA9-1A11500E1D7D}" 15 | EndProject 16 | Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "06-deploy-to-iot-edge", "06-deploy-to-iot-edge\06-deploy-to-iot-edge.pyproj", "{4CEC97C0-1883-4240-B2E8-6AB72D1D6363}" 17 | EndProject 18 | Global 19 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 20 | Debug|Any CPU = Debug|Any CPU 21 | Release|Any CPU = Release|Any CPU 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {3DD87218-537C-45FB-B7DB-9735F5694C83}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {3DD87218-537C-45FB-B7DB-9735F5694C83}.Release|Any CPU.ActiveCfg = Release|Any CPU 26 | {894D287B-11C2-4F93-B751-3A18FACB6F62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {894D287B-11C2-4F93-B751-3A18FACB6F62}.Release|Any CPU.ActiveCfg = Release|Any CPU 28 | {87FA8233-9906-4555-8BE8-D26DE841FBD1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {87FA8233-9906-4555-8BE8-D26DE841FBD1}.Release|Any CPU.ActiveCfg = Release|Any CPU 30 | {692E2964-B5B8-4F23-8C78-C1BDF024108C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 31 | {692E2964-B5B8-4F23-8C78-C1BDF024108C}.Release|Any CPU.ActiveCfg = Release|Any CPU 32 | {549C3865-479B-454A-9EA9-1A11500E1D7D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {549C3865-479B-454A-9EA9-1A11500E1D7D}.Release|Any CPU.ActiveCfg = Release|Any CPU 34 | {4CEC97C0-1883-4240-B2E8-6AB72D1D6363}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 35 | {4CEC97C0-1883-4240-B2E8-6AB72D1D6363}.Release|Any CPU.ActiveCfg = Release|Any CPU 36 | EndGlobalSection 37 | GlobalSection(SolutionProperties) = preSolution 38 | HideSolutionNode = FALSE 39 | EndGlobalSection 40 | GlobalSection(ExtensibilityGlobals) = postSolution 41 | SolutionGuid = {516C5110-A7CA-477D-87AF-F2EAB057F5C9} 42 | EndGlobalSection 43 | EndGlobal 44 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/training/train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import numpy as np 4 | import pandas as pd 5 | 6 | from sklearn import linear_model 7 | from sklearn.externals import joblib 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.metrics import accuracy_score 10 | 11 | from azureml.core import Run 12 | from sklearn.model_selection import train_test_split 13 | 14 | # let user feed in 2 parameters, the location of the data files (from datastore), and the training set percentage to use 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('--data-folder', type=str, dest='data_folder', help='data folder mounting point') 17 | parser.add_argument('--training-set-percentage', type=float, dest='training_set_percentage', default=0.25, help='percentage of dataset to use for training') 18 | args = parser.parse_args() 19 | 20 | data_folder = os.path.join(args.data_folder, 'used_cars') 21 | print('Data folder:', data_folder) 22 | data_csv_path = os.path.join(data_folder, 'UsedCars_Affordability.csv') 23 | print('Path to CSV file dataset:' + data_csv_path) 24 | 25 | # Load the data 26 | df_affordability = pd.read_csv(data_csv_path) 27 | full_X = df_affordability[["Age", "KM"]] 28 | full_Y = df_affordability[["Affordable"]] 29 | 30 | def train_eval_register_model(full_X, full_Y,training_set_percentage): 31 | 32 | # Acquire the current run 33 | run = Run.get_context() 34 | 35 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 36 | 37 | scaler = StandardScaler() 38 | X_scaled = scaler.fit_transform(train_X) 39 | clf = linear_model.LogisticRegression(C=1) 40 | clf.fit(X_scaled, train_Y) 41 | 42 | scaled_inputs = scaler.transform(test_X) 43 | predictions = clf.predict(scaled_inputs) 44 | score = accuracy_score(test_Y, predictions) 45 | 46 | print("With %0.2f percent of data, model accuracy reached %0.4f." % (training_set_percentage, score)) 47 | 48 | # Log the training metrics to Azure Machine Learning service run history 49 | run.log("Training_Set_Percentage", training_set_percentage) 50 | run.log("Accuracy", score) 51 | run.complete() 52 | 53 | # note file saved in the outputs folder is automatically uploaded into experiment record 54 | joblib.dump(value=clf, filename='outputs/model.pkl') 55 | 56 | # Register this version of the model with Azure Machine Learning service 57 | registered_model = run.register_model(model_name='usedcarsmodel', model_path='outputs/model.pkl') 58 | 59 | print(registered_model.name, registered_model.id, registered_model.version, sep = '\t') 60 | 61 | return (clf, score) 62 | 63 | 64 | training_set_percentage = 0.75 65 | model, score = train_eval_register_model(full_X, full_Y, training_set_percentage) 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/02-model-management/training/train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import numpy as np 4 | import pandas as pd 5 | 6 | from sklearn import linear_model 7 | from sklearn.externals import joblib 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.metrics import accuracy_score 10 | 11 | from azureml.core import Run 12 | from sklearn.model_selection import train_test_split 13 | 14 | # let user feed in 2 parameters, the location of the data files (from datastore), and the training set percentage to use 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('--data-folder', type=str, dest='data_folder', help='data folder mounting point') 17 | parser.add_argument('--training-set-percentage', type=float, dest='training_set_percentage', default=0.25, help='percentage of dataset to use for training') 18 | args = parser.parse_args() 19 | 20 | data_folder = os.path.join(args.data_folder, 'used_cars') 21 | print('Data folder:', data_folder) 22 | data_csv_path = os.path.join(data_folder, 'UsedCars_Affordability.csv') 23 | print('Path to CSV file dataset:' + data_csv_path) 24 | 25 | # Load the data 26 | df_affordability = pd.read_csv(data_csv_path) 27 | full_X = df_affordability[["Age", "KM"]] 28 | full_Y = df_affordability[["Affordable"]] 29 | 30 | def train_eval_register_model(full_X, full_Y,training_set_percentage): 31 | 32 | # Acquire the current run 33 | run = Run.get_context() 34 | 35 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 36 | 37 | scaler = StandardScaler() 38 | X_scaled = scaler.fit_transform(train_X) 39 | clf = linear_model.LogisticRegression(C=1) 40 | clf.fit(X_scaled, train_Y) 41 | 42 | scaled_inputs = scaler.transform(test_X) 43 | predictions = clf.predict(scaled_inputs) 44 | score = accuracy_score(test_Y, predictions) 45 | 46 | print("With %0.2f percent of data, model accuracy reached %0.4f." % (training_set_percentage, score)) 47 | 48 | # Log the training metrics to Azure Machine Learning service run history 49 | run.log("Training_Set_Percentage", training_set_percentage) 50 | run.log("Accuracy", score) 51 | run.complete() 52 | 53 | # note file saved in the outputs folder is automatically uploaded into experiment record 54 | joblib.dump(value=clf, filename='outputs/model.pkl') 55 | 56 | # Register this version of the model with Azure Machine Learning service 57 | registered_model = run.register_model(model_name='usedcarsmodel', model_path='outputs/model.pkl') 58 | 59 | print(registered_model.name, registered_model.id, registered_model.version, sep = '\t') 60 | 61 | return (clf, score) 62 | 63 | 64 | training_set_percentage = 0.75 65 | model, score = train_eval_register_model(full_X, full_Y, training_set_percentage) 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /lab-4/pycharm/README.md: -------------------------------------------------------------------------------- 1 | # Lab 4 - Model Training with AutoML 2 | 3 | In this lab you will us the automated machine learning (Auto ML) capabilities within the Azure Machine Learning service to automatically train multiple models with varying algorithms and hyperparameters, select the best performing model and register that model. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/pycharm`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 1. Within PyCharm, select open Existing Project and navigate to the directory where you cloned the repo to open the project `azure-ml-labs`. 10 | 2. In the Project window, expand External Libraries. You should see one environment called `` where the path points to your AzureML Anaconda environment. This Anaconda environment will be used when you execute code. 11 | 3. In the Project tool window expand the folder `04-automl`. 12 | 4. Expand the `data` folder. This folder contains the CSV file `UsedCars_Affordability.csv` which contains the complete data set with labels (Affordable is 1 for affordable, 0 for not affordable). 13 | 5. Open `04_automl.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 14 | 15 | 16 | ## Exercise 2 - Train a model using AutoML 17 | This lab builds upon the lessons learned in the previous lab, but is self contained so you work thru this lab without having to run a previous lab. 18 | 1. Begin with Step 1. In this step you are loading the data prepared in previous labs and acquiring (or creating) an instance of your Azure Machine Learning Workspace. In this step, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. Read thru the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Control + Enter` to execute the selected code in the Python Immediate Window. 19 | 2. To train a model using AutoML you need only provide a configuration for AutoML that defines items such as the type of model (classification or regression), the performance metric to optimize, exit criteria in terms of max training time and iterations and desired performance, any algorithms that should not be used, and the path into which to output the results. This configuration is specified using the `AutomMLConfig` class, which is then used to drive the submission of an experiment via `experiment.submit`. When AutoML finishes the parent run, you can easily get the best performing run and model from the returned run object by using `run.get_output()`. Select and execute Step 2 to define the helper function that wraps the AutoML job submission. 20 | 3. In Step 3, you invoke the AutoML job. Select and execute Step 3. 21 | 4. Try out the best model by using Step 4. 22 | 23 | ## Exercise 3 - Register an AutoML created model 24 | 1. You can register models created by AutoML with Azure Machine Learning just as you would any other model. Select and execute Step 5 to register this model. 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /lab-4/visual-studio/README.md: -------------------------------------------------------------------------------- 1 | # Lab 4 - Model Training with AutoML 2 | 3 | In this lab you will us the automated machine learning (Auto ML) capabilities within the Azure Machine Learning service to automatically train multiple models with varying algorithms and hyperparameters, select the best performing model and register that model. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/visual-studio`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 10 | 1. Navigate to the directory where you cloned the repo and then open the solution `azure-ml-labs.sln`. 11 | 2. With the solution open in Visual Studio, look at Solution Explorer and expand the project `04-automl`. 12 | 3. Under that, expand Python Environments. You should see one environment called `AzureML (3.6, 64-bit)`. This Anaconda environment will be used when you execute code either by running the script using F5 or by executing Python code in the Python Interactive Window. 13 | 4. Expand the `data` folder. This folder contains one CSV files. `UsedCars_Affordability.csv` contains the complete, cleaned data set with labels (Affordable is 1 for affordable, 0 for not affordable). 14 | 5. Open `_04_automl.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 15 | 16 | 17 | ## Exercise 2 - Train a model using AutoML 18 | This lab builds upon the lessons learned in the previous lab, but is self contained so you work thru this lab without having to run a previous lab. 19 | 1. Begin with Step 1. In this step you are loading the data prepared in previous labs and acquiring (or creating) an instance of your Azure Machine Learning Workspace. In this step, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. Read thru the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Control + Enter` to execute the selected code in the Python Immediate Window. 20 | 2. To train a model using AutoML you need only provide a configuration for AutoML that defines items such as the type of model (classification or regression), the performance metric to optimize, exit criteria in terms of max training time and iterations and desired performance, any algorithms that should not be used, and the path into which to output the results. This configuration is specified using the `AutomMLConfig` class, which is then used to drive the submission of an experiment via `experiment.submit`. When AutoML finishes the parent run, you can easily get the best performing run and model from the returned run object by using `run.get_output()`. Select and execute Step 2 to define the helper function that wraps the AutoML job submission. 21 | 3. In Step 3, you invoke the AutoML job. Select and execute Step 3. 22 | 4. Try out the best model by using Step 4. 23 | 24 | ## Exercise 3 - Register an AutoML created model 25 | 1. You can register models created by AutoML with Azure Machine Learning just as you would any other model. Select and execute Step 5 to register this model. 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /lab-2/pycharm/README.md: -------------------------------------------------------------------------------- 1 | # Lab 2 - Using Azure Machine Learning service Model Versioning and Run History 2 | 3 | In this lab you will use the capabilities of the Azure Machine Learning service to collect model performance metrics and to capture model version, as well as query the experimentation run history to retrieve captured metrics. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/pycharm`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 1. Within PyCharm, select open Existing Project and navigate to the directory where you cloned the repo to open the project `azure-ml-labs`. 10 | 2. In the Project window, expand External Libraries. You should see one environment called `` where the path points to your AzureML Anaconda environment. This Anaconda environment will be used when you execute code. 11 | 3. In the Project tool window expand the folder `02-model-training`. 12 | 4. Expand the `data` folder. This folder contains the CSV file `UsedCars_Affordability.csv` which contains the complete data set with labels (Affordable is 1 for affordable, 0 for not affordable). 13 | 5. Expand `training`. This folder contains train.py which will be used later in the lab to train the model using a remote cluster provided by Azure Batch AI. 14 | 6. Open `02_model_management.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 15 | 16 | 17 | ## Exercise 2 - Train a simple model locally 18 | This lab builds upon the lessons learned in the previous lab, but is self contained so you work thru this lab without having to run a previous lab. As such Steps 1, 2 and 3 in the lab are not explored in detail as their goal is to setup a few experiment runs, which was covered in detail in Lab 1. 19 | 1. Read thru and select the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Alt + Shift + Enter` to execute the selected code in the Python Immediate Window. Take a moment to look at the data loaded into the Pandas Dataframe - it contains data about used cars such as the price (in dollars), age (in years), KM (kilometers driven) and other attributes like weather it is automatic transimission, the number of doors, and the weight. 20 | 2. In Step 2, we will define a helper method that locally trains, evaluates and then registers the trained model with Azure Machine Learning. Select and execute Step #2. 21 | 3. In Step 3, we retrieve an existing Azure Machine Learning Workspace (or create a new one if desired). In this step, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. With the Workspace retrieved, we will train 3 different models using different subsets of the training data. Select and execute Step #3. 22 | 23 | 24 | ## Exercise 3 - Use Azure Machine Learning to query for performance metrics 25 | 1. As was demonstrated in the previous lab, you can use the Workspace to get a list of Experiments. You can also query for a particular Experiment by name. With an Experiment in hand, you review all runs associated with that Experiment and retrieve the metrics associated with each run. Select and execute Step #4 to see this process. What was the accuracy of the only run for Experiment-02-03? 26 | 27 | 28 | ## Exercise 4 - Remotely train a model in Azure Batch AI 29 | 1. Remote model training was covered in the previous lab. Execute Step #5 to create or retreive your Azure Batch AI cluster and the submit to it a model training job. Wait for the run to complete before proceeding to the next exercise. 30 | 31 | ## Exercise 5 - Retrieve metrics for the remote Run 32 | 1. You can easily retrieve the metrics for a Run executed remotely by using `run` object returned by the call to `Experiment.submit`. Execute Step 6 to retrieve metrics for the run you just executed. What was the accuracy of the run? 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /lab-2/visual-studio/README.md: -------------------------------------------------------------------------------- 1 | # Lab 2 - Using Azure Machine Learning service Model Versioning and Run History 2 | 3 | In this lab you will use the capabilities of the Azure Machine Learning service to collect model performance metrics and to capture model version, as well as query the experimentation run history to retrieve captured metrics. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/visual-studio`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 10 | 1. Navigate to the directory where you cloned the repo and then open the solution `azure-ml-labs.sln`. 11 | 2. With the solution open in Visual Studio, look at Solution Explorer and expand the project `02-model-management`. 12 | 3. Under that, expand Python Environments. You should see one environment called `AzureML (3.6, 64-bit)`. This Anaconda environment will be used when you execute code either by running the script using F5 or by executing Python code in the Python Interactive Window. 13 | 4. Expand the `data` folder. This folder contains one CSV files. `UsedCars_Affordability.csv` contains the complete, cleaned data set with labels (Affordable is 1 for affordable, 0 for not affordable). 14 | 5. Expand `training`. This folder contains train.py which will be used later in the lab to train the model using a remote cluster provided by Azure Batch AI. 15 | 6. Open `_02_model_management.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 16 | 17 | 18 | ## Exercise 2 - Train a simple model locally 19 | This lab builds upon the lessons learned in the previous lab, but is self contained so you work thru this lab without having to run a previous lab. As such Steps 1, 2 and 3 in the lab are not explored in detail as their goal is to setup a few experiment runs, which was covered in detail in Lab 1. 20 | 1. Read thru and select the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Control + Enter` to execute the selected code in the Python Immediate Window. Take a moment to look at the data loaded into the Pandas Dataframe - it contains data about used cars such as the price (in dollars), age (in years), KM (kilometers driven) and other attributes like weather it is automatic transimission, the number of doors, and the weight. 21 | 2. In Step 2, we will define a helper method that locally trains, evaluates and then registers the trained model with Azure Machine Learning. Select and execute Step #2. 22 | 3. In Step 3, we retrieve an existing Azure Machine Learning Workspace (or create a new one if desired). In this step, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. With the Workspace retrieved, we will train 3 different models using different subsets of the training data. Select and execute Step #3. 23 | 24 | 25 | ## Exercise 3 - Use Azure Machine Learning to query for performance metrics 26 | 1. As was demonstrated in the previous lab, you can use the Workspace to get a list of Experiments. You can also query for a particular Experiment by name. With an Experiment in hand, you review all runs associated with that Experiment and retrieve the metrics associated with each run. Select and execute Step #4 to see this process. What was the accuracy of the only run for Experiment-02-03? 27 | 28 | 29 | ## Exercise 4 - Remotely train a model in Azure Batch AI 30 | 1. Remote model training was covered in the previous lab. Execute Step #5 to create or retreive your Azure Batch AI cluster and the submit to it a model training job. Wait for the run to complete before proceeding to the next exercise. 31 | 32 | ## Exercise 5 - Retrieve metrics for the remote Run 33 | 1. You can easily retrieve the metrics for a Run executed remotely by using `run` object returned by the call to `Experiment.submit`. Execute Step 6 to retrieve metrics for the run you just executed. What was the accuracy of the run? 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/04-automl/outputs/aml_config/local.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: Python 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: false 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/04-automl/outputs/aml_config/local.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: Python 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: false 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/training/aml_config/docker.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: PySpark 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: true 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/01-model-training/training/aml_config/local.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: Python 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: false 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/training/aml_config/docker.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: PySpark 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: true 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/training/aml_config/local.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: Python 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: false 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/01-model-training/training/aml_config/local.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: Python 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: false 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/01-model-training/training/aml_config/docker.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: PySpark 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: true 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/02-model-management/training/aml_config/docker.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: PySpark 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: true 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/02-model-management/training/aml_config/local.runconfig: -------------------------------------------------------------------------------- 1 | # The script to run. 2 | script: train.py 3 | # The arguments to the script file. 4 | arguments: [] 5 | # The name of the compute target to use for this run. 6 | target: local 7 | # Framework to execute inside. Allowed values are "Python" , "PySpark", "TensorFlowParameterServer" and "PythonMPI". 8 | framework: Python 9 | # Automatically prepare the run environment as part of the run itself. 10 | autoPrepareEnvironment: true 11 | # Maximum allowed duration for the run. 12 | maxRunDurationSeconds: 13 | # Number of nodes to use for running job. 14 | nodeCount: 1 15 | # Environment details. 16 | environment: 17 | # Environment variables set for the run. 18 | environmentVariables: 19 | EXAMPLE_ENV_VAR: EXAMPLE_VALUE 20 | # Python details 21 | python: 22 | # user_managed_dependencies=True indicates that the environmentwill be user managed. False indicates that AzureML willmanage the user environment. 23 | userManagedDependencies: false 24 | # The python interpreter path 25 | interpreterPath: python 26 | # Path to the conda dependencies file to use for this run. If a project 27 | # contains multiple programs with different sets of dependencies, it may be 28 | # convenient to manage those environments with separate files. 29 | condaDependenciesFile: aml_config/conda_dependencies.yml 30 | # Docker details 31 | docker: 32 | # Set True to perform this run inside a Docker container. 33 | enabled: false 34 | # Base image used for Docker-based runs. 35 | baseImage: mcr.microsoft.com/azureml/base:0.1.4 36 | # Set False if necessary to work around shared volume bugs. 37 | sharedVolumes: true 38 | # Run with NVidia Docker extension to support GPUs. 39 | gpuSupport: false 40 | # Extra arguments to the Docker run command. 41 | arguments: [] 42 | # Image registry that contains the base image. 43 | baseImageRegistry: 44 | # DNS name or IP address of azure container registry(ACR) 45 | address: 46 | # The username for ACR 47 | username: 48 | # The password for ACR 49 | password: 50 | # Spark details 51 | spark: 52 | # List of spark repositories. 53 | repositories: 54 | - https://mmlspark.azureedge.net/maven 55 | packages: 56 | - group: com.microsoft.ml.spark 57 | artifact: mmlspark_2.11 58 | version: '0.12' 59 | precachePackages: true 60 | # Databricks details 61 | databricks: 62 | # List of maven libraries. 63 | mavenLibraries: [] 64 | # List of PyPi libraries 65 | pypiLibraries: [] 66 | # List of RCran libraries 67 | rcranLibraries: [] 68 | # List of JAR libraries 69 | jarLibraries: [] 70 | # List of Egg libraries 71 | eggLibraries: [] 72 | # History details. 73 | history: 74 | # Enable history tracking -- this allows status, logs, metrics, and outputs 75 | # to be collected for a run. 76 | outputCollection: true 77 | # whether to take snapshots for history. 78 | snapshotProject: true 79 | # Spark configuration details. 80 | spark: 81 | configuration: 82 | spark.app.name: Azure ML Experiment 83 | spark.yarn.maxAppAttempts: 1 84 | # HDI details. 85 | hdi: 86 | # Yarn deploy mode. Options are cluster and client. 87 | yarnDeployMode: cluster 88 | # Tensorflow details. 89 | tensorflow: 90 | # The number of worker tasks. 91 | workerCount: 1 92 | # The number of parameter server tasks. 93 | parameterServerCount: 1 94 | # Mpi details. 95 | mpi: 96 | # When using MPI, number of processes per node. 97 | processCountPerNode: 1 98 | # Container instance details. 99 | containerInstance: 100 | # Number of cores to allocate for the container 101 | cpuCores: 1 102 | # Memory to allocate for the container in GB 103 | memoryGb: 4 104 | # Azure region for the container; defaults to the same as workspace 105 | region: 106 | # data reference configuration details 107 | dataReferences: {} 108 | # AmlCompute details. 109 | amlcompute: 110 | # VM size of the Cluster to be created.Allowed values are Azure vm sizes.The list of vm sizes is available in 'https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs 111 | vmSize: 112 | # VM priority of the Cluster to be created.Allowed values are "dedicated" , "lowpriority". 113 | vmPriority: 114 | # A bool that indicates if the cluster has to be retained after job completion. 115 | retainCluster: false 116 | # Name of the cluster to be created. If not specified, runId will be used as cluster name. 117 | name: 118 | # Maximum number of nodes in the AmlCompute cluster to be created. Minimum number of nodes will always be set to 0. 119 | clusterMaxNodeCount: 1 120 | -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/04-automl/_04_automl.py: -------------------------------------------------------------------------------- 1 | # Step 1 - Load training data and prepare Workspace 2 | ################################################### 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | from sklearn import linear_model 7 | from sklearn.externals import joblib 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.metrics import accuracy_score 10 | from sklearn.model_selection import train_test_split 11 | import azureml 12 | from azureml.core import Run 13 | from azureml.core import Workspace 14 | from azureml.core.run import Run 15 | from azureml.core.experiment import Experiment 16 | from azureml.train.automl import AutoMLConfig 17 | import pickle 18 | 19 | # Verify AML SDK Installed 20 | # view version history at https://pypi.org/project/azureml-sdk/#history 21 | print("SDK Version:", azureml.core.VERSION) 22 | 23 | 24 | # Load our training data set 25 | print("Current working directory is ", os.path.abspath(os.path.curdir)) 26 | df_affordability = pd.read_csv('data/UsedCars_Affordability.csv', delimiter=',') 27 | print(df_affordability) 28 | 29 | full_X = df_affordability[["Age", "KM"]] 30 | full_Y = df_affordability[["Affordable"]] 31 | 32 | # Create a Workspace 33 | #Provide the Subscription ID of your existing Azure subscription 34 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 35 | 36 | #Provide values for the Resource Group and Workspace that will be created 37 | resource_group = "aml-workspace-z" 38 | workspace_name = "aml-workspace-z" 39 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 40 | 41 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 42 | ws = Workspace.create( 43 | name = workspace_name, 44 | subscription_id = subscription_id, 45 | resource_group = resource_group, 46 | location = workspace_region, 47 | exist_ok = True) 48 | 49 | print("Workspace Provisioning complete.") 50 | 51 | 52 | 53 | # Step 2 - Define a helper method that will use AutoML to train multiple models and pick the best one 54 | ##################################################################################################### 55 | def auto_train_model(ws, experiment_name, model_name, full_X, full_Y,training_set_percentage, training_target_accuracy): 56 | 57 | # start a training run by defining an experiment 58 | experiment = Experiment(ws, experiment_name) 59 | 60 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 61 | 62 | train_Y_array = train_Y.values.flatten() 63 | 64 | # Configure the automated ML job 65 | # The model training is configured to run on the local machine 66 | # The values for all settings are documented at https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train 67 | # Notice we no longer have to scale the input values, as Auto ML will try various data scaling approaches automatically 68 | Automl_config = AutoMLConfig(task = 'classification', 69 | primary_metric = 'accuracy', 70 | max_time_sec = 12000, 71 | iterations = 20, 72 | n_cross_validations = 3, 73 | exit_score = training_target_accuracy, 74 | blacklist_algos = ['kNN','LinearSVM'], 75 | X = train_X, 76 | y = train_Y_array, 77 | path='.\\outputs') 78 | 79 | # Execute the job 80 | run = experiment.submit(Automl_config, show_output=True) 81 | 82 | # Get the run with the highest accuracy value. 83 | best_run, best_model = run.get_output() 84 | 85 | return (best_model, run, best_run) 86 | 87 | 88 | # Step 3 - Execute the AutoML driven training 89 | ############################################# 90 | experiment_name = "Experiment-AutoML-04" 91 | model_name = "usedcarsmodel" 92 | training_set_percentage = 0.50 93 | training_target_accuracy = 0.93 94 | best_model, run, best_run = auto_train_model(ws, experiment_name, model_name, full_X, full_Y, training_set_percentage, training_target_accuracy) 95 | 96 | # Examine some of the metrics for the best performing run 97 | import pprint 98 | pprint.pprint({k: v for k, v in best_run.get_metrics().items() if isinstance(v, float)}) 99 | 100 | # Step 4 - Try the best model 101 | ############################# 102 | age = 60 103 | km = 40000 104 | print(best_model.predict( [[age,km]] )) 105 | 106 | # Step 5 - Register the best performing model for later use and deployment 107 | ################################################################# 108 | # notice the use of the root run (not best_run) to register the best model 109 | run.register_model(description='AutoML trained used cars classifier') -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/04-automl/04_automl.py: -------------------------------------------------------------------------------- 1 | # Step 1 - Load training data and prepare Workspace 2 | ################################################### 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | from sklearn import linear_model 7 | from sklearn.externals import joblib 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.metrics import accuracy_score 10 | from sklearn.model_selection import train_test_split 11 | import azureml 12 | from azureml.core import Run 13 | from azureml.core import Workspace 14 | from azureml.core.run import Run 15 | from azureml.core.experiment import Experiment 16 | from azureml.train.automl import AutoMLConfig 17 | import pickle 18 | 19 | # Verify AML SDK Installed 20 | # view version history at https://pypi.org/project/azureml-sdk/#history 21 | print("SDK Version:", azureml.core.VERSION) 22 | 23 | 24 | # Load our training data set 25 | print("Current working directory is ", os.path.abspath(os.path.curdir)) 26 | df_affordability = pd.read_csv('./04-automl/data/UsedCars_Affordability.csv', delimiter=',') 27 | print(df_affordability) 28 | 29 | full_X = df_affordability[["Age", "KM"]] 30 | full_Y = df_affordability[["Affordable"]] 31 | 32 | # Create a Workspace 33 | #Provide the Subscription ID of your existing Azure subscription 34 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 35 | 36 | #Provide values for the Resource Group and Workspace that will be created 37 | resource_group = "aml-workspace-z" 38 | workspace_name = "aml-workspace-z" 39 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 40 | 41 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 42 | ws = Workspace.create( 43 | name = workspace_name, 44 | subscription_id = subscription_id, 45 | resource_group = resource_group, 46 | location = workspace_region, 47 | exist_ok = True) 48 | 49 | print("Workspace Provisioning complete.") 50 | 51 | 52 | 53 | # Step 2 - Define a helper method that will use AutoML to train multiple models and pick the best one 54 | ##################################################################################################### 55 | def auto_train_model(ws, experiment_name, model_name, full_X, full_Y,training_set_percentage, training_target_accuracy): 56 | 57 | # start a training run by defining an experiment 58 | experiment = Experiment(ws, experiment_name) 59 | 60 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 61 | 62 | train_Y_array = train_Y.values.flatten() 63 | 64 | # Configure the automated ML job 65 | # The model training is configured to run on the local machine 66 | # The values for all settings are documented at https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-configure-auto-train 67 | # Notice we no longer have to scale the input values, as Auto ML will try various data scaling approaches automatically 68 | Automl_config = AutoMLConfig(task = 'classification', 69 | primary_metric = 'accuracy', 70 | max_time_sec = 12000, 71 | iterations = 20, 72 | n_cross_validations = 3, 73 | exit_score = training_target_accuracy, 74 | blacklist_algos = ['kNN','LinearSVM'], 75 | X = train_X, 76 | y = train_Y_array, 77 | path='./04-automl/outputs') 78 | 79 | # Execute the job 80 | run = experiment.submit(Automl_config, show_output=True) 81 | 82 | # Get the run with the highest accuracy value. 83 | best_run, best_model = run.get_output() 84 | 85 | return (best_model, run, best_run) 86 | 87 | 88 | # Step 3 - Execute the AutoML driven training 89 | ############################################# 90 | experiment_name = "Experiment-AutoML-04" 91 | model_name = "usedcarsmodel" 92 | training_set_percentage = 0.50 93 | training_target_accuracy = 0.93 94 | best_model, run, best_run = auto_train_model(ws, experiment_name, model_name, full_X, full_Y, training_set_percentage, training_target_accuracy) 95 | 96 | # Examine some of the metrics for the best performing run 97 | import pprint 98 | pprint.pprint({k: v for k, v in best_run.get_metrics().items() if isinstance(v, float)}) 99 | 100 | # Step 4 - Try the best model 101 | ############################# 102 | age = 60 103 | km = 40000 104 | print(best_model.predict( [[age,km]] )) 105 | 106 | # Step 5 - Register the best performing model for later use and deployment 107 | ################################################################# 108 | # notice the use of the root run (not best_run) to register the best model 109 | run.register_model(description='AutoML trained used cars classifier') -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/05-deep-learning/05_deep_learning.py: -------------------------------------------------------------------------------- 1 | # Step 1 - Train an autoencoder using GPU 2 | ######################################### 3 | import numpy as np 4 | import keras 5 | from keras import backend as K 6 | from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D 7 | from keras.models import Model 8 | from keras.utils.vis_utils import plot_model 9 | import azureml 10 | from azureml.core import Run 11 | from azureml.core import Workspace 12 | from azureml.core.run import Run 13 | from azureml.core.experiment import Experiment 14 | from azureml.core.model import AMLModel 15 | import pickle 16 | 17 | # Verify we have a GPU available 18 | # The output of the following should not be an empty array 19 | # If you get an empty array back, it means no GPU was detected, which might mean you need to 20 | # uninstall keras/tensorflow/tensorflow-gpu and then reinstall tensorflow-gpu and keras 21 | K.tensorflow_backend._get_available_gpus() 22 | 23 | # We use Fashion mnist dataset 24 | from keras.datasets import fashion_mnist 25 | 26 | # We download and load the data 27 | (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() 28 | 29 | 30 | # Build the encoder 31 | input_img = Input(shape=(28, 28, 1)) 32 | 33 | x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img) 34 | x = MaxPooling2D((2, 2), padding='same')(x) 35 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) 36 | x = MaxPooling2D((2, 2), padding='same')(x) 37 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) 38 | encoded_feature_vector = MaxPooling2D((2, 2), padding='same', name='feature_vector')(x) 39 | 40 | # at this point the representation is (4, 4, 8) i.e. 128-dimensional compressed feature vector 41 | 42 | # Build the decoder 43 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(encoded_feature_vector) 44 | x = UpSampling2D((2, 2))(x) 45 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) 46 | x = UpSampling2D((2, 2))(x) 47 | x = Conv2D(16, (3, 3), activation='relu')(x) 48 | x = UpSampling2D((2, 2))(x) 49 | decoded_output = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x) 50 | 51 | 52 | # The first model is autoencoder model, it takes the input image and results in a decoded image 53 | autoencoder_model = Model(input_img, decoded_output) 54 | # Compile the first model 55 | autoencoder_model.compile(optimizer='adadelta', loss='binary_crossentropy') 56 | 57 | 58 | # The second NN model is only a half of the first model, it take the input image and gives the encoded vector as output 59 | encoder_model = Model(inputs=autoencoder_model.input, 60 | outputs=autoencoder_model.get_layer('feature_vector').output) # <---- take the output from the feature vector 61 | # Compile the second model 62 | encoder_model.compile(optimizer='adadelta', loss='binary_crossentropy') 63 | 64 | # We need to scale the image from [0-255] to [0-1] for better performance of activation functions 65 | x_train = x_train / 255. 66 | x_test = x_test / 255. 67 | 68 | 69 | # We train the NN in batches (groups of images), so we reshape the dataset 70 | x_train = np.reshape(x_train, (len(x_train), 28, 28, 1)) 71 | x_test = np.reshape(x_test, (len(x_test), 28, 28, 1)) 72 | 73 | print("Train dataset size is {0}".format(x_train.shape)) 74 | print("Test dataset size is {0}".format(x_test.shape)) 75 | 76 | # Step 2 - Train a neural network 77 | ################################# 78 | # It takes several minutes to train this neural network, depending on the configuration of your cluster. 79 | learning_history=autoencoder_model.fit(x=x_train, y=x_train, epochs=10, batch_size=128, 80 | shuffle=True, validation_data=(x_test, x_test), verbose=1) 81 | 82 | # Step 3 - Test the model 83 | ########################## 84 | encoded_decoded_image=autoencoder_model.predict(x_test) 85 | 86 | 87 | # Step 4 - Export and Register the model 88 | ######################################## 89 | #Provide the Subscription ID of your existing Azure subscription 90 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 91 | 92 | #Provide values for the Resource Group and Workspace that will be created 93 | resource_group = "aml-workspace-z" 94 | workspace_name = "aml-workspace-z" 95 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 96 | 97 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 98 | ws = Workspace.create( 99 | name = workspace_name, 100 | subscription_id = subscription_id, 101 | resource_group = resource_group, 102 | location = workspace_region, 103 | exist_ok = True) 104 | 105 | print("Workspace Provisioning complete.") 106 | 107 | # Serialize the model to a pickle file in the outputs folder 108 | model_name = 'autoencoder' 109 | output_model_path = 'outputs/' + model_name + '.pkl' 110 | pickle.dump(learning_history,open(output_model_path,'wb')) 111 | print('Exported model to ', output_model_path) 112 | 113 | # notice for the model_path, we supply the name of the outputs folder without a trailing slash 114 | registered_model = Model.register(model_path='outputs', model_name=model_name, workspace=ws) -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/05-deep-learning/_05_deep_learning.py: -------------------------------------------------------------------------------- 1 | # Step 1 - Train an autoencoder using GPU 2 | ######################################### 3 | import numpy as np 4 | import keras 5 | from keras import backend as K 6 | from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D 7 | from keras.models import Model 8 | from keras.utils.vis_utils import plot_model 9 | import azureml 10 | from azureml.core import Run 11 | from azureml.core import Workspace 12 | from azureml.core.run import Run 13 | from azureml.core.experiment import Experiment 14 | from azureml.core.model import Model as AMLModel 15 | import pickle 16 | 17 | # Verify we have a GPU available 18 | # The output of the following should not be an empty array 19 | # If you get an empty array back, it means no GPU was detected, which might mean you need to 20 | # uninstall keras/tensorflow/tensorflow-gpu and then reinstall tensorflow-gpu and keras 21 | K.tensorflow_backend._get_available_gpus() 22 | 23 | # We use Fashion mnist dataset 24 | from keras.datasets import fashion_mnist 25 | 26 | # We download and load the data 27 | (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data() 28 | 29 | 30 | # Build the encoder 31 | input_img = Input(shape=(28, 28, 1)) # This will be the input 👚 32 | 33 | x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img) 34 | x = MaxPooling2D((2, 2), padding='same')(x) 35 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) 36 | x = MaxPooling2D((2, 2), padding='same')(x) 37 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) 38 | encoded_feature_vector = MaxPooling2D((2, 2), padding='same', name='feature_vector')(x) # <---- name your feature vector somehow 39 | 40 | # at this point the representation is (4, 4, 8) i.e. 128-dimensional compressed feature vector 41 | 42 | # Build the decoder 43 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(encoded_feature_vector) 44 | x = UpSampling2D((2, 2))(x) 45 | x = Conv2D(8, (3, 3), activation='relu', padding='same')(x) 46 | x = UpSampling2D((2, 2))(x) 47 | x = Conv2D(16, (3, 3), activation='relu')(x) 48 | x = UpSampling2D((2, 2))(x) 49 | decoded_output = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x) 50 | 51 | 52 | # The first model is autoencoder model, it takes the input image and results in a decoded image 53 | autoencoder_model = Model(input_img, decoded_output) 54 | # Compile the first model 55 | autoencoder_model.compile(optimizer='adadelta', loss='binary_crossentropy') 56 | 57 | 58 | # The second NN model is only a half of the first model, it take the input image and gives the encoded vector as output 59 | encoder_model = Model(inputs=autoencoder_model.input, 60 | outputs=autoencoder_model.get_layer('feature_vector').output) # <---- take the output from the feature vector 61 | # Compile the second model 62 | encoder_model.compile(optimizer='adadelta', loss='binary_crossentropy') 63 | 64 | # We need to scale the image from [0-255] to [0-1] for better performance of activation functions 65 | x_train = x_train / 255. 66 | x_test = x_test / 255. 67 | 68 | 69 | # We train the NN in batches (groups of images), so we reshape the dataset 70 | x_train = np.reshape(x_train, (len(x_train), 28, 28, 1)) 71 | x_test = np.reshape(x_test, (len(x_test), 28, 28, 1)) 72 | 73 | print("Train dataset size is {0}".format(x_train.shape)) 74 | print("Test dataset size is {0}".format(x_test.shape)) 75 | 76 | # Step 2 - Train a neural network 77 | ################################# 78 | # It takes several minutes to train this neural network, depending on the configuration of your cluster. 79 | learning_history=autoencoder_model.fit(x=x_train, y=x_train, epochs=10, batch_size=128, 80 | shuffle=True, validation_data=(x_test, x_test), verbose=1) 81 | 82 | # Step 3 - Test the model 83 | ########################## 84 | encoded_decoded_image=autoencoder_model.predict(x_test) 85 | 86 | 87 | # Step 4 - Export and Register the model 88 | ######################################## 89 | #Provide the Subscription ID of your existing Azure subscription 90 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 91 | 92 | #Provide values for the Resource Group and Workspace that will be created 93 | resource_group = "aml-workspace-z" 94 | workspace_name = "aml-workspace-z" 95 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 96 | 97 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 98 | ws = Workspace.create( 99 | name = workspace_name, 100 | subscription_id = subscription_id, 101 | resource_group = resource_group, 102 | location = workspace_region, 103 | exist_ok = True) 104 | 105 | print("Workspace Provisioning complete.") 106 | 107 | # Serialize the model to a pickle file in the outputs folder 108 | model_name = 'autoencoder' 109 | output_model_path = 'outputs/' + model_name + '.pkl' 110 | pickle.dump(learning_history,open(output_model_path,'wb')) 111 | print('Exported model to ', output_model_path) 112 | 113 | # notice for the model_path, we supply the name of the outputs folder without a trailing slash 114 | registered_model = AMLModel.register(model_path='outputs', model_name=model_name, workspace=ws) -------------------------------------------------------------------------------- /lab-0/README.md: -------------------------------------------------------------------------------- 1 | # Lab 0: Setting up your environment 2 | 3 | If a lab environmnet has not be provided for you, this lab provides the instructions to get started in your own Azure Subscription. 4 | 5 | The following summarizes the lab requirements if you want to setup your own environment (for example, on your local machine). If this is your first time peforming these labs, it is highly recommended you follow the Quick Start instructions below rather that setup your own environment from scratch. 6 | 7 | The labs have the following requirements: 8 | - Azure subscription. You will need a valid and active Azure account to complete this Azure lab. If you do not have one, you can sign up for a [free trial](https://azure.microsoft.com/en-us/free/). 9 | - One of the following environments: 10 | - Visual Studio 2017 and the Visual Studio Tools for AI 11 | - PyCharm 12 | - Azure Databricks Workspace 13 | - For the deep learning lab, you will need a VM or cluster with CPU capabilities. 14 | 15 | Depending on which environment you use, there are different requirements. These are summarized as follows: 16 | - Visual Studio 2017 and PyCharm 17 | - A Python 3.x Anaconda environment named `azureml` with: 18 | - The latest version of the Azure Machine Learning Python SDK installed. Use `pip install --upgrade azureml-sdk[notebooks,automl] azureml-dataprep` to install the latest version. 19 | - The following pip installable packages: 20 | - numpy, pandas, scikitlearn, keras and tensorflow-gpu 21 | - For the deep learning lab, make sure you have your GPU drivers properly installed. 22 | - Azure Databricks 23 | - An Azure Databricks Workspace 24 | - A two-node Azure Databricks cluster with the following Python libraries attached: 25 | - numpy, pandas, scikitlearn, keras and tensorflow-gpu 26 | 27 | The following sections describe the setup process for each environment. 28 | 29 | # Quickstart: Visual Studio 2017 and PyCharm 30 | The quickest way to get going with the labs is to deploy the Deep Learning Virtual Machine (DLVM). 31 | 32 | 1. Follow these instructions for [creating an Deep Learning Virtual Machine](https://docs.microsoft.com/en-us/azure/machine-learning/data-science-virtual-machine/provision-deep-learning-dsvm). Be sure you do the following: 33 | - OS Type: Select Windows 2016 34 | - Location: Choose a region that provides NC series VM's, such as East US, East US 2, North Central US, South Central US and West US 2. Be sure to visit the [Azure Products by Region](https://azure.microsoft.com/regions/services/) website for the latest. 35 | - Virtual Machine size: NC6 36 | 2. Once the VM is ready, download the remote desktop (RDP) file from the Overview blade of your VM in the Azure Portal and login. If you are unfamiliar with this process, see [Connect to a VM](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/connect-logon). 37 | 3. Once you are connected to your DLVM, open the Start Menu and run `Anaconda Prompt`. 38 | 4. Activate the azureml conda environment by running `activate azureml`. 39 | 5. To ensure tensorflow uses a GPU, you will need to uninstall and reinstall Keras and Tensorflow in a specific order. Run these commands in sequence: 40 | - `pip uninstall keras` 41 | - `pip uninstall tensorflow-gpu` 42 | - `pip uninstall tensorflow` 43 | - `pip install keras` 44 | - `pip install tensorflow-gpu==1.10.0` 45 | 6. Upgrade the installed version of the Azure Machine Learning SDK by running the following command: 46 | - `pip install --upgrade azureml-sdk[notebooks,automl] azureml-dataprep` 47 | 7. If you will be using Visual Studio for the labs, launch `Visual Studio 2017` from the Start menu and login with your Microsoft Account. Allow Visual Studio a few moments to get ready. Once you see the Tools for AI Start Page displayed in Visual Studio, the setup is complete. 48 | 8. If you will be using PyCharm for the labs, launch `JetBrains PyCharm Community Edition` from the Start menu. On the Complete Installation dialog, leave `Do not import settings` selected, accept the license agreement and choose an option for Data Sharing. On the Customize PyCharm screen, select `Skip Remaining and Set Defaults`. Once you are at the Welcome to PyCharm new project dialog the setup is complete. 49 | 9. Your Virtual Machine is now ready to support any of the labs using either the Visual Studio or PyCharm environments. 50 | 51 | 52 | # Quickstart: Azure Databricks 53 | 54 | 1. Click the following button to open the ARM template in the Azure Portal. 55 | [Deploy Databricks from the ARM Template](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fazure-quickstart-templates%2Fmaster%2F101-databricks-workspace%2Fazuredeploy.json) 56 | 57 | 2. Provide the required values to create your Azure Databricks workspace: 58 | - Subscription: Choose the Azure Subscription in which to deploy the workspace. 59 | - Resource Group: Leave at Create new and provide a name for the new resource group. 60 | - Location: Select a location near you for deployment that supports both Azure Databricks AND provides NC series GPU enabled Virtual Machines. This currently includes East US, East US 2, North Central US, South Central US and West US 2. For the latest list, see [Azure services available by region](https://azure.microsoft.com/regions/services/). 61 | - Workspace Name: Provide a name for your workspace. 62 | - Pricing Tier: Ensure `premium` is selected. 63 | 64 | 3. Accept the terms and conditions. 65 | 4. Select Purchase. 66 | 5. The workspace creation takes a few minutes. During workspace creation, the portal displays the Submitting deployment for Azure Databricks tile on the right side. You may need to scroll right on your dashboard to see the tile. There is also a progress bar displayed near the top of the screen. You can watch either area for progress. -------------------------------------------------------------------------------- /lab-3/pycharm/README.md: -------------------------------------------------------------------------------- 1 | # Lab 3 - Model Deployment using Azure Machine Learning service 2 | 3 | In this lab you will deploy a trained model to containers using an Azure Container Instance and and Azure Kubernetes Service using the Azure Machine Learning SDK. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/pycharm`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 1. Within PyCharm, select open Existing Project and navigate to the directory where you cloned the repo to open the project `azure-ml-labs`. 10 | 2. In the Project window, expand External Libraries. You should see one environment called `` where the path points to your AzureML Anaconda environment. This Anaconda environment will be used when you execute code. 11 | 3. In the Project tool window expand the folder `03-model-deployment`. 12 | 4. Expand the `data` folder. This folder contains the CSV file `UsedCars_Affordability.csv` which contains the complete data set with labels (Affordable is 1 for affordable, 0 for not affordable). 13 | 5. Open `03_model_deployment.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 14 | 15 | 16 | ## Exercise 2 - Train a simple model locally 17 | This lab builds upon the lessons learned in the previous lab, but is self contained so you work thru this lab without having to run a previous lab. 18 | 1. Read thru and select the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Alt + Shift + Enter` to execute the selected code in the Python Immediate Window. Take a moment to look at the data loaded into the Pandas Dataframe - it contains data about used cars such as the price (in dollars), age (in years), KM (kilometers driven) and other attributes like weather it is automatic transimission, the number of doors, and the weight. In the function `train_eval_register_model` observe how the trained model is saved to the ./outputs folder along with the scaler that will be needed to scale inputs used later when scoring. Observe that we use `Model.register` to upload all files in the ./outputs folder to Azure Machine Learning as the model files. These model files will be retrieved later when the model is deployed into a container and operationalized as a web service. 19 | 2. In Step 2, we retrieve or create the AML Workspace and then train one instance of the model that we will deploy. In this step, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. Select and execute Step #2. 20 | 21 | 22 | ## Exercise 3 - Download a version of a model from Azure Machine Learning 23 | 1. Once a model is registered with Azure Machine Learning, we can download the model files to any client and use them for scoring. In Step 3, you download the model you just registered, load both the scaler and model files retrieved by deserializing them into objects and then use them to perform a single prediction. Select and execute Step 3. 24 | 25 | 26 | ## Exercise 4 - Create the container image configuration 27 | 1. When you deploy a model as web service to either ACI or AKS, you are deploying a Docker container. The first steps towards deploying involve defining the contents of that container. In Step 4, you create Conda Dependencies YAML file that describes what Python packages need to be installed in the container- in this case you specify scikit-learn, numpy and pandas. Select and execute Step 4. 28 | 2. With Azure Machine Learning, you have full control over the logic of the webservice which includes how it loads your model, transforms web service inputs, uses the model for scoring and returns the result. From the Project window, open `03-model-deployment-score.py` and read thru the code that defines the webservice. You do not need to execute this code as the file will be deployed in the contents of the container image you are about to create. 29 | 3. Return to `03_model_deployment.py`. To create a Container Image, you need three things: the scoring script file, the runtime configuration (defining whether Python or PySpark should be used) and the Conda Dependencies file. Calling `ContainerImage.image_configuration` will capture all of the container image configuration in a single object. Select and execute Step 5. 30 | 31 | ## Exercise 5 - Deploy the container image to ACI 32 | 1. With the Container Image configuration in hand, you are almost ready to deploy to ACI. The next step is to define the size of the VM that ACI will use to run your Container. Execute Step 6 to create this configuration. 33 | 2. To deploy the container that operationalizes your model as a webservice, you can use `Webservice.deploy_from_model` which will use your registered model, and automate the creation of a new Container Image, and run the created container in ACI. Execute Step 7 to deploy your webservice to ACI. This step will take 5-7 minutes to complete. 34 | 3. Once the webservice deployment completes, you can use the returned webservice object to invoke the webservice. Execute Step 8 to invoke your webservice deployed to ACI. 35 | 36 | ## Exercise 6 - Deploy the container image to AKS 37 | 1. Once you are familiar with the process for deploying a webservice to ACI, you will find the process for deploying to AKS to be similar with one additional step that creates the AKS cluster first. Execute Step 9 to provision a small AKS cluster. This step will take about 15-20 minutes. 38 | 2. With your AKS cluster ready, now you can deploy your webservice. Once again, you need to provide a configuration for the size of resources allocated from the AKS cluster to run instances of your Container. Execute Step 10 to deploy your webservice. This step will take 5-7 minutes. 39 | 3. As before, you can use the webservice object returned by the deploy_from_model method to invoke your deployed webservice. Execute Step 11 to verify you can invoke the web service. 40 | 41 | 42 | -------------------------------------------------------------------------------- /lab-3/visual-studio/README.md: -------------------------------------------------------------------------------- 1 | # Lab 3 - Model Deployment using Azure Machine Learning service 2 | 3 | In this lab you will deploy a trained model to containers using an Azure Container Instance and and Azure Kubernetes Service using the Azure Machine Learning SDK. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/visual-studio`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 10 | 1. Navigate to the directory where you cloned the repo and then open the solution `azure-ml-labs.sln`. 11 | 2. With the solution open in Visual Studio, look at Solution Explorer and expand the project `03-model-deployment`. 12 | 3. Under that, expand Python Environments. You should see one environment called `AzureML (3.6, 64-bit)`. This Anaconda environment will be used when you execute code either by running the script using F5 or by executing Python code in the Python Interactive Window. 13 | 4. Expand the `data` folder. This folder contains one CSV files. `UsedCars_Affordability.csv` contains the complete, cleaned data set with labels (Affordable is 1 for affordable, 0 for not affordable). 14 | 5. Open `_03_model_deployment.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 15 | 16 | 17 | ## Exercise 2 - Train a simple model locally 18 | This lab builds upon the lessons learned in the previous lab, but is self contained so you work thru this lab without having to run a previous lab. 19 | 1. Read thru and select the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Control + Enter` to execute the selected code in the Python Immediate Window. Take a moment to look at the data loaded into the Pandas Dataframe - it contains data about used cars such as the price (in dollars), age (in years), KM (kilometers driven) and other attributes like weather it is automatic transimission, the number of doors, and the weight. In the function `train_eval_register_model` observe how the trained model is saved to the ./outputs folder along with the scaler that will be needed to scale inputs used later when scoring. Observe that we use `Model.register` to upload all files in the ./outputs folder to Azure Machine Learning as the model files. These model files will be retrieved later when the model is deployed into a container and operationalized as a web service. 20 | 2. In Step 2, we retrieve or create the AML Workspace and then train one instance of the model that we will deploy. In this step, be sure to set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. Select and execute Step #2. 21 | 22 | 23 | ## Exercise 3 - Download a version of a model from Azure Machine Learning 24 | 1. Once a model is registered with Azure Machine Learning, we can download the model files to any client and use them for scoring. In Step 3, you download the model you just registered, load both the scaler and model files retrieved by deserializing them into objects and then use them to perform a single prediction. Select and execute Step 3. 25 | 26 | 27 | ## Exercise 4 - Create the container image configuration 28 | 1. When you deploy a model as web service to either ACI or AKS, you are deploying a Docker container. The first steps towards deploying involve defining the contents of that container. In Step 4, you create Conda Dependencies YAML file that describes what Python packages need to be installed in the container- in this case you specify scikit-learn, numpy and pandas. Select and execute Step 4. 29 | 2. With Azure Machine Learning, you have full control over the logic of the webservice which includes how it loads your model, transforms web service inputs, uses the model for scoring and returns the result. From Solution Explorer, open Score.py and read thru the code that defines the webservice. You do not need to execute this code as the file will be deployed in the contents of the container image you are about to create. 30 | 3. Return to _03_model_deployment.py. To create a Container Image, you need three things: the scoring script file, the runtime configuration (defining whether Python or PySpark should be used) and the Conda Dependencies file. Calling `ContainerImage.image_configuration` will capture all of the container image configuration in a single object. Select and execute Step 5. 31 | 32 | ## Exercise 5 - Deploy the container image to ACI 33 | 1. With the Container Image configuration in hand, you are almost ready to deploy to ACI. The next step is to define the size of the VM that ACI will use to run your Container. Execute Step 6 to create this configuration. 34 | 2. To deploy the container that operationalizes your model as a webservice, you can use `Webservice.deploy_from_model` which will use your registered model, and automate the creation of a new Container Image, and run the created container in ACI. Execute Step 7 to deploy your webservice to ACI. This step will take 5-7 minutes to complete. 35 | 3. Once the webservice deployment completes, you can use the returned webservice object to invoke the webservice. Execute Step 8 to invoke your webservice deployed to ACI. 36 | 37 | ## Exercise 6 - Deploy the container image to AKS 38 | 1. Once you are familiar with the process for deploying a webservice to ACI, you will find the process for deploying to AKS to be similar with one additional step that creates the AKS cluster first. Execute Step 9 to provision a small AKS cluster. This step will take about 15-20 minutes. 39 | 2. With your AKS cluster ready, now you can deploy your webservice. Once again, you need to provide a configuration for the size of resources allocated from the AKS cluster to run instances of your Container. Execute Step 10 to deploy your webservice. This step will take 5-7 minutes. 40 | 3. As before, you can use the webservice object returned by the deploy_from_model method to invoke your deployed webservice. Execute Step 11 to verify you can invoke the web service. 41 | 42 | 43 | -------------------------------------------------------------------------------- /lab-1/pycharm/README.md: -------------------------------------------------------------------------------- 1 | # Lab 1 - Training a Machine Learning Model using Azure Machine Learning service 2 | 3 | In this lab you will setup the Azure Machine Learning service from code and create a classical machine learning model that logs metrics collected during model training. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/pycharm`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 10 | 1. Within PyCharm, select open Existing Project and navigate to the directory where you cloned the repo to open the project `azure-ml-labs`. 11 | 2. In the Project window, expand External Libraries. You should see one environment called `` where the path points to your AzureML Anaconda environment. This Anaconda environment will be used when you execute code. 12 | 3. In the Project tool window expand the folder `01-model-training`. 13 | 4. Expand the `data` folder. This folder contains two CSV files. `UsedCars_Clean.csv` represents the unlabeled data and `UsedCars_Affordability.csv` contains the complete data set with labels (Affordable is 1 for affordable, 0 for not affordable). 14 | 5. Expand `training`. This folder contains train.py which will be used later in the lab to train the model using a remote cluster provided by Azure Batch AI. 15 | 6. Open `01_model_training.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 16 | 17 | 18 | ## Exercise 2 - Train a simple model locally 19 | 1. Read thru and select the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Alt + Shift + Enter` to execute the selected code in the Python Console. Take a moment to look at the data loaded into the Pandas Dataframe - it contains data about used cars such as the price (in dollars), age (in years), KM (kilometers driven) and other attributes like weather it is automatic transimission, the number of doors, and the weight. 20 | 2. In Step 2, we are going to try and build a model that can answer the question "Can I afford a car that is X months old and has Y kilometers on it, given I have $12,000 to spend?". We will engineer the label for affordable. Select the code starting with # Step 2 all the way down to but NOT including # Step 3. Use `Alt + Shift + Enter` to execute the selected code in the Python Console. 21 | 3. We are going to train a Logistic Regression model locally. This type of model requires us to standardize the scale of our training features Age and KM, so we use the `StandardScaler` from Scikit-Learn to transform these features so that they have values centered with a mean around 0 (mostly between -2.96 and 1.29). Select Step 3 and execute the code. Observe the difference in min and max values between the un-scaled and scaled Dataframes. 22 | 4. Train the model by fitting a LogisticRegression against the scaled input features (X_scaled) and the labels (y). Select Step 4 and execute the code. 23 | 5. Try prediction - if you set the age to 60 months and km to 40,000, does the model predict you can afford the car? Execute Step 5 and find out. 24 | 6. Now, let's get a sense for how accurate the model is. Select and execute Step 6. What was your model's accuracy? 25 | 7. One thing that can affect the model's performance is how much data of all the labeled training data available is used to train the model. In Step 7, you define a method that uses train_test_split from Scikit-Learn that will enable you to split the data using different percentages. Execute Step 7 to register this function. 26 | 27 | ## Exercise 3 - Use Azure Machine Learning to log performance metrics 28 | In the steps that follow, you will train multiple models using different sizes of training data and observe the impact on performance (accuracy). Each time you create new model, you are executing a Run in the terminology of Azure Machine Learning service. In this case, you will create one Experiment and execute multiple Runs within it, each with different training percentages (and resultant varying accuracies). 29 | 30 | 1. Execute Step 8 to quickly verify you have the Azure Machine Learning SDK installed. If you get a version number back without error, you are ready to proceed. 31 | 2. All Azure Machine Learning entities are organized within a Workspace. You can create an AML Workspace in the Azure Portal, but as the code in Step 9 shows, you can also create a Workspace directly from code. Set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. Execute Step 9. You will be prompted to log in to your Azure Subscription. 32 | 3. To begin capturing metrics, you must first create an Experiment and then call `start_logging()` on that Experiment. The return value of this call is a Run. This root run can have other child runs. When you are finished with an experiment run, use `complete()` to close out the root run. Execute Step 10 to train four different models using differing amounts of training data and log the results to Azure Machine Learning. 33 | 4. Now that you have captured history for various runs, you can review the runs. You could use the Azure Portal for this - go to the Azure Portal, find your Azure Machine Learning Workspace, select Experiments and select the UsedCars_Experiment. However, in this case we will use the AML SDK to query for the runs. Select and execute Step 11 to view the runs and their status. 34 | 35 | ## Exercise 4 - Train remotely using Azure Batch AI 36 | Up until now, all of your training was executed locally on the same machine running Visual Studio. Now you will execute the same logic targeting a remote Azure Batch AI cluster, which you will provision from code. 37 | 38 | 1. Read thru and then execute Step 12 in which you will create an Azure Batch AI cluster using code. Once your cluster is ready, you should see output similar to the following: 39 | ``` 40 | Creating a new compute target... 41 | Creating 42 | succeeded..... 43 | BatchAI wait for completion finished 44 | Minimum number of nodes requested have been provisioned 45 | {'allocationState': 'steady', 'allocationStateTransitionTime': '2018-11-17T17:56:07.361000+00:00', 'creationTime': '2018-11-17T17:52:53.601000+00:00', 'currentNodeCount': 1, 'errors': None, 'nodeStateCounts': {'idleNodeCount': 0, 'leavingNodeCount': 0, 'preparingNodeCount': 1, 'runningNodeCount': 0, 'unusableNodeCount': 0}, 'provisioningState': 'succeeded', 'provisioningStateTransitionTime': '2018-11-17T17:53:59.653000+00:00', 'scaleSettings': {'manual': None, 'autoScale': {'maximumNodeCount': 3, 'minimumNodeCount': 1, 'initialNodeCount': 1}}, 'vmPriority': 'lowpriority', 'vmSize': 'STANDARD_DS11_V2'} 46 | 47 | ``` 48 | 2. With your cluster ready, you need to upload the training data to the default DataStore for your AML Workspace (which uses Azure Storage). Execute Step 13 to upload the data folder. 49 | 3. Next, you will need to create a training script that is similar to the code you have executed locally to train the model. Open `training/train.py` and read thru it. You do not need to execute this script, as you will send it to Azure Batch AI for execution. 50 | 4. Return to `01_model_training.py`. You will create an estimator that describes the configuration of the job that will execute your model training script. Execute Step 14 to create this estimator. 51 | 5. As the last step, submit the job using the `submit()` method of the Experiment object. Execute Step 15 to remotely execute your training script. The output you should see will begin with the creation of a Docker Container that contains your configured dependencies, followed by the execution of your training script. 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /lab-1/visual-studio/README.md: -------------------------------------------------------------------------------- 1 | # Lab 1 - Training a Machine Learning Model using Azure Machine Learning service 2 | 3 | In this lab you will setup the Azure Machine Learning service from code and create a classical machine learning model that logs metrics collected during model training. 4 | 5 | ## Exercise 0 - Get the lab files 6 | If you have not cloned this repository to your working environment, do so now. All of the artifacts for this lab are located under `starter-artifacts/visual-studio`. 7 | 8 | ## Exercise 1 - Get oriented to the lab files 9 | 10 | 1. Navigate to the directory where you cloned the repo and then open the solution `azure-ml-labs.sln`. 11 | 2. With the solution open in Visual Studio, look at Solution Explorer and expand the project `01-model-training`. 12 | 3. Under that, expand Python Environments. You should see one environment called `AzureML (3.6, 64-bit)`. This Anaconda environment will be used when you execute code either by running the script using F5 or by executing Python code in the Python Interactive Window. 13 | 4. Expand the `data` folder. This folder contains two CSV files. `UsedCars_Clean.csv` represents the unlabeled data and `UsedCars_Affordability.csv` contains the complete data set with labels (Affordable is 1 for affordable, 0 for not affordable). 14 | 5. Expand `training`. This folder contains train.py which will be used later in the lab to train the model using a remote cluster provided by Azure Batch AI. 15 | 6. Open `_01_model_training.py`. This is the Python file you will step thru executing in this lab. Leave it open and continue to the next exercise. 16 | 17 | 18 | ## Exercise 2 - Train a simple model locally 19 | 1. Read thru and select the code starting with # Step 1 all the way down to but NOT including # Step 2. Use `Control + Enter` to execute the selected code in the Python Immediate Window. Take a moment to look at the data loaded into the Pandas Dataframe - it contains data about used cars such as the price (in dollars), age (in years), KM (kilometers driven) and other attributes like weather it is automatic transimission, the number of doors, and the weight. 20 | 2. In Step 2, we are going to try and build a model that can answer the question "Can I afford a car that is X months old and has Y kilometers on it, given I have $12,000 to spend?". We will engineer the label for affordable. Select the code starting with # Step 2 all the way down to but NOT including # Step 3. Use `Control + Enter` to execute the selected code in the Python Immediate Window. 21 | 3. We are going to train a Logistic Regression model locally. This type of model requires us to standardize the scale of our training features Age and KM, so we use the `StandardScaler` from Scikit-Learn to transform these features so that they have values centered with a mean around 0 (mostly between -2.96 and 1.29). Select Step 3 and execute the code. Observe the difference in min and max values between the un-scaled and scaled Dataframes. 22 | 4. Train the model by fitting a LogisticRegression against the scaled input features (X_scaled) and the labels (y). Select Step 4 and execute the code. 23 | 5. Try prediction - if you set the age to 60 months and km to 40,000, does the model predict you can afford the car? Execute Step 5 and find out. 24 | 6. Now, let's get a sense for how accurate the model is. Select and execute Step 6. What was your model's accuracy? 25 | 7. One thing that can affect the model's performance is how much data of all the labeled training data available is used to train the model. In Step 7, you define a method that uses train_test_split from Scikit-Learn that will enable you to split the data using different percentages. Execute Step 7 to register this function. 26 | 27 | ## Exercise 3 - Use Azure Machine Learning to log performance metrics 28 | In the steps that follow, you will train multiple models using different sizes of training data and observe the impact on performance (accuracy). Each time you create new model, you are executing a Run in the terminology of Azure Machine Learning service. In this case, you will create one Experiment and execute multiple Runs within it, each with different training percentages (and resultant varying accuracies). 29 | 30 | 1. Execute Step 8 to quickly verify you have the Azure Machine Learning SDK installed. If you get a version number back without error, you are ready to proceed. 31 | 2. All Azure Machine Learning entities are organized within a Workspace. You can create an AML Workspace in the Azure Portal, but as the code in Step 9 shows, you can also create a Workspace directly from code. Set the values for `subscription_id`, `resource_group`, `workspace_name` and `workspace_region` as directed by the comments. Execute Step 9. You will be prompted to log in to your Azure Subscription. 32 | 3. To begin capturing metrics, you must first create an Experiment and then call `start_logging()` on that Experiment. The return value of this call is a Run. This root run can have other child runs. When you are finished with an experiment run, use `complete()` to close out the root run. Execute Step 10 to train four different models using differing amounts of training data and log the results to Azure Machine Learning. 33 | 4. Now that you have captured history for various runs, you can review the runs. You could use the Azure Portal for this - go to the Azure Portal, find your Azure Machine Learning Workspace, select Experiments and select the UsedCars_Experiment. However, in this case we will use the AML SDK to query for the runs. Select and execute Step 11 to view the runs and their status. 34 | 35 | ## Exercise 4 - Train remotely using Azure Batch AI 36 | Up until now, all of your training was executed locally on the same machine running Visual Studio. Now you will execute the same logic targeting a remote Azure Batch AI cluster, which you will provision from code. 37 | 38 | 1. Read thru and then execute Step 12 in which you will create an Azure Batch AI cluster using code. Once your cluster is ready, you should see output similar to the following: 39 | ``` 40 | Creating a new compute target... 41 | Creating 42 | succeeded..... 43 | BatchAI wait for completion finished 44 | Minimum number of nodes requested have been provisioned 45 | {'allocationState': 'steady', 'allocationStateTransitionTime': '2018-11-17T17:56:07.361000+00:00', 'creationTime': '2018-11-17T17:52:53.601000+00:00', 'currentNodeCount': 1, 'errors': None, 'nodeStateCounts': {'idleNodeCount': 0, 'leavingNodeCount': 0, 'preparingNodeCount': 1, 'runningNodeCount': 0, 'unusableNodeCount': 0}, 'provisioningState': 'succeeded', 'provisioningStateTransitionTime': '2018-11-17T17:53:59.653000+00:00', 'scaleSettings': {'manual': None, 'autoScale': {'maximumNodeCount': 3, 'minimumNodeCount': 1, 'initialNodeCount': 1}}, 'vmPriority': 'lowpriority', 'vmSize': 'STANDARD_DS11_V2'} 46 | 47 | ``` 48 | 2. With your cluster ready, you need to upload the training data to the default DataStore for your AML Workspace (which uses Azure Storage). Execute Step 13 to upload the data folder. 49 | 3. Next, you will need to create a training script that is similar to the code you have executed locally to train the model. Open `training/train.py` and read thru it. You do not need to execute this script, as you will send it to Azure Batch AI for execution. 50 | 4. Return to `_01_model_training.py`. You will create an estimator that describes the configuration of the job that will execute your model training script. Execute Step 14 to create this estimator. 51 | 5. As the last step, submit the job using the `submit()` method of the Experiment object. Execute Step 15 to remotely execute your training script. The output you should see will begin with the creation of a Docker Container that contains your configured dependencies, followed by the execution of your training script. 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/02-model-management/02_model_management.py: -------------------------------------------------------------------------------- 1 | # Step 1 - load the training data locally 2 | ######################################### 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | from sklearn import linear_model 7 | from sklearn.externals import joblib 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.metrics import accuracy_score 10 | from sklearn.model_selection import train_test_split 11 | import azureml 12 | from azureml.core import Run 13 | from azureml.core import Workspace 14 | from azureml.core.run import Run 15 | from azureml.core.experiment import Experiment 16 | import pickle 17 | 18 | print("Current working directory is ", os.path.abspath(os.path.curdir)) 19 | df_affordability = pd.read_csv('./02-model-management/data/UsedCars_Affordability.csv', delimiter=',') 20 | print(df_affordability) 21 | 22 | full_X = df_affordability[["Age", "KM"]] 23 | full_Y = df_affordability[["Affordable"]] 24 | 25 | # Step 2 - Define a helper method for training, evaluating and registering a model 26 | ################################################################################### 27 | def train_eval_register_model(experiment_name, full_X, full_Y,training_set_percentage): 28 | 29 | # start a training run by defining an experiment 30 | myexperiment = Experiment(ws, experiment_name) 31 | run = myexperiment.start_logging() 32 | 33 | 34 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 35 | 36 | scaler = StandardScaler() 37 | X_scaled = scaler.fit_transform(train_X) 38 | clf = linear_model.LogisticRegression(C=1) 39 | clf.fit(X_scaled, train_Y) 40 | 41 | scaled_inputs = scaler.transform(test_X) 42 | predictions = clf.predict(scaled_inputs) 43 | score = accuracy_score(test_Y, predictions) 44 | 45 | print("With %0.2f percent of data, model accuracy reached %0.4f." % (training_set_percentage, score)) 46 | 47 | # Log the training metrics to Azure Machine Learning service run history 48 | run.log("Training_Set_Percentage", training_set_percentage) 49 | run.log("Accuracy", score) 50 | run.complete() 51 | 52 | output_model_path = 'outputs/' + experiment_name + '.pkl' 53 | pickle.dump(clf,open(output_model_path,'wb')) 54 | 55 | # Register and upload this version of the model with Azure Machine Learning service 56 | registered_model = run.register_model(model_name='usedcarsmodel', model_path=output_model_path) 57 | 58 | print(registered_model.name, registered_model.id, registered_model.version, sep = '\t') 59 | 60 | return (clf, score) 61 | 62 | # Step 3 - Run a few experiments in your Azure ML Workspace 63 | ########################################################### 64 | # Verify AML SDK Installed 65 | print("SDK Version:", azureml.core.VERSION) 66 | 67 | 68 | # Create a new Workspace or retrieve the existing one 69 | #Provide the Subscription ID of your existing Azure subscription 70 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 71 | 72 | #Provide values for the Resource Group and Workspace that will be created 73 | resource_group = "aml-workspace-z" 74 | workspace_name = "aml-workspace-z" 75 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 76 | 77 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 78 | ws = Workspace.create( 79 | name = workspace_name, 80 | subscription_id = subscription_id, 81 | resource_group = resource_group, 82 | location = workspace_region, 83 | exist_ok = True) 84 | 85 | print("Workspace Provisioning complete.") 86 | 87 | 88 | # Create an experiment, log metrics and register the created models for multiple training runs 89 | experiment_name = "Experiment-02-01" 90 | training_set_percentage = 0.25 91 | model, score = train_eval_register_model(experiment_name, full_X, full_Y, training_set_percentage) 92 | 93 | experiment_name = "Experiment-02-02" 94 | training_set_percentage = 0.5 95 | model, score = train_eval_register_model(experiment_name, full_X, full_Y, training_set_percentage) 96 | 97 | experiment_name = "Experiment-02-03" 98 | training_set_percentage = 0.75 99 | model, score = train_eval_register_model(experiment_name, full_X, full_Y, training_set_percentage) 100 | 101 | 102 | # Step 4 - Query for all Experiments. 103 | ##################################### 104 | # You can retreive the list of all experiments in Workspace using the following: 105 | all_experiments = ws.experiments 106 | 107 | print(all_experiments) 108 | 109 | # Query for the metrics of a particular experiment 110 | # You can retrieve an existing experiment by constructing an Experiment object using the name of an existing experiment. 111 | my_experiment = Experiment(ws, "Experiment-02-03") 112 | print(my_experiment) 113 | 114 | # Query an experiment for metrics 115 | # With an experiment in hand, you retrieve any metrics collected for any of its child runs 116 | my_experiment_runs = my_experiment.get_runs() 117 | print( [ (run.experiment.name, run.id, run.get_metrics()) for run in my_experiment_runs] ) 118 | 119 | 120 | 121 | # Step 5 - Submit an experiment to Azure Batch AI and log metrics for multiple training runs 122 | ############################################################################################ 123 | experiment_name = "UsedCars_Batch_02" 124 | 125 | from azureml.core import Experiment 126 | exp = Experiment(workspace=ws, name=experiment_name) 127 | 128 | from azureml.core.compute import AmlCompute 129 | from azureml.core.compute import ComputeTarget 130 | import os 131 | 132 | # choose a name for your cluster 133 | batchai_cluster_name = "carscluster02" 134 | cluster_min_nodes = 1 135 | cluster_max_nodes = 3 136 | vm_size = "STANDARD_DS11_V2" 137 | autoscale_enabled = True 138 | 139 | 140 | if batchai_cluster_name in ws.compute_targets: 141 | compute_target = ws.compute_targets[batchai_cluster_name] 142 | if compute_target and type(compute_target) is AmlCompute: 143 | print('Found existing compute target, using this compute target instead of creating: ' + batchai_cluster_name) 144 | else: 145 | print('Creating a new compute target...') 146 | provisioning_config = AmlCompute.provisioning_configuration(vm_size = vm_size, 147 | vm_priority = 'lowpriority', # optional 148 | autoscale_enabled = autoscale_enabled, 149 | min_nodes = cluster_min_nodes, 150 | max_nodes = cluster_max_nodes) 151 | 152 | # create the cluster 153 | compute_target = ComputeTarget.create(ws, batchai_cluster_name, provisioning_config) 154 | 155 | # can poll for a minimum number of nodes and for a specific timeout. 156 | # if no min node count is provided it will use the scale settings for the cluster 157 | compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20) 158 | 159 | # For a more detailed view of current BatchAI cluster status, use the 'status' property 160 | print(compute_target.status.serialize()) 161 | 162 | # Upload the dataset to the DataStore 163 | ds = ws.get_default_datastore() 164 | print(ds.datastore_type, ds.account_name, ds.container_name) 165 | ds.upload(src_dir='./data', target_path='used_cars', overwrite=True, show_progress=True) 166 | 167 | 168 | # Prepare batch training script 169 | # - See ./training/train.py 170 | 171 | 172 | # Create estimator 173 | from azureml.train.estimator import Estimator 174 | 175 | script_params = { 176 | '--data-folder': ds.as_mount(), 177 | '--training-set-percentage': 0.3 178 | } 179 | 180 | est_config = Estimator(source_directory='./02-model-management/training', 181 | script_params=script_params, 182 | compute_target=compute_target, 183 | entry_script='train.py', 184 | conda_packages=['scikit-learn','pandas']) 185 | 186 | # Execute the job 187 | run = exp.submit(config=est_config) 188 | run 189 | 190 | # Poll for job status 191 | run.wait_for_completion(show_output=True) # value of True will display a verbose, streaming log 192 | 193 | # Step 6 - Retrieve the metrics for the model trained remotely in Azure Batch AI 194 | ################################################################################ 195 | # Examine the recorded metrics from the run 196 | print(run.get_metrics()) -------------------------------------------------------------------------------- /starter-artifacts/visual-studio/02-model-management/_02_model_management.py: -------------------------------------------------------------------------------- 1 | # Step 1 - load the training data locally 2 | ######################################### 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | from sklearn import linear_model 7 | from sklearn.externals import joblib 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.metrics import accuracy_score 10 | from sklearn.model_selection import train_test_split 11 | import azureml 12 | from azureml.core import Run 13 | from azureml.core import Workspace 14 | from azureml.core.run import Run 15 | from azureml.core.experiment import Experiment 16 | import pickle 17 | 18 | print("Current working directory is ", os.path.abspath(os.path.curdir)) 19 | df_affordability = pd.read_csv('data/UsedCars_Affordability.csv', delimiter=',') 20 | print(df_affordability) 21 | 22 | full_X = df_affordability[["Age", "KM"]] 23 | full_Y = df_affordability[["Affordable"]] 24 | 25 | # Step 2 - Define a helper method for training, evaluating and registering a model 26 | ################################################################################### 27 | def train_eval_register_model(experiment_name, full_X, full_Y,training_set_percentage): 28 | 29 | # start a training run by defining an experiment 30 | myexperiment = Experiment(ws, experiment_name) 31 | run = myexperiment.start_logging() 32 | 33 | 34 | train_X, test_X, train_Y, test_Y = train_test_split(full_X, full_Y, train_size=training_set_percentage, random_state=42) 35 | 36 | scaler = StandardScaler() 37 | X_scaled = scaler.fit_transform(train_X) 38 | clf = linear_model.LogisticRegression(C=1) 39 | clf.fit(X_scaled, train_Y) 40 | 41 | scaled_inputs = scaler.transform(test_X) 42 | predictions = clf.predict(scaled_inputs) 43 | score = accuracy_score(test_Y, predictions) 44 | 45 | print("With %0.2f percent of data, model accuracy reached %0.4f." % (training_set_percentage, score)) 46 | 47 | # Log the training metrics to Azure Machine Learning service run history 48 | run.log("Training_Set_Percentage", training_set_percentage) 49 | run.log("Accuracy", score) 50 | run.complete() 51 | 52 | output_model_path = 'outputs/' + experiment_name + '.pkl' 53 | pickle.dump(clf,open(output_model_path,'wb')) 54 | 55 | # Register and upload this version of the model with Azure Machine Learning service 56 | registered_model = run.register_model(model_name='usedcarsmodel', model_path=output_model_path) 57 | 58 | print(registered_model.name, registered_model.id, registered_model.version, sep = '\t') 59 | 60 | return (clf, score) 61 | 62 | # Step 3 - Run a few experiments in your Azure ML Workspace 63 | ########################################################### 64 | # Verify AML SDK Installed 65 | print("SDK Version:", azureml.core.VERSION) 66 | 67 | 68 | # Create a new Workspace or retrieve the existing one 69 | #Provide the Subscription ID of your existing Azure subscription 70 | subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" 71 | 72 | #Provide values for the Resource Group and Workspace that will be created 73 | resource_group = "aml-workspace-z" 74 | workspace_name = "aml-workspace-z" 75 | workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope 76 | 77 | # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error 78 | ws = Workspace.create( 79 | name = workspace_name, 80 | subscription_id = subscription_id, 81 | resource_group = resource_group, 82 | location = workspace_region, 83 | exist_ok = True) 84 | 85 | print("Workspace Provisioning complete.") 86 | 87 | 88 | # Create an experiment, log metrics and register the created models for multiple training runs 89 | experiment_name = "Experiment-02-01" 90 | training_set_percentage = 0.25 91 | model, score = train_eval_register_model(experiment_name, full_X, full_Y, training_set_percentage) 92 | 93 | experiment_name = "Experiment-02-02" 94 | training_set_percentage = 0.5 95 | model, score = train_eval_register_model(experiment_name, full_X, full_Y, training_set_percentage) 96 | 97 | experiment_name = "Experiment-02-03" 98 | training_set_percentage = 0.75 99 | model, score = train_eval_register_model(experiment_name, full_X, full_Y, training_set_percentage) 100 | 101 | 102 | # Step 4 - Query for all Experiments. 103 | ##################################### 104 | # You can retreive the list of all experiments in Workspace using the following: 105 | all_experiments = ws.experiments 106 | 107 | print(all_experiments) 108 | 109 | # Query for the metrics of a particular experiment 110 | # You can retrieve an existing experiment by constructing an Experiment object using the name of an existing experiment. 111 | my_experiment = Experiment(ws, "Experiment-02-03") 112 | print(my_experiment) 113 | 114 | # Query an experiment for metrics 115 | # With an experiment in hand, you retrieve any metrics collected for any of its child runs 116 | my_experiment_runs = my_experiment.get_runs() 117 | print( [ (run.experiment.name, run.id, run.get_metrics()) for run in my_experiment_runs] ) 118 | 119 | 120 | 121 | # Step 5 - Submit an experiment to Azure Batch AI and log metrics for multiple training runs 122 | ############################################################################################ 123 | 124 | 125 | #ws = Workspace.get(name=workspace_name, subscription_id=subscription_id,resource_group=resource_group) 126 | #print(ws.name, ws.location, ws.resource_group, ws.location, sep = '\t') 127 | 128 | experiment_name = "UsedCars_Batch_02" 129 | 130 | from azureml.core import Experiment 131 | exp = Experiment(workspace=ws, name=experiment_name) 132 | 133 | from azureml.core.compute import AmlCompute 134 | from azureml.core.compute import ComputeTarget 135 | import os 136 | 137 | # choose a name for your cluster 138 | batchai_cluster_name = "carscluster02" 139 | cluster_min_nodes = 1 140 | cluster_max_nodes = 3 141 | vm_size = "STANDARD_DS11_V2" 142 | autoscale_enabled = True 143 | 144 | 145 | if batchai_cluster_name in ws.compute_targets: 146 | compute_target = ws.compute_targets[batchai_cluster_name] 147 | if compute_target and type(compute_target) is AmlCompute: 148 | print('Found existing compute target, using this compute target instead of creating: ' + batchai_cluster_name) 149 | else: 150 | print('Creating a new compute target...') 151 | provisioning_config = AmlCompute.provisioning_configuration(vm_size = vm_size, 152 | vm_priority = 'lowpriority', # optional 153 | autoscale_enabled = autoscale_enabled, 154 | min_nodes = cluster_min_nodes, 155 | max_nodes = cluster_max_nodes) 156 | 157 | # create the cluster 158 | compute_target = ComputeTarget.create(ws, batchai_cluster_name, provisioning_config) 159 | 160 | # can poll for a minimum number of nodes and for a specific timeout. 161 | # if no min node count is provided it will use the scale settings for the cluster 162 | compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20) 163 | 164 | # For a more detailed view of current BatchAI cluster status, use the 'status' property 165 | print(compute_target.status.serialize()) 166 | 167 | # Upload the dataset to the DataStore 168 | ds = ws.get_default_datastore() 169 | print(ds.datastore_type, ds.account_name, ds.container_name) 170 | ds.upload(src_dir='./data', target_path='used_cars', overwrite=True, show_progress=True) 171 | 172 | 173 | # Prepare batch training script 174 | # - See ./training/train.py 175 | 176 | 177 | # Create estimator 178 | from azureml.train.estimator import Estimator 179 | 180 | script_params = { 181 | '--data-folder': ds.as_mount(), 182 | '--training-set-percentage': 0.3 183 | } 184 | 185 | est_config = Estimator(source_directory='./training', 186 | script_params=script_params, 187 | compute_target=compute_target, 188 | entry_script='train.py', 189 | conda_packages=['scikit-learn','pandas']) 190 | 191 | # Execute the job 192 | run = exp.submit(config=est_config) 193 | run 194 | 195 | # Poll for job status 196 | run.wait_for_completion(show_output=True) # value of True will display a verbose, streaming log 197 | 198 | # Step 6 - Retrieve the metrics for the model trained remotely in Azure Batch AI 199 | ################################################################################ 200 | # Examine the recorded metrics from the run 201 | print(run.get_metrics()) -------------------------------------------------------------------------------- /starter-artifacts/pycharm/azure-ml-labs/.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 31 | 32 | 33 | 34 | /data 35 | data/ 36 | 37 | 38 | 39 | 48 | 49 | 50 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 |