├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── Python ├── .gitattributes ├── .gitignore ├── azureml-pipeline-databricks │ ├── .amlignore │ ├── .gitignore │ ├── README.md │ ├── azure-pipelines.yml │ ├── ci_dependencies.yml │ ├── code │ │ ├── prepare │ │ │ └── feature_engineering.py │ │ └── tests │ │ │ ├── diabetes_missing_values.csv │ │ │ ├── feature_engineering_expected.csv │ │ │ └── feature_engineering_test.py │ ├── docs │ │ └── images │ │ │ ├── azureml-pipeline.png │ │ │ └── output-dataset.png │ ├── environment_setup │ │ ├── Dockerfile │ │ ├── docker-image-pipeline.yml │ │ ├── install_requirements.sh │ │ ├── terraform-init-template.yml │ │ ├── terraform-outputs-template.yml │ │ ├── terraform-pipeline.yml │ │ ├── terraform-stages-template.yml │ │ └── terraform │ │ │ ├── .artifactignore │ │ │ ├── azureml │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ │ │ ├── backend.tf │ │ │ ├── databricks │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ ├── provider.tf │ │ │ ├── training-data │ │ │ ├── diabetes.csv │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ │ │ ├── variables.tf │ │ │ └── vnet │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf │ ├── ml_service │ │ ├── __init__.py │ │ └── build_ml_pipeline.py │ ├── tox.ini │ └── variables.yml └── packages │ ├── databricks-client │ ├── MAINTAINER.md │ ├── README.md │ ├── conftest.py │ ├── databricks_client │ │ └── __init__.py │ ├── setup.py │ ├── tests │ │ └── client_test.py │ └── tox.ini │ └── databricks-test │ ├── Dockerfile │ ├── MAINTAINER.md │ ├── README.md │ ├── conftest.py │ ├── databricks_test │ └── __init__.py │ ├── setup.py │ ├── tests │ ├── context_test.py │ ├── etl_expected.csv │ ├── etl_input.csv │ ├── etl_notebook.py │ ├── etl_test.py │ ├── fs_notebook.py │ ├── fs_test.py │ ├── functions_notebook.py │ ├── functions_test.py │ ├── library_notebook.py │ ├── library_test.py │ ├── multiple_runs_notebook.py │ ├── multiple_runs_notebook2.py │ ├── multiple_runs_test.py │ ├── patch_notebook.py │ ├── patch_notebook2.py │ ├── patch_test.py │ ├── secrets_notebook.py │ ├── secrets_test.py │ ├── sqldw_expected.csv │ ├── sqldw_notebook.py │ ├── sqldw_test.py │ ├── widgets_notebook.py │ ├── widgets_test.py │ ├── workflow_notebook.py │ └── workflow_test.py │ └── tox.ini ├── README.md ├── SECURITY.md └── adf └── utils ├── Invoke-ADFPipeline.ps1 ├── New-DataFactory.ps1 ├── New-StorageAccount.ps1 └── Utilities.psm1 /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /Python/.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /Python/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # pipenv 86 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 87 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 88 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 89 | # install all needed dependencies. 90 | #Pipfile.lock 91 | 92 | # celery beat schedule file 93 | celerybeat-schedule 94 | 95 | # SageMath parsed files 96 | *.sage.py 97 | 98 | # Environments 99 | .env 100 | .venv 101 | env/ 102 | venv/ 103 | ENV/ 104 | env.bak/ 105 | venv.bak/ 106 | 107 | # Spyder project settings 108 | .spyderproject 109 | .spyproject 110 | 111 | # Rope project settings 112 | .ropeproject 113 | 114 | # mkdocs documentation 115 | /site 116 | 117 | # mypy 118 | .mypy_cache/ 119 | .dmypy.json 120 | dmypy.json 121 | 122 | # Pyre type checker 123 | .pyre/ 124 | 125 | .vscode 126 | 127 | # Spark tests artifacts 128 | metastore_db 129 | spark-warehouse 130 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/.amlignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | azureml-logs 3 | .azureml 4 | .git 5 | outputs 6 | azureml-setup 7 | docs 8 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | venv/ 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | *-testresults.xml 51 | test-output.xml 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | *.tmp 91 | .terraform* 92 | *.tfstate* 93 | test-output.xml 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | *.vscode 100 | 101 | # Spyder project settings 102 | .spyderproject 103 | .spyproject 104 | 105 | # Rope project settings 106 | .ropeproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | 114 | .DS_Store 115 | 116 | # Azure ML 117 | .azureml 118 | 119 | # Terraform 120 | .terraform* 121 | *.tfstate* 122 | *.tmp 123 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/README.md: -------------------------------------------------------------------------------- 1 | # Integrating Databricks into Azure ML Pipelines with Terraform 2 | 3 | This sample automates the provisioning of an ML execution environment using Terraform, and the provisioning and execution of an [Azure ML Pipeline](https://docs.microsoft.com/en-us/azure/machine-learning/concept-ml-pipelines) that runs a Databricks notebook doing data engineering. 4 | 5 | This sample demonstrates: 6 | * Deployment of Azure ML and Databricks infrastructure using Terraform (based on the [Terraform Azure DevOps starter sample](https://github.com/microsoft/terraform-azure-devops-starter)). 7 | * Provisioning of Databricks accounts and notebooks with Azure AD authentication, using the [databricks-client](https://pypi.org/project/databricks-client/) module. 8 | * Unit testing of Databricks notebooks with PySpark, using the [databricks-test](https://pypi.org/project/databricks-test/) module. 9 | * Integrating a [Databricks step](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-create-your-first-pipeline#databricks) into an Azure ML pipeline. 10 | 11 | This sample is meant to be combined with [the MLOpsPython repository](https://github.com/microsoft/MLOpsPython) 12 | in order to add ETL / feature engineering to an ML training pipeline. The MLOpsPython repository 13 | contains templates for subsequent steps of an MLOps pipeline, such as ML model building, 14 | validation and scoring image deployment. 15 | 16 | In this sample, Databricks is used for feature engineering previously to building an ML model. A Databricks step 17 | can also be used for model training and model batch scoring. 18 | 19 | ## Contents 20 | 21 | | File/folder | Description | 22 | |-------------------------|-------------------------------------------------------------| 23 | | `README.md` | This README file. | 24 | | `azure-pipelines.yml` | The Azure ML build and integration pipeline. | 25 | | `ci_dependencies.yml` | The image definition for the CI environment. | 26 | | `code` | The Databricks feature engineering notebook and unit tests. | 27 | | `docs` | Images for this README file. | 28 | | `environment_setup` | Pipelines and configuration for building the environment. | 29 | | `ml_service` | Python script for provisioning the Azure environment. | 30 | | `tox.ini` | Linting and unit test configuration. | 31 | 32 | ## About the sample 33 | 34 | The sample contains [Terraform configuration](environment_setup/terraform) to deploy an entire environment for creating and executing a data engineering pipeline in Azure ML Pipelines. 35 | 36 | The [Databricks notebook](code/prepare/feature_engineering.py) performs very basic feature engineering by 37 | removing lines with NA values from an initial dataset [diabetes.csv](./environment_setup/terraform/training-data/diabetes.csv). 38 | 39 | A [unit test for the notebook](code/tests/feature_engineering_test.py) is provided and runs in CI using the [databricks-test](https://pypi.org/project/databricks-test/) module. 40 | 41 | A [Python script](ml_service/build_ml_pipeline.py) uses the [Azure ML Python SDK](https://docs.microsoft.com/en-us/python/api/overview/azure/ml/?view=azure-ml-py) to provision the notebook and a cluster pool into the Databricks environment, and programmatically define the structure of the Azure ML Pipeline, and submit the pipeline into the Azure ML workspace. The [CI/CD pipeline](azure-pipelines.yml) then proceeds to execute the Azure ML Pipeline. 42 | 43 | ## Running the sample 44 | 45 | ### Getting the code 46 | 47 | Fork this repository within GitHub, or clone it into your Azure DevOps project. 48 | 49 | ### Create an Azure DevOps organization 50 | 51 | We use Azure DevOps for running our MLOps pipeline with build (CI), ML training and scoring service release 52 | (CD) stages. If you don't already have an Azure DevOps organization, create one by 53 | following the instructions [here](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/create-organization?view=azure-devops). 54 | 55 | If you already have an Azure DevOps organization, create a [new project](https://docs.microsoft.com/en-us/azure/devops/organizations/projects/create-project?view=azure-devops). 56 | 57 | ### Install Azure DevOps extensions 58 | 59 | Install the [Terraform extension for Azure DevOps](https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks) from the Azure DevOps marketplace into your Azure DevOps organization. 60 | 61 | Also install the [Azure Machine Learning extension](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml). 62 | 63 | ### Create an ARM service connection for Terraform 64 | 65 | The `DataOpsML ARM Connection` service connection is used by the [Azure DevOps pipeline](environment_setup/terraform-init-template.yml) to create the Azure ML workspace and associated resources through Terraform. The pipeline requires an **Azure Resource Manager** 66 | [service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection) at the subscription level. 67 | 68 | Leave the **``Resource Group``** field empty. 69 | 70 | **Note:** Creating the ARM service connection scope requires 'Owner' or 'User Access Administrator' permissions on the subscription. 71 | You must also have sufficient permissions to register an application with 72 | your Azure AD tenant, or receive the ID and secret of a service principal 73 | from your Azure AD Administrator. That principal must have 'Contributor' 74 | permissions on the subscription. 75 | 76 | ### Create a storage account for the Terraform state 77 | 78 | [Create an Azure storage account](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create) with an arbitrary name. In the storage account, create a storage container named `terraformstate`. 79 | 80 | ### Create a Variable Group for your Pipeline 81 | 82 | We make use of a variable group inside Azure DevOps to store variables and their 83 | values that we want to make available across multiple pipelines or pipeline stages. You can either 84 | store the values directly in [Azure DevOps](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=designer#create-a-variable-group) 85 | or connect to an Azure Key Vault in your subscription. Please refer to the 86 | documentation [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=designer#create-a-variable-group) to 87 | learn more about how to create a variable group and 88 | [link](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=designer#use-a-variable-group) it to your pipeline. 89 | Click on **Library** in the **Pipelines** section as indicated below: 90 | 91 | Create a variable group named **``terraform``**. The YAML pipeline definitions in this repository refer to this variable group by name. 92 | 93 | The variable group should contain the following required variables: 94 | 95 | | Variable Name | Suggested Value | 96 | | ------------------------- | -------------------------------------------------------- | 97 | | BASE_NAME | mydataops | 98 | | LOCATION | [The [region of your Azure DevOps organization](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/change-organization-location?view=azure-devops), e.g. `westus2`] | 99 | | TERRAFORM_BACKEND_STORAGE | [The name of the storage account you created for the Terraform state] | 100 | | TERRAFORM_BACKEND_RG | [The resource group of the Terraform state storage account] | 101 | 102 | **Note:** 103 | 104 | The **BASE_NAME** parameter is used throughout the solution for naming 105 | Azure resources. When the solution is used in a shared subscription, there can 106 | be naming collisions with resources that require unique names like azure blob 107 | storage and registry DNS naming. Make sure to give a unique value to the 108 | BASE_NAME variable (e.g. mydataops), so that the created resources will have 109 | unique names. The length of 110 | the BASE_NAME value should not exceed 10 characters and it should contain numbers and lowercase letters only. 111 | 112 | Make sure to select the **Allow access to all pipelines** checkbox in the 113 | variable group configuration. 114 | 115 | ### Run the Terraform pipeline 116 | 117 | In your [Azure DevOps](https://dev.azure.com) project create a new build 118 | pipeline referring to the 119 | [environment_setup/terraform-pipeline.yml](environment_setup/terraform-pipeline.yml) 120 | pipeline definition in your forked repository. 121 | 122 | Save and run the pipeline. This will deploy the environment using Terraform, creating a resource group named `rg-[BASE_NAME]-test-main` containing the following resources: 123 | 124 | * A Machine Learning workspace named `aml-[BASE_NAME]-test` for managing the AML pipeline 125 | * A Container Registry named `acr[BASE_NAME]test`, required to provision the Azure Machine Learning workspace 126 | * A Key Vault named `kv-[BASE_NAME]-test`, required to provision the Azure Machine Learning workspace 127 | * A Storage account named `st[BASE_NAME]test`, required to provision the Azure Machine Learning workspace, used for storing the output of the AML pipeline 128 | * An Application Insights instance named `appinsights-[BASE_NAME]-test`, required to provision the Azure Machine Learning workspace 129 | * An Azure Databricks workspace named `dbricks[BASE_NAME]test`, used for running the data engineering notebook 130 | * A Storage account named `st[BASE_NAME]trtest`, where Terraform has copied a training dataset file [diabetes.csv](./environment_setup/terraform/training-data/diabetes.csv). 131 | 132 | **Note:** 133 | 134 | The Terraform pipeline only runs Terraform if it is run on the `master` branch. 135 | If running from another branch, set the variable `RUN_FLAG_TERRAFORM` to the 136 | value `true` at queue time. 137 | 138 | ### Create a Registry Service Connection 139 | 140 | [Create a service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#sep-docreg) to your Azure Container Registry: 141 | - As *Connection type*, select *Docker Registry* 142 | - As *Registry type*, select *Azure Container Registry* 143 | - As *Azure container registry*, select your Container registry instance (deployed by Terraform) 144 | - As *Service connection name*, enter `DataOpsML Azure Container Registry` 145 | 146 | ### Create a container build pipeline 147 | 148 | In your [Azure DevOps](https://dev.azure.com) project create a new build 149 | pipeline referring to the 150 | [docker-image-pipeline.yml](environment_setup/docker-image-pipeline.yml) 151 | pipeline definition in your forked repository. 152 | 153 | Save and run the pipeline. This will build and push a container image to your Azure Container Registry. 154 | 155 | ### Create an Azure DevOps Azure ML Workspace Service Connection 156 | 157 | Create a service connection to your ML workspace via the [Azure DevOps Azure ML task instructions](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml) to be able to execute the Azure ML training pipeline. Name the connection `DataOpsML Azure ML Workspace` (this name is used in the variable `WORKSPACE_SVC_CONNECTION` in [azure-pipelines.yml](azure-pipelines.yml)). 158 | 159 | **Note:** Creating service connection with Azure Machine Learning workspace scope requires 'Owner' or 'User Access Administrator' permissions on the Workspace. 160 | You must also have sufficient permissions to register an application with 161 | your Azure AD tenant, or receive the ID and secret of a service principal 162 | from your Azure AD Administrator. That principal must have Contributor 163 | permissions on the Azure ML Workspace. 164 | 165 | ### Set up MLOps Pipeline 166 | 167 | Now that you have all the required resources created from the IaC pipeline, 168 | you can set up the Azure DeveOps pipeline that will run the Azure ML pipeline. 169 | 170 | ### Set up the Pipeline 171 | 172 | In your [Azure DevOps](https://dev.azure.com) project create and run a new build 173 | pipeline referring to the 174 | [azure-pipelines.yml](azure-pipelines.yml) 175 | pipeline definition in your forked repository. 176 | 177 | After the pipeline has run, you can navigate in the Azure Portal to the Azure ML Workspace 178 | and visualize the Azure ML pipeline result. 179 | 180 | ![ML lifecycle](docs/images/azureml-pipeline.png) 181 | 182 | The pipeline output shows a reference to the storage account location where the output 183 | data is stored. You can navigate to that dataset in the Azure Portal. 184 | 185 | ![ML lifecycle](docs/images/output-dataset.png) 186 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | name: ML-Pipeline 2 | 3 | resources: 4 | containers: 5 | - container: mlops 6 | endpoint: DataOpsML Azure Container Registry 7 | image: azureml-pipeline-databricks 8 | 9 | pr: none 10 | trigger: 11 | branches: 12 | include: 13 | - master 14 | paths: 15 | include: 16 | - Python/azureml-pipeline-databricks/ 17 | 18 | variables: 19 | - group: terraform 20 | - template: variables.yml 21 | 22 | pool: 23 | vmImage: ubuntu-latest 24 | 25 | stages: 26 | 27 | - stage: AML_Pipeline_CI 28 | displayName: Azure ML CI 29 | jobs: 30 | - job: ReadTerraform 31 | displayName: Use Terraform outputs 32 | steps: 33 | 34 | - template: environment_setup/terraform-init-template.yml 35 | parameters: 36 | TerraformDirectory: Python/azureml-pipeline-databricks/environment_setup/terraform 37 | environment: test 38 | 39 | - template: environment_setup/terraform-outputs-template.yml 40 | parameters: 41 | TerraformDirectory: Python/azureml-pipeline-databricks/environment_setup/terraform 42 | 43 | - job: Build_Pipeline 44 | dependsOn: ReadTerraform 45 | displayName: Build ML Training Pipeline 46 | container: mlops 47 | timeoutInMinutes: 0 48 | variables: 49 | SUBSCRIPTION_ID: $[ dependencies.ReadTerraform.outputs['Outputs.subscription_id'] ] 50 | RESOURCE_GROUP: $[ dependencies.ReadTerraform.outputs['Outputs.resource_group_name'] ] 51 | AML_WORKSPACE_NAME: $[ dependencies.ReadTerraform.outputs['Outputs.aml_workspace_name'] ] 52 | DATABRICKS_WORKSPACE_NAME: $[ dependencies.ReadTerraform.outputs['Outputs.databricks_workspace_name'] ] 53 | TRAINING_DATA_ACCOUNT_NAME: $[ dependencies.ReadTerraform.outputs['Outputs.training_data_account_name'] ] 54 | 55 | steps: 56 | - script: | 57 | flake8 --output-file=lint-testresults.xml --format junit-xml 58 | displayName: 'Run lint tests' 59 | workingDirectory: Python/azureml-pipeline-databricks 60 | 61 | - script: | 62 | python -m pytest . --cov=code --cov-report=html --cov-report=xml --junitxml=unit-testresults.xml 63 | condition: succeededOrFailed() 64 | displayName: 'Run unit tests' 65 | workingDirectory: Python/azureml-pipeline-databricks 66 | 67 | - task: PublishTestResults@2 68 | condition: succeededOrFailed() 69 | inputs: 70 | testResultsFiles: '**/*-testresults.xml' 71 | testRunTitle: 'Linting & Unit tests' 72 | failTaskOnFailedTests: true 73 | displayName: 'Publish test results' 74 | 75 | - task: PublishCodeCoverageResults@1 76 | displayName: 'Publish coverage report' 77 | condition: succeededOrFailed() 78 | inputs: 79 | codeCoverageTool: Cobertura 80 | summaryFileLocation: '**/coverage.xml' 81 | reportDirectory: '**/htmlcov' 82 | failIfCoverageEmpty: true 83 | 84 | - task: AzureCLI@1 85 | name: TrainPipeline 86 | inputs: 87 | azureSubscription: '$(ARM_SVC_CONNECTION)' 88 | scriptLocation: inlineScript 89 | workingDirectory: Python/azureml-pipeline-databricks 90 | inlineScript: | 91 | set -e # fail on error 92 | 93 | # Set up environment and build AML pipeline. 94 | # Also sets the AMLPIPELINE_ID job variable. 95 | python -m ml_service.build_ml_pipeline 96 | 97 | displayName: 'Publish Azure Machine Learning Pipeline' 98 | 99 | - job: "Run_ML_Pipeline" 100 | dependsOn: Build_Pipeline 101 | displayName: "Trigger ML Training Pipeline" 102 | pool: server 103 | variables: 104 | SUBSCRIPTION_ID: $[ dependencies.ReadTerraform.outputs['Outputs.subscription_id'] ] 105 | RESOURCE_GROUP: $[ dependencies.ReadTerraform.outputs['Outputs.resource_group_name'] ] 106 | AML_WORKSPACE_NAME: $[ dependencies.ReadTerraform.outputs['Outputs.aml_workspace_name'] ] 107 | DATABRICKS_WORKSPACE_NAME: $[ dependencies.ReadTerraform.outputs['Outputs.databricks_workspace_name'] ] 108 | AMLPIPELINE_ID: $[ dependencies.Build_Pipeline.outputs['TrainPipeline.AMLPIPELINE_ID'] ] 109 | steps: 110 | - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 111 | displayName: 'Invoke ML pipeline' 112 | inputs: 113 | azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' 114 | PipelineId: '$(AMLPIPELINE_ID)' 115 | ExperimentName: '$(EXPERIMENT_NAME)' 116 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/ci_dependencies.yml: -------------------------------------------------------------------------------- 1 | name: dataops_ci 2 | 3 | dependencies: 4 | 5 | # The python interpreter version. 6 | - python=3.7.5 7 | 8 | - psutil=5.6 #latest 9 | 10 | - pip=20.0.2 11 | - pip: 12 | 13 | # dependencies shared with other environment .yml files. 14 | - azureml-sdk==1.0.85 15 | 16 | # Additional pip dependencies for the CI environment. 17 | - databricks-client==0.0.2 18 | - databricks-test==0.0.1 19 | - pytest==5.3.1 20 | - pytest-mock==1.13.0 21 | - pytest-cov==2.8.1 22 | - requests==2.22.0 23 | - python-dotenv==0.10.3 24 | - flake8==3.7.9 25 | - flake8_formatter_junit_xml==0.0.6 26 | - azure-cli==2.1.0 27 | - tox==3.14.3 28 | # required for azure.common.credentials.get_azure_cli_credentials 29 | - azure-core==1.2.2 30 | # Runtime modules for Spark - match Databricks runtime 6.2 31 | - pyspark==2.4.4 32 | - pandas==0.24.2 33 | - pyarrow==0.13.0 34 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/code/prepare/feature_engineering.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | # This notebook processed the training dataset (imported by Data Factory) 3 | # and computes a cleaned dataset with additional feature 4 | 5 | # COMMAND ---------- 6 | 7 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 8 | if 'dbutils' not in locals(): 9 | import databricks_test 10 | databricks_test.inject_variables() 11 | 12 | # COMMAND ---------- 13 | 14 | # Widgets for interactive development. Values are automatically passed when run in Azure ML. 15 | # In interactive development, use Databricks CLI to provision the proper secrets. 16 | 17 | # Data input: training data 18 | 19 | dbutils.widgets.text("secretscope", "amlscope") 20 | dbutils.widgets.text( 21 | "training", "wasbs://trainingdata@stmyappdataopstrstage.blob.core.windows.net/" 22 | ) 23 | dbutils.widgets.text( 24 | "training_blob_config", 25 | "fs.azure.account.key.stmyappdataopstrstage.blob.core.windows.net", 26 | ) 27 | dbutils.widgets.text( 28 | "training_blob_secretname", "trainingdata@stmyappdataopstrstage" 29 | ) 30 | 31 | # Data output: feature engineered data 32 | 33 | dbutils.widgets.text( 34 | "feature_engineered", "wasbs://feature_engineereddata@stmyappdataopstrstage.blob.core.windows.net/" 35 | ) 36 | dbutils.widgets.text( 37 | "feature_engineered_blob_config", 38 | "fs.azure.account.key.MYACCOUNT.blob.core.windows.net" 39 | ) 40 | dbutils.widgets.text( 41 | "feature_engineered_blob_secretname", "MYCONTAINER@MYACCOUNT" 42 | ) 43 | 44 | # COMMAND ---------- 45 | 46 | # Connect to Azure ML 47 | dbutils.library.installPyPI( 48 | "azureml-sdk", 49 | version="1.0.85", 50 | extras="databricks") 51 | from azureml.core import Run 52 | # In an Azure ML run, settings get imported from passed --AZUREML_* parameters 53 | run = Run.get_context(allow_offline=True) 54 | 55 | # COMMAND ---------- 56 | 57 | # Set up storage credentials 58 | 59 | spark.conf.set( 60 | dbutils.widgets.get("training_blob_config"), 61 | dbutils.secrets.get( 62 | scope=dbutils.widgets.get("secretscope"), 63 | key=dbutils.widgets.get("training_blob_secretname") 64 | ), 65 | ) 66 | 67 | spark.conf.set( 68 | dbutils.widgets.get("feature_engineered_blob_config"), 69 | dbutils.secrets.get( 70 | scope=dbutils.widgets.get("secretscope"), 71 | key=dbutils.widgets.get("feature_engineered_blob_secretname") 72 | ), 73 | ) 74 | 75 | # COMMAND ---------- 76 | 77 | rawdata = ( 78 | spark.read.format("csv") 79 | .options(header="true", mode="FAILFAST") 80 | .load(dbutils.widgets.get('training')) 81 | ) 82 | 83 | # COMMAND ---------- 84 | 85 | # Basic data cleanup - remove rows with missing data 86 | cleandata = rawdata.na.drop() 87 | 88 | # COMMAND ---------- 89 | 90 | # Write data to CSV 91 | 92 | dest_path_csv = dbutils.widgets.get('feature_engineered') 93 | cleandata.toPandas().to_csv("/tmp/output.csv", index=False) 94 | csv_file = "%s/engineered.csv" % (dest_path_csv) 95 | assert dbutils.fs.cp("file:/tmp/output.csv", csv_file) 96 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/code/tests/feature_engineering_expected.csv: -------------------------------------------------------------------------------- 1 | AGE,SEX,BMI,BP,S1,S2,S3,S4,S5,S6,Y 2 | 0.0852989062966783,0.0506801187398187,0.0444512133365941,-0.00567061055493425,-0.0455994512826475,-0.0341944659141195,-0.0323559322397657,-0.00259226199818282,0.00286377051894013,-0.0259303389894746,141.0 3 | -0.0890629393522603,-0.044641636506989,-0.0115950145052127,-0.0366564467985606,0.0121905687618,0.0249905933641021,-0.0360375700438527,0.0343088588777263,0.0226920225667445,-0.0093619113301358,206.0 4 | 0.00538306037424807,-0.044641636506989,-0.0363846922044735,0.0218723549949558,0.00393485161259318,0.0155961395104161,0.0081420836051921,-0.00259226199818282,-0.0319914449413559,-0.0466408735636482,135.0 5 | -0.0926954778032799,-0.044641636506989,-0.0406959404999971,-0.0194420933298793,-0.0689906498720667,-0.0792878444118122,0.0412768238419757,-0.076394503750001,-0.0411803851880079,-0.0963461565416647,97.0 6 | -0.0454724779400257,0.0506801187398187,-0.0471628129432825,-0.015999222636143,-0.040095639849843,-0.0248000120604336,0.000778807997017968,-0.0394933828740919,-0.0629129499162512,-0.0383566597339788,138.0 7 | 0.063503675590561,0.0506801187398187,-0.00189470584028465,0.0666296740135272,0.0906198816792644,0.108914381123697,0.0228686348215404,0.0177033544835672,-0.0358167281015492,0.00306440941436832,63.0 8 | 0.0417084448844436,0.0506801187398187,0.0616962065186885,-0.0400993174922969,-0.0139525355440215,0.00620168565673016,-0.0286742944356786,-0.00259226199818282,-0.0149564750249113,0.0113486232440377,110.0 9 | -0.0709002470971626,-0.044641636506989,0.0390621529671896,-0.0332135761048244,-0.0125765826858204,-0.034507614375909,-0.0249926566315915,-0.00259226199818282,0.0677363261102861,-0.0135040182449705,310.0 10 | -0.0963280162542995,-0.044641636506989,-0.0838084234552331,0.0081008722200108,-0.103389471327095,-0.0905611890362353,-0.0139477432193303,-0.076394503750001,-0.0629129499162512,-0.0342145528191441,101.0 11 | 0.0271782910803654,0.0506801187398187,0.0175059114895716,-0.0332135761048244,-0.00707277125301585,0.0459715403040008,-0.0654906724765493,0.0712099797536354,-0.096433222891784,-0.0590671943081523,69.0 12 | 0.0162806757273067,-0.044641636506989,-0.0288400076873072,-0.00911348124867051,-0.00432086553661359,-0.00976888589453599,0.0449584616460628,-0.0394933828740919,-0.0307512098645563,-0.0424987666488135,179.0 13 | 0.00538306037424807,0.0506801187398187,-0.00189470584028465,0.0081008722200108,-0.00432086553661359,-0.0157187066685371,-0.0029028298070691,-0.00259226199818282,0.0383932482116977,-0.0135040182449705,185.0 14 | 0.0453409833354632,-0.044641636506989,-0.0256065714656645,-0.0125563519424068,0.0176943801946045,-6.12835790604833e-05,0.0817748396869335,-0.0394933828740919,-0.0319914449413559,-0.0756356219674911,118.0 15 | -0.0527375548420648,0.0506801187398187,-0.0180618869484982,0.0804011567884723,0.0892439288210632,0.107661787276539,-0.0397192078479398,0.108111100629544,0.0360557900898319,-0.0424987666488135,171.0 16 | -0.00551455497881059,-0.044641636506989,0.0422955891888323,0.0494153205448459,0.0245741444856101,-0.0238605666750649,0.0744115640787594,-0.0394933828740919,0.0522799997967812,0.0279170509033766,166.0 17 | 0.0707687524926,0.0506801187398187,0.0121168511201671,0.0563010619323185,0.034205814493018,0.0494161733836856,-0.0397192078479398,0.0343088588777263,0.027367707542609,-0.00107769750046639,144.0 18 | -0.0382074010379866,-0.044641636506989,-0.0105172024313319,-0.0366564467985606,-0.0373437341334407,-0.0194764882100115,-0.0286742944356786,-0.00259226199818282,-0.0181182673078967,-0.0176461251598052,97.0 19 | -0.0273097856849279,-0.044641636506989,-0.0180618869484982,-0.0400993174922969,-0.00294491267841247,-0.0113346282034837,0.0375951860378887,-0.0394933828740919,-0.0089440189577978,-0.0549250873933176,168.0 20 | -0.0491050163910452,-0.044641636506989,-0.0568631216082106,-0.0435421881860331,-0.0455994512826475,-0.043275771306016,0.000778807997017968,-0.0394933828740919,-0.0119006848015081,0.0154907301588724,68.0 21 | -0.0854304009012408,0.0506801187398187,-0.0223731352440218,0.00121513083253827,-0.0373437341334407,-0.0263657543693812,0.0155053592133662,-0.0394933828740919,-0.072128454601956,-0.0176461251598052,49.0 22 | -0.0854304009012408,-0.044641636506989,-0.00405032998804645,-0.00911348124867051,-0.00294491267841247,0.00776742796567782,0.0228686348215404,-0.0394933828740919,-0.0611765950943345,-0.0135040182449705,68.0 23 | 0.0453409833354632,0.0506801187398187,0.0606183944448076,0.0310533436263482,0.0287020030602135,-0.0473467013092799,-0.0544457590642881,0.0712099797536354,0.133598980013008,0.135611830689079,245.0 24 | -0.0636351701951234,-0.044641636506989,0.0358287167455469,-0.0228849640236156,-0.0304639698424351,-0.0188501912864324,-0.00658446761115617,-0.00259226199818282,-0.0259524244351894,-0.0549250873933176,184.0 25 | -0.067267708646143,0.0506801187398187,-0.0126728265790937,-0.0400993174922969,-0.0153284884022226,0.0046359433477825,-0.0581273968683752,0.0343088588777263,0.0191990330785671,-0.0342145528191441,202.0 26 | -0.107225631607358,-0.044641636506989,-0.0773415510119477,-0.0263278347173518,-0.0896299427450836,-0.0961978613484469,0.0265502726256275,-0.076394503750001,-0.0425721049227942,-0.0052198044153011,137.0 27 | -0.0236772472339084,-0.044641636506989,0.0595405823709267,-0.0400993174922969,-0.0428475455662452,-0.0435889197678055,0.0118237214092792,-0.0394933828740919,-0.0159982677581387,0.0403433716478807,85.0 28 | 0.0526060602375023,-0.044641636506989,-0.0212953231701409,-0.0745280244296595,-0.040095639849843,-0.0376390989938044,-0.00658446761115617,-0.0394933828740919,-0.000609254186102297,-0.0549250873933176,131.0 29 | 0.0671362140415805,0.0506801187398187,-0.00620595413580824,0.063186803319791,-0.0428475455662452,-0.0958847128866574,0.052321737254237,-0.076394503750001,0.0594238004447941,0.0527696923923848,283.0 30 | -0.0600026317441039,-0.044641636506989,0.0444512133365941,-0.0194420933298793,-0.00982467696941811,-0.00757684666200928,0.0228686348215404,-0.0394933828740919,-0.0271286455543265,-0.0093619113301358,129.0 31 | -0.0236772472339084,-0.044641636506989,-0.0654856181992578,-0.081413765817132,-0.0387196869916418,-0.0536096705450705,0.0596850128624111,-0.076394503750001,-0.0371283460104736,-0.0424987666488135,59.0 32 | 0.0344433679824045,0.0506801187398187,0.125287118877662,0.0287580963824284,-0.0538551684318543,-0.0129003705124313,-0.10230705051742,0.108111100629544,0.000271485727907132,0.0279170509033766,341.0 33 | 0.030810829531385,-0.044641636506989,-0.0503962491649252,-0.00222773986119799,-0.0442234984244464,-0.0899348921126563,0.118591217727804,-0.076394503750001,-0.0181182673078967,0.00306440941436832,87.0 34 | 0.0162806757273067,-0.044641636506989,-0.063329994051496,-0.0573136709609782,-0.0579830270064577,-0.0489124436182275,0.0081420836051921,-0.0394933828740919,-0.0594726974107223,-0.0673514081378217,65.0 35 | 0.0489735217864827,0.0506801187398187,-0.030995631835069,-0.0492803060204031,0.0493412959332305,-0.00413221358232442,0.133317768944152,-0.0535158088069373,0.0213108465682448,0.0196328370737072,102.0 36 | 0.0126481372762872,-0.044641636506989,0.0228949718589761,0.0528581912385822,0.00806271018719657,-0.0285577936019079,0.0375951860378887,-0.0394933828740919,0.0547240033481791,-0.0259303389894746,265.0 37 | -0.00914709342983014,-0.044641636506989,0.0110390390462862,-0.0573136709609782,-0.0249601584096305,-0.0429626228442264,0.0302319104297145,-0.0394933828740919,0.01703713241478,-0.0052198044153011,276.0 38 | -0.00188201652779104,0.0506801187398187,0.0713965151836166,0.0976155102571536,0.0878679759628621,0.0754074957122168,-0.0213110188275045,0.0712099797536354,0.0714240327805764,0.0237749439885419,252.0 39 | -0.00188201652779104,0.0506801187398187,0.0142724752679289,-0.0745280244296595,0.00255889875439205,0.00620168565673016,-0.0139477432193303,-0.00259226199818282,0.0191990330785671,0.00306440941436832,90.0 40 | 0.00538306037424807,0.0506801187398187,-0.00836157828357004,0.0218723549949558,0.054845107366035,0.07321545647969,-0.0249926566315915,0.0343088588777263,0.0125531528133893,0.094190761540732,100.0 41 | -0.099960554705319,-0.044641636506989,-0.0676412423470196,-0.108956731367022,-0.0744944613048712,-0.072711726714232,0.0155053592133662,-0.0394933828740919,-0.0498684677352306,-0.0093619113301358,55.0 42 | -0.0600026317441039,0.0506801187398187,-0.0105172024313319,-0.0148515990830405,-0.0497273098572509,-0.0235474182132754,-0.0581273968683752,0.0158582984397717,-0.00991895736315477,-0.0342145528191441,61.0 43 | 0.0199132141783263,-0.044641636506989,-0.0234509473179027,-0.0710851537359232,0.0204462859110067,-0.0100820343563255,0.118591217727804,-0.076394503750001,-0.0425721049227942,0.0734802269665584,92.0 44 | 0.0453409833354632,0.0506801187398187,0.068163078961974,0.0081008722200108,-0.0167044412604238,0.0046359433477825,-0.0765355858888105,0.0712099797536354,0.0324332257796019,-0.0176461251598052,259.0 45 | 0.0271782910803654,0.0506801187398187,-0.0353068801305926,0.0322009670761646,-0.0112006298276192,0.00150445872988718,-0.0102661054152432,-0.00259226199818282,-0.0149564750249113,-0.0507829804784829,53.0 46 | -0.0563700932930843,-0.044641636506989,-0.0115950145052127,-0.0332135761048244,-0.0469754041408486,-0.0476598497710694,0.00446044580110504,-0.0394933828740919,-0.00797939755454164,-0.0880619427119953,190.0 47 | -0.0781653239992017,-0.044641636506989,-0.0730303027164241,-0.0573136709609782,-0.0841261313122791,-0.0742774690231797,-0.0249926566315915,-0.0394933828740919,-0.0181182673078967,-0.0839198357971606,142.0 48 | 0.0671362140415805,0.0506801187398187,-0.041773752573878,0.0115437429137471,0.00255889875439205,0.00588853719494063,0.0412768238419757,-0.0394933828740919,-0.0594726974107223,-0.0217882320746399,75.0 49 | -0.0418399394890061,0.0506801187398187,0.0142724752679289,-0.00567061055493425,-0.0125765826858204,0.00620168565673016,-0.0728539480847234,0.0712099797536354,0.0354619386607697,-0.0135040182449705,142.0 50 | 0.0344433679824045,-0.044641636506989,-0.00728376620968916,0.0149866136074833,-0.0442234984244464,-0.0373259505320149,-0.0029028298070691,-0.0394933828740919,-0.02139368094036,0.00720651632920303,155.0 51 | 0.0598711371395414,0.0506801187398187,0.0164280994156907,0.0287580963824284,-0.0414715927080441,-0.029184090525487,-0.0286742944356786,-0.00259226199818282,-0.00239668149341427,-0.0217882320746399,225.0 52 | -0.0527375548420648,-0.044641636506989,-0.00943939035745095,-0.00567061055493425,0.0397096259258226,0.0447189464568426,0.0265502726256275,-0.00259226199818282,-0.0181182673078967,-0.0135040182449705,59.0 53 | -0.00914709342983014,-0.044641636506989,-0.0159062628007364,0.0700725447072635,0.0121905687618,0.0221722572079963,0.0155053592133662,-0.00259226199818282,-0.0332487872476258,0.0486275854775501,104.0 54 | -0.0491050163910452,-0.044641636506989,0.0250505960067379,0.0081008722200108,0.0204462859110067,0.0177881787429428,0.052321737254237,-0.0394933828740919,-0.0411803851880079,0.00720651632920303,182.0 55 | -0.0418399394890061,-0.044641636506989,-0.0493184370910443,-0.0366564467985606,-0.00707277125301585,-0.0226079728279068,0.0854564774910206,-0.0394933828740919,-0.0664881482228354,0.00720651632920303,128.0 56 | -0.0418399394890061,-0.044641636506989,0.0412177771149514,-0.0263278347173518,-0.0318399227006362,-0.0304366843726451,-0.0360375700438527,0.00294290613320356,0.0336568129023847,-0.0176461251598052,52.0 57 | -0.0273097856849279,-0.044641636506989,-0.063329994051496,-0.0504279295735057,-0.0896299427450836,-0.104339721354975,0.052321737254237,-0.076394503750001,-0.0561575730950062,-0.0673514081378217,37.0 58 | 0.0417084448844436,-0.044641636506989,-0.064407806125377,0.0356438377699009,0.0121905687618,-0.057993749010124,0.181179060397284,-0.076394503750001,-0.000609254186102297,-0.0507829804784829,170.0 59 | 0.063503675590561,0.0506801187398187,-0.0256065714656645,0.0115437429137471,0.0644767773734429,0.048476727998317,0.0302319104297145,-0.00259226199818282,0.0383932482116977,0.0196328370737072,170.0 60 | -0.0709002470971626,-0.044641636506989,-0.00405032998804645,-0.0400993174922969,-0.0662387441556644,-0.0786615474882331,0.052321737254237,-0.076394503750001,-0.0514005352605825,-0.0342145528191441,61.0 61 | -0.0418399394890061,0.0506801187398187,0.00457216660300077,-0.0538708002672419,-0.0442234984244464,-0.0273051997547498,-0.0802172236928976,0.0712099797536354,0.0366457977933988,0.0196328370737072,144.0 62 | -0.0273097856849279,0.0506801187398187,-0.00728376620968916,-0.0400993174922969,-0.0112006298276192,-0.0138398158977999,0.0596850128624111,-0.0394933828740919,-0.0823814832581028,-0.0259303389894746,52.0 63 | -0.034574862586967,-0.044641636506989,-0.0374625042783544,-0.0607565416547144,0.0204462859110067,0.0434663526096845,-0.0139477432193303,-0.00259226199818282,-0.0307512098645563,-0.0714935150526564,128.0 64 | 0.0671362140415805,0.0506801187398187,-0.0256065714656645,-0.0400993174922969,-0.0634868384392622,-0.0598726397808612,-0.0029028298070691,-0.0394933828740919,-0.0191970476139445,0.0113486232440377,71.0 65 | -0.0454724779400257,0.0506801187398187,-0.0245287593917836,0.0597439326260547,0.00531080447079431,0.0149698425868371,-0.0544457590642881,0.0712099797536354,0.0423448954496075,0.0154907301588724,163.0 66 | -0.00914709342983014,0.0506801187398187,-0.0180618869484982,-0.0332135761048244,-0.0208322998350272,0.0121515064307313,-0.0728539480847234,0.0712099797536354,0.000271485727907132,0.0196328370737072,150.0 67 | 0.0417084448844436,0.0506801187398187,-0.0148284507268555,-0.0171468461892456,-0.00569681839481472,0.00839372488925688,-0.0139477432193303,-0.00185423958066465,-0.0119006848015081,0.00306440941436832,97.0 68 | 0.0380759064334241,0.0506801187398187,-0.0299178197611881,-0.0400993174922969,-0.0332158755588373,-0.0241737151368545,-0.0102661054152432,-0.00259226199818282,-0.0129079422541688,0.00306440941436832,160.0 69 | 0.0162806757273067,-0.044641636506989,-0.0460850008694016,-0.00567061055493425,-0.0758704141630723,-0.0614383820898088,-0.0139477432193303,-0.0394933828740919,-0.0514005352605825,0.0196328370737072,178.0 70 | -0.00188201652779104,-0.044641636506989,-0.0697968664947814,-0.0125563519424068,-0.000193006962010205,-0.00914258897095694,0.0707299262746723,-0.0394933828740919,-0.0629129499162512,0.0403433716478807,48.0 71 | -0.00188201652779104,-0.044641636506989,0.0336730925977851,0.125158475807044,0.0245741444856101,0.0262431872112602,-0.0102661054152432,-0.00259226199818282,0.0267142576335128,0.0610539062220542,270.0 72 | 0.063503675590561,0.0506801187398187,-0.00405032998804645,-0.0125563519424068,0.103003457403075,0.0487898764601065,0.056003375058324,-0.00259226199818282,0.0844952822124031,-0.0176461251598052,202.0 73 | 0.0126481372762872,0.0506801187398187,-0.02021751109626,-0.00222773986119799,0.0383336730676214,0.05317395492516,-0.00658446761115617,0.0343088588777263,-0.00514530798026311,-0.0093619113301358,111.0 74 | 0.0126481372762872,0.0506801187398187,0.00241654245523897,0.0563010619323185,0.0273260502020124,0.0171618818193638,0.0412768238419757,-0.0394933828740919,0.00371173823343597,0.0734802269665584,85.0 75 | -0.00914709342983014,0.0506801187398187,-0.030995631835069,-0.0263278347173518,-0.0112006298276192,-0.00100072896442909,-0.0213110188275045,-0.00259226199818282,0.0062093156165054,0.0279170509033766,42.0 76 | -0.0309423241359475,0.0506801187398187,0.0282840322283806,0.0700725447072635,-0.126780669916514,-0.106844909049291,-0.0544457590642881,-0.047980640675551,-0.0307512098645563,0.0154907301588724,170.0 77 | -0.0963280162542995,-0.044641636506989,-0.0363846922044735,-0.0745280244296595,-0.0387196869916418,-0.0276183482165393,0.0155053592133662,-0.0394933828740919,-0.0740888714915354,-0.00107769750046639,200.0 78 | 0.00538306037424807,-0.044641636506989,-0.0579409336820915,-0.0228849640236156,-0.0676146970138656,-0.0683276482491785,-0.0544457590642881,-0.00259226199818282,0.0428956878925287,-0.0839198357971606,252.0 79 | -0.103593093156339,-0.044641636506989,-0.0374625042783544,-0.0263278347173518,0.00255889875439205,0.0199802179754696,0.0118237214092792,-0.00259226199818282,-0.0683297436244215,-0.0259303389894746,113.0 80 | 0.0707687524926,-0.044641636506989,0.0121168511201671,0.0425295791573734,0.0713565416644485,0.0534871033869495,0.052321737254237,-0.00259226199818282,0.0253931349154494,-0.0052198044153011,143.0 81 | 0.0126481372762872,0.0506801187398187,-0.0223731352440218,-0.0297707054110881,0.0108146159035988,0.0284352264437869,-0.0213110188275045,0.0343088588777263,-0.00608024819631442,-0.00107769750046639,51.0 82 | -0.0164121703318693,-0.044641636506989,-0.0353068801305926,-0.0263278347173518,0.0328298616348169,0.0171618818193638,0.100183028707369,-0.0394933828740919,-0.0702093127286876,-0.0797777288823259,52.0 83 | -0.0382074010379866,-0.044641636506989,0.00996122697240527,-0.0469850588797694,-0.0593589798646588,-0.0529833736214915,-0.0102661054152432,-0.0394933828740919,-0.0159982677581387,-0.0424987666488135,210.0 84 | 0.00175052192322852,-0.044641636506989,-0.0396181284261162,-0.100923366426447,-0.0290880169842339,-0.0301235359108556,0.0449584616460628,-0.0501947079281055,-0.0683297436244215,-0.129483011860342,65.0 85 | 0.0453409833354632,-0.044641636506989,0.0713965151836166,0.00121513083253827,-0.00982467696941811,-0.00100072896442909,0.0155053592133662,-0.0394933828740919,-0.0411803851880079,-0.0714935150526564,141.0 86 | -0.0709002470971626,0.0506801187398187,-0.0751859268641859,-0.0400993174922969,-0.051103262715452,-0.015092409744958,-0.0397192078479398,-0.00259226199818282,-0.096433222891784,-0.0342145528191441,55.0 87 | 0.0453409833354632,-0.044641636506989,-0.00620595413580824,0.0115437429137471,0.0631008245152418,0.0162224364339952,0.0965013909032818,-0.0394933828740919,0.0428956878925287,-0.0383566597339788,134.0 88 | -0.0527375548420648,0.0506801187398187,-0.0406959404999971,-0.067642283042187,-0.0318399227006362,-0.0370128020702253,0.0375951860378887,-0.0394933828740919,-0.0345237153303495,0.0693381200517237,42.0 89 | -0.0454724779400257,-0.044641636506989,-0.0482406250171634,-0.0194420933298793,-0.000193006962010205,-0.0160318551303266,0.0670482884705852,-0.0394933828740919,-0.0247911874324607,0.0196328370737072,111.0 90 | 0.0126481372762872,-0.044641636506989,-0.0256065714656645,-0.0400993174922969,-0.0304639698424351,-0.0451546620767532,0.0780932018828464,-0.076394503750001,-0.072128454601956,0.0113486232440377,98.0 91 | 0.0453409833354632,-0.044641636506989,0.0519958978537604,-0.0538708002672419,0.0631008245152418,0.0647604480113727,-0.0102661054152432,0.0343088588777263,0.0372320112089689,0.0196328370737072,164.0 92 | -0.0200447087828888,-0.044641636506989,0.00457216660300077,0.0976155102571536,0.00531080447079431,-0.0207290820571696,0.0633666506664982,-0.0394933828740919,0.0125531528133893,0.0113486232440377,48.0 93 | -0.0491050163910452,-0.044641636506989,-0.064407806125377,-0.10207098997955,-0.00294491267841247,-0.0154055582067476,0.0633666506664982,-0.0472426182580328,-0.0332487872476258,-0.0549250873933176,96.0 94 | -0.0781653239992017,-0.044641636506989,-0.0169840748746173,-0.0125563519424068,-0.000193006962010205,-0.0135266674360104,0.0707299262746723,-0.0394933828740919,-0.0411803851880079,-0.09220404962683,90.0 95 | -0.0709002470971626,-0.044641636506989,-0.0579409336820915,-0.081413765817132,-0.0455994512826475,-0.0288709420636975,-0.0434008456520269,-0.00259226199818282,0.00114379737951254,-0.0052198044153011,162.0 96 | 0.0562385986885218,0.0506801187398187,0.00996122697240527,0.0494153205448459,-0.00432086553661359,-0.0122740735888523,-0.0434008456520269,0.0343088588777263,0.060787754150744,0.0320591578182113,150.0 97 | -0.0273097856849279,-0.044641636506989,0.088641508365711,-0.0251802111642493,0.0218222387692079,0.0425269072243159,-0.0323559322397657,0.0343088588777263,0.00286377051894013,0.0776223338813931,279.0 98 | 0.00175052192322852,0.0506801187398187,-0.00512814206192736,-0.0125563519424068,-0.0153284884022226,-0.0138398158977999,0.0081420836051921,-0.0394933828740919,-0.00608024819631442,-0.0673514081378217,92.0 99 | -0.00188201652779104,-0.044641636506989,-0.064407806125377,0.0115437429137471,0.0273260502020124,0.0375165318356834,-0.0139477432193303,0.0343088588777263,0.0117839003835759,-0.0549250873933176,83.0 100 | 0.0162806757273067,-0.044641636506989,0.0175059114895716,-0.0228849640236156,0.0603489187988395,0.0444057979950531,0.0302319104297145,-0.00259226199818282,0.0372320112089689,-0.00107769750046639,128.0 101 | 0.0162806757273067,0.0506801187398187,-0.0450071887955207,0.063186803319791,0.0108146159035988,-0.00037443204085002,0.0633666506664982,-0.0394933828740919,-0.0307512098645563,0.036201264733046,102.0 102 | -0.0926954778032799,-0.044641636506989,0.0282840322283806,-0.015999222636143,0.0369577202094203,0.0249905933641021,0.056003375058324,-0.0394933828740919,-0.00514530798026311,-0.00107769750046639,302.0 103 | 0.0598711371395414,0.0506801187398187,0.0412177771149514,0.0115437429137471,0.0410855787840237,0.0707102687853738,-0.0360375700438527,0.0343088588777263,-0.0109044358473771,-0.0300724459043093,198.0 104 | -0.0273097856849279,-0.044641636506989,0.0649296427403312,-0.00222773986119799,-0.0249601584096305,-0.0172844489774848,0.0228686348215404,-0.0394933828740919,-0.0611765950943345,-0.063209301222987,95.0 105 | 0.0235457526293458,0.0506801187398187,-0.0320734439089499,-0.0400993174922969,-0.0318399227006362,-0.0216685274425382,-0.0139477432193303,-0.00259226199818282,-0.0109044358473771,0.0196328370737072,53.0 106 | -0.0963280162542995,-0.044641636506989,-0.0762637389380668,-0.0435421881860331,-0.0455994512826475,-0.0348207628376986,0.0081420836051921,-0.0394933828740919,-0.0594726974107223,-0.0839198357971606,134.0 107 | 0.0271782910803654,-0.044641636506989,0.0498402737059986,-0.0550184238203444,-0.00294491267841247,0.0406480164535787,-0.0581273968683752,0.0527594193156808,-0.0529587932392004,-0.0052198044153011,144.0 108 | 0.0199132141783263,0.0506801187398187,0.045529025410475,0.0299057198322448,-0.062110885581061,-0.0558017097775973,-0.0728539480847234,0.0269286347025444,0.0456008084141249,0.0403433716478807,232.0 109 | 0.0380759064334241,0.0506801187398187,-0.00943939035745095,0.0023627543856408,0.00118294589619092,0.0375165318356834,-0.0544457590642881,0.0501763408543672,-0.0259524244351894,0.106617082285236,81.0 110 | 0.0417084448844436,0.0506801187398187,-0.0320734439089499,-0.0228849640236156,-0.0497273098572509,-0.0401442866881206,0.0302319104297145,-0.0394933828740919,-0.126097385560409,0.0154907301588724,104.0 111 | 0.0199132141783263,-0.044641636506989,0.00457216660300077,-0.0263278347173518,0.023198191627409,0.0102726156599941,0.0670482884705852,-0.0394933828740919,-0.0236445575721341,-0.0466408735636482,59.0 112 | -0.0854304009012408,-0.044641636506989,0.0207393477112143,-0.0263278347173518,0.00531080447079431,0.01966706951368,-0.0029028298070691,-0.00259226199818282,-0.0236445575721341,0.00306440941436832,246.0 113 | 0.0199132141783263,0.0506801187398187,0.0142724752679289,0.063186803319791,0.0149424744782022,0.0202933664372591,-0.0470824834561139,0.0343088588777263,0.0466607723568145,0.0900486546258972,297.0 114 | 0.0235457526293458,-0.044641636506989,0.110197749843329,0.063186803319791,0.0135665216200011,-0.0329418720669614,-0.0249926566315915,0.0206554441536399,0.09924022573399,0.0237749439885419,258.0 115 | -0.0309423241359475,0.0506801187398187,0.00133873038135806,-0.00567061055493425,0.0644767773734429,0.0494161733836856,-0.0470824834561139,0.108111100629544,0.0837967663655224,0.00306440941436832,229.0 116 | 0.0489735217864827,0.0506801187398187,0.0584627702970458,0.0700725447072635,0.0135665216200011,0.0206065148990486,-0.0213110188275045,0.0343088588777263,0.0220040504561505,0.0279170509033766,275.0 117 | 0.0598711371395414,-0.044641636506989,-0.0212953231701409,0.0872868981759448,0.0452134373586271,0.0315667110616823,-0.0470824834561139,0.0712099797536354,0.0791210813896579,0.135611830689079,281.0 118 | -0.0563700932930843,0.0506801187398187,-0.0105172024313319,0.0253152256886921,0.023198191627409,0.0400217195299996,-0.0397192078479398,0.0343088588777263,0.0206123307213641,0.0569117993072195,179.0 119 | 0.0162806757273067,-0.044641636506989,-0.0471628129432825,-0.00222773986119799,-0.019456346976826,-0.0429626228442264,0.0339135482338016,-0.0394933828740919,0.027367707542609,0.0279170509033766,200.0 120 | -0.0491050163910452,-0.044641636506989,0.00457216660300077,0.0115437429137471,-0.0373437341334407,-0.0185370428246429,-0.0176293810234174,-0.00259226199818282,-0.0398095943643375,-0.0217882320746399,200.0 121 | 0.063503675590561,-0.044641636506989,0.0175059114895716,0.0218723549949558,0.00806271018719657,0.0215459602844172,-0.0360375700438527,0.0343088588777263,0.0199084208763183,0.0113486232440377,173.0 122 | 0.0489735217864827,0.0506801187398187,0.0810968238485447,0.0218723549949558,0.0438374845004259,0.0641341510877936,-0.0544457590642881,0.0712099797536354,0.0324332257796019,0.0486275854775501,180.0 123 | 0.00538306037424807,0.0506801187398187,0.034750904671666,-0.00108011630809546,0.152537760298315,0.198787989657293,-0.0618090346724622,0.185234443260194,0.0155668445407018,0.0734802269665584,84.0 124 | -0.00551455497881059,-0.044641636506989,0.023972783932857,0.0081008722200108,-0.0345918284170385,-0.0388916928409625,0.0228686348215404,-0.0394933828740919,-0.0159982677581387,-0.0135040182449705,121.0 125 | -0.00551455497881059,0.0506801187398187,-0.00836157828357004,-0.00222773986119799,-0.0332158755588373,-0.0636304213223356,-0.0360375700438527,-0.00259226199818282,0.0805854642386665,0.00720651632920303,161.0 126 | -0.0890629393522603,-0.044641636506989,-0.0611743699037342,-0.0263278347173518,-0.0552311212900554,-0.0545491159304391,0.0412768238419757,-0.076394503750001,-0.0939356455087147,-0.0549250873933176,99.0 127 | 0.0344433679824045,0.0506801187398187,-0.00189470584028465,-0.0125563519424068,0.0383336730676214,0.0137172487396789,0.0780932018828464,-0.0394933828740919,0.00455189046612778,-0.0963461565416647,109.0 128 | -0.0527375548420648,-0.044641636506989,-0.0622521819776151,-0.0263278347173518,-0.00569681839481472,-0.005071658967693,0.0302319104297145,-0.0394933828740919,-0.0307512098645563,-0.0714935150526564,115.0 129 | 0.00901559882526763,-0.044641636506989,0.0164280994156907,0.00465800152627453,0.0094386630453977,0.0105857641217836,-0.0286742944356786,0.0343088588777263,0.0389683660308856,0.11904340302974,268.0 130 | -0.0636351701951234,0.0506801187398187,0.0961861928828773,0.104501251644626,-0.00294491267841247,-0.00475851050590347,-0.00658446761115617,-0.00259226199818282,0.0226920225667445,0.0734802269665584,274.0 131 | -0.0963280162542995,-0.044641636506989,-0.0697968664947814,-0.067642283042187,-0.019456346976826,-0.0107083312799046,0.0155053592133662,-0.0394933828740919,-0.0468794828442166,-0.0797777288823259,158.0 132 | 0.0162806757273067,0.0506801187398187,-0.0212953231701409,-0.00911348124867051,0.034205814493018,0.047850431074738,0.000778807997017968,-0.00259226199818282,-0.0129079422541688,0.0237749439885419,107.0 133 | -0.0418399394890061,0.0506801187398187,-0.0536296853865679,-0.0400993174922969,-0.0841261313122791,-0.0717722813288634,-0.0029028298070691,-0.0394933828740919,-0.072128454601956,-0.0300724459043093,83.0 134 | -0.0745327855481821,-0.044641636506989,0.0433734012627132,-0.0332135761048244,0.0121905687618,0.000251864882729031,0.0633666506664982,-0.0394933828740919,-0.0271286455543265,-0.0466408735636482,103.0 135 | -0.00551455497881059,-0.044641636506989,0.056307146149284,-0.0366564467985606,-0.0483513569990498,-0.0429626228442264,-0.0728539480847234,0.0379989709653172,0.0507815133629732,0.0569117993072195,272.0 136 | -0.0926954778032799,-0.044641636506989,-0.0816527993074713,-0.0573136709609782,-0.0607349327228599,-0.068014499787389,0.0486400994501499,-0.076394503750001,-0.0664881482228354,-0.0217882320746399,85.0 137 | 0.00538306037424807,-0.044641636506989,0.0498402737059986,0.0976155102571536,-0.0153284884022226,-0.0163450035921162,-0.00658446761115617,-0.00259226199818282,0.01703713241478,-0.0135040182449705,280.0 138 | 0.0344433679824045,0.0506801187398187,0.11127556191721,0.076958286094736,-0.0318399227006362,-0.03388131745233,-0.0213110188275045,-0.00259226199818282,0.028016506523264,0.0734802269665584,336.0 139 | 0.0235457526293458,-0.044641636506989,0.0616962065186885,0.0528581912385822,-0.0345918284170385,-0.0489124436182275,-0.0286742944356786,-0.00259226199818282,0.0547240033481791,-0.0052198044153011,281.0 140 | 0.0417084448844436,0.0506801187398187,0.0142724752679289,0.0425295791573734,-0.0304639698424351,-0.00131387742621863,-0.0434008456520269,-0.00259226199818282,-0.0332487872476258,0.0154907301588724,118.0 141 | -0.0273097856849279,-0.044641636506989,0.0476846495582368,-0.0469850588797694,0.034205814493018,0.0572448849284239,-0.0802172236928976,0.13025177315509,0.0450661683362615,0.131469723774244,317.0 142 | 0.0417084448844436,0.0506801187398187,0.0121168511201671,0.0390867084636372,0.054845107366035,0.0444057979950531,0.00446044580110504,-0.00259226199818282,0.0456008084141249,-0.00107769750046639,235.0 143 | -0.0309423241359475,-0.044641636506989,0.00564997867688165,-0.00911348124867051,0.0190703330528056,0.00682798258030921,0.0744115640787594,-0.0394933828740919,-0.0411803851880079,-0.0424987666488135,60.0 144 | 0.030810829531385,0.0506801187398187,0.0466068374843559,-0.015999222636143,0.0204462859110067,0.0506687672308438,-0.0581273968683752,0.0712099797536354,0.0062093156165054,0.00720651632920303,174.0 145 | -0.0418399394890061,-0.044641636506989,0.128520555099304,0.063186803319791,-0.0332158755588373,-0.0326287236051719,0.0118237214092792,-0.0394933828740919,-0.0159982677581387,-0.0507829804784829,259.0 146 | -0.0309423241359475,0.0506801187398187,0.0595405823709267,0.00121513083253827,0.0121905687618,0.0315667110616823,-0.0434008456520269,0.0343088588777263,0.0148227108412663,0.00720651632920303,178.0 147 | -0.0563700932930843,-0.044641636506989,0.0929527566612346,-0.0194420933298793,0.0149424744782022,0.0234248510551544,-0.0286742944356786,0.0254525898675081,0.0260560896336847,0.0403433716478807,128.0 148 | -0.0600026317441039,0.0506801187398187,0.0153502873418098,-0.0194420933298793,0.0369577202094203,0.0481635795365275,0.0191869970174533,-0.00259226199818282,-0.0307512098645563,-0.00107769750046639,96.0 149 | -0.0491050163910452,0.0506801187398187,-0.00512814206192736,-0.0469850588797694,-0.0208322998350272,-0.0204159335953801,-0.0691723102806364,0.0712099797536354,0.061237907519701,-0.0383566597339788,126.0 150 | 0.0235457526293458,-0.044641636506989,0.0703187031097357,0.0253152256886921,-0.0345918284170385,-0.014466112821379,-0.0323559322397657,-0.00259226199818282,-0.0191970476139445,-0.0093619113301358,288.0 151 | 0.00175052192322852,-0.044641636506989,-0.00405032998804645,-0.00567061055493425,-0.00844872411121698,-0.0238605666750649,0.052321737254237,-0.0394933828740919,-0.0089440189577978,-0.0135040182449705,88.0 152 | -0.034574862586967,0.0506801187398187,-0.000816893766403737,0.0700725447072635,0.0397096259258226,0.0669524872438994,-0.0654906724765493,0.108111100629544,0.0267142576335128,0.0734802269665584,292.0 153 | 0.0417084448844436,0.0506801187398187,-0.0439293767216398,0.063186803319791,-0.00432086553661359,0.0162224364339952,-0.0139477432193303,-0.00259226199818282,-0.0345237153303495,0.0113486232440377,71.0 154 | 0.0671362140415805,0.0506801187398187,0.0207393477112143,-0.00567061055493425,0.0204462859110067,0.0262431872112602,-0.0029028298070691,-0.00259226199818282,0.00864028293306308,0.00306440941436832,197.0 155 | -0.0273097856849279,0.0506801187398187,0.0606183944448076,0.0494153205448459,0.0851160702464598,0.0863676918748504,-0.0029028298070691,0.0343088588777263,0.0378144788263439,0.0486275854775501,186.0 156 | -0.0164121703318693,-0.044641636506989,-0.0105172024313319,0.00121513083253827,-0.0373437341334407,-0.0357602082230672,0.0118237214092792,-0.0394933828740919,-0.02139368094036,-0.0342145528191441,25.0 157 | -0.00188201652779104,0.0506801187398187,-0.0331512559828308,-0.0182944697767768,0.0314539087766158,0.0428400556861055,-0.0139477432193303,0.0199174217361217,0.0102256424049578,0.0279170509033766,84.0 158 | -0.0127796318808497,-0.044641636506989,-0.0654856181992578,-0.0699375301828207,0.00118294589619092,0.0168487333575743,-0.0029028298070691,-0.00702039650329191,-0.0307512098645563,-0.0507829804784829,96.0 159 | -0.00551455497881059,-0.044641636506989,0.0433734012627132,0.0872868981759448,0.0135665216200011,0.00714113104209875,-0.0139477432193303,-0.00259226199818282,0.0423448954496075,-0.0176461251598052,195.0 160 | -0.00914709342983014,-0.044641636506989,-0.0622521819776151,-0.0745280244296595,-0.0235842055514294,-0.0132135189742209,0.00446044580110504,-0.0394933828740919,-0.0358167281015492,-0.0466408735636482,53.0 161 | -0.0454724779400257,0.0506801187398187,0.0638518306664503,0.0700725447072635,0.133274420283499,0.131461070372543,-0.0397192078479398,0.108111100629544,0.0757375884575476,0.0859065477110625,217.0 162 | -0.0527375548420648,-0.044641636506989,0.0304396563761424,-0.0745280244296595,-0.0235842055514294,-0.0113346282034837,-0.0029028298070691,-0.00259226199818282,-0.0307512098645563,-0.00107769750046639,172.0 163 | 0.0162806757273067,0.0506801187398187,0.0724743272574975,0.076958286094736,-0.00844872411121698,0.00557538873315109,-0.00658446761115617,-0.00259226199818282,-0.0236445575721341,0.0610539062220542,131.0 164 | 0.0453409833354632,-0.044641636506989,-0.019139699022379,0.0218723549949558,0.0273260502020124,-0.0135266674360104,0.100183028707369,-0.0394933828740919,0.0177634778671173,-0.0135040182449705,214.0 165 | -0.0418399394890061,-0.044641636506989,-0.0665634302731387,-0.0469850588797694,-0.0373437341334407,-0.043275771306016,0.0486400994501499,-0.0394933828740919,-0.0561575730950062,-0.0135040182449705,59.0 166 | -0.0563700932930843,0.0506801187398187,-0.0600965578298533,-0.0366564467985606,-0.0882539898868825,-0.0708328359434948,-0.0139477432193303,-0.0394933828740919,-0.0781409106690696,-0.104630370371334,70.0 167 | 0.0707687524926,-0.044641636506989,0.0692408910358548,0.0379390850138207,0.0218222387692079,0.00150445872988718,-0.0360375700438527,0.0391060045915944,0.0776327891955595,0.106617082285236,220.0 168 | 0.00175052192322852,0.0506801187398187,0.0595405823709267,-0.00222773986119799,0.0617248716570406,0.063194705702425,-0.0581273968683752,0.108111100629544,0.0689822116363026,0.12732761685941,268.0 169 | -0.00188201652779104,-0.044641636506989,-0.0266843835395454,0.0494153205448459,0.0589729659406384,-0.0160318551303266,-0.0470824834561139,0.0712099797536354,0.133598980013008,0.0196328370737072,152.0 170 | 0.0235457526293458,0.0506801187398187,-0.02021751109626,-0.0366564467985606,-0.0139525355440215,-0.015092409744958,0.0596850128624111,-0.0394933828740919,-0.096433222891784,-0.0176461251598052,47.0 171 | -0.0200447087828888,-0.044641636506989,-0.0460850008694016,-0.0986281192858133,-0.0758704141630723,-0.0598726397808612,-0.0176293810234174,-0.0394933828740919,-0.0514005352605825,-0.0466408735636482,74.0 172 | 0.0417084448844436,0.0506801187398187,0.0713965151836166,0.0081008722200108,0.0383336730676214,0.0159092879722056,-0.0176293810234174,0.0343088588777263,0.0734100780491161,0.0859065477110625,295.0 173 | -0.0636351701951234,0.0506801187398187,-0.0794971751597095,-0.00567061055493425,-0.071742555588469,-0.0664487574784414,-0.0102661054152432,-0.0394933828740919,-0.0181182673078967,-0.0549250873933176,101.0 174 | 0.0162806757273067,0.0506801187398187,0.00996122697240527,-0.0435421881860331,-0.0965097070360893,-0.0946321190394993,-0.0397192078479398,-0.0394933828740919,0.01703713241478,0.00720651632920303,151.0 175 | 0.0671362140415805,-0.044641636506989,-0.0385403163522353,-0.0263278347173518,-0.0318399227006362,-0.0263657543693812,0.0081420836051921,-0.0394933828740919,-0.0271286455543265,0.00306440941436832,127.0 176 | 0.0453409833354632,0.0506801187398187,0.0196615356373334,0.0390867084636372,0.0204462859110067,0.0259300387494707,0.0081420836051921,-0.00259226199818282,-0.003303712578677,0.0196328370737072,237.0 177 | 0.0489735217864827,-0.044641636506989,0.0272062201544997,-0.0251802111642493,0.023198191627409,0.0184144756665219,-0.0618090346724622,0.0800662487638535,0.0722236508199124,0.0320591578182113,225.0 178 | 0.0417084448844436,-0.044641636506989,-0.00836157828357004,-0.0263278347173518,0.0245741444856101,0.0162224364339952,0.0707299262746723,-0.0394933828740919,-0.0483617248028919,-0.0300724459043093,81.0 179 | -0.0236772472339084,-0.044641636506989,-0.0159062628007364,-0.0125563519424068,0.0204462859110067,0.0412743133771578,-0.0434008456520269,0.0343088588777263,0.0140724525157685,-0.0093619113301358,151.0 180 | -0.0382074010379866,0.0506801187398187,0.00457216660300077,0.0356438377699009,-0.0112006298276192,0.00588853719494063,-0.0470824834561139,0.0343088588777263,0.0163049527999418,-0.00107769750046639,107.0 181 | 0.0489735217864827,-0.044641636506989,-0.0428515646477589,-0.0538708002672419,0.0452134373586271,0.0500424703072647,0.0339135482338016,-0.00259226199818282,-0.0259524244351894,-0.063209301222987,64.0 182 | 0.0453409833354632,0.0506801187398187,0.00564997867688165,0.0563010619323185,0.0644767773734429,0.0891860280309562,-0.0397192078479398,0.0712099797536354,0.0155668445407018,-0.0093619113301358,138.0 183 | 0.0453409833354632,0.0506801187398187,-0.0353068801305926,0.063186803319791,-0.00432086553661359,-0.00162702588800815,-0.0102661054152432,-0.00259226199818282,0.0155668445407018,0.0569117993072195,185.0 184 | 0.0162806757273067,-0.044641636506989,0.023972783932857,-0.0228849640236156,-0.0249601584096305,-0.0260526059075917,-0.0323559322397657,-0.00259226199818282,0.0372320112089689,0.0320591578182113,265.0 185 | -0.0745327855481821,0.0506801187398187,-0.0180618869484982,0.0081008722200108,-0.019456346976826,-0.0248000120604336,-0.0654906724765493,0.0343088588777263,0.0673172179146849,-0.0176461251598052,101.0 186 | -0.0817978624502212,0.0506801187398187,0.0422955891888323,-0.0194420933298793,0.0397096259258226,0.0575580333902134,-0.0691723102806364,0.108111100629544,0.0471861678860197,-0.0383566597339788,137.0 187 | -0.067267708646143,-0.044641636506989,-0.0547074974604488,-0.0263278347173518,-0.0758704141630723,-0.082106180567918,0.0486400994501499,-0.076394503750001,-0.0868289932162924,-0.104630370371334,143.0 188 | 0.00538306037424807,-0.044641636506989,-0.00297251791416553,0.0494153205448459,0.0741084473808508,0.0707102687853738,0.0449584616460628,-0.00259226199818282,-0.00149858682029207,-0.0093619113301358,141.0 189 | -0.00188201652779104,-0.044641636506989,-0.0665634302731387,0.00121513083253827,-0.00294491267841247,0.00307020103883484,0.0118237214092792,-0.00259226199818282,-0.0202887477516296,-0.0259303389894746,79.0 190 | 0.00901559882526763,-0.044641636506989,-0.0126728265790937,0.0287580963824284,-0.0180803941186249,-0.005071658967693,-0.0470824834561139,0.0343088588777263,0.0233748412798208,-0.0052198044153011,292.0 191 | -0.00551455497881059,0.0506801187398187,-0.041773752573878,-0.0435421881860331,-0.0799982727376757,-0.0761563597939169,-0.0323559322397657,-0.0394933828740919,0.0102256424049578,-0.0093619113301358,178.0 192 | 0.0562385986885218,0.0506801187398187,-0.030995631835069,0.0081008722200108,0.0190703330528056,0.0212328118226277,0.0339135482338016,-0.0394933828740919,-0.0295276227417736,-0.0590671943081523,91.0 193 | 0.00901559882526763,0.0506801187398187,-0.00512814206192736,-0.0641994123484507,0.0699805888062474,0.0838625041805342,-0.0397192078479398,0.0712099797536354,0.0395398780720242,0.0196328370737072,116.0 194 | -0.067267708646143,-0.044641636506989,-0.0590187457559724,0.0322009670761646,-0.051103262715452,-0.0495387405418066,-0.0102661054152432,-0.0394933828740919,0.00200784054982379,0.0237749439885419,86.0 195 | 0.0271782910803654,0.0506801187398187,0.0250505960067379,0.0149866136074833,0.0259500973438113,0.048476727998317,-0.0397192078479398,0.0343088588777263,0.00783714230182385,0.0237749439885419,122.0 196 | -0.0236772472339084,-0.044641636506989,-0.0460850008694016,-0.0332135761048244,0.0328298616348169,0.0362639379885253,0.0375951860378887,-0.00259226199818282,-0.0332487872476258,0.0113486232440377,72.0 197 | 0.0489735217864827,0.0506801187398187,0.00349435452911985,0.0700725447072635,-0.00844872411121698,0.0134041002778894,-0.0544457590642881,0.0343088588777263,0.0133159679089277,0.036201264733046,129.0 198 | -0.0527375548420648,-0.044641636506989,0.0541515220015222,-0.0263278347173518,-0.0552311212900554,-0.03388131745233,-0.0139477432193303,-0.0394933828740919,-0.0740888714915354,-0.0590671943081523,142.0 199 | 0.0417084448844436,-0.044641636506989,-0.0450071887955207,0.0344962143200845,0.0438374845004259,-0.0157187066685371,0.0375951860378887,-0.0144006206784737,0.089898693277671,0.00720651632920303,90.0 200 | 0.0562385986885218,-0.044641636506989,-0.0579409336820915,-0.00796585769556799,0.0520932016496327,0.0491030249218961,0.056003375058324,-0.0214118336448964,-0.0283202425479987,0.0444854785627154,158.0 201 | -0.034574862586967,0.0506801187398187,-0.0557853095343297,-0.015999222636143,-0.00982467696941811,-0.00788999512379879,0.0375951860378887,-0.0394933828740919,-0.0529587932392004,0.0279170509033766,39.0 202 | 0.0816663678456587,0.0506801187398187,0.00133873038135806,0.0356438377699009,0.126394655992494,0.0910649188016934,0.0191869970174533,0.0343088588777263,0.0844952822124031,-0.0300724459043093,196.0 203 | -0.00188201652779104,0.0506801187398187,0.0304396563761424,0.0528581912385822,0.0397096259258226,0.0566185880048449,-0.0397192078479398,0.0712099797536354,0.0253931349154494,0.0279170509033766,222.0 204 | 0.110726675453815,0.0506801187398187,0.00672779075076256,0.0287580963824284,-0.0277120641260328,-0.00726369820021974,-0.0470824834561139,0.0343088588777263,0.00200784054982379,0.0776223338813931,277.0 205 | -0.0309423241359475,-0.044641636506989,0.0466068374843559,0.0149866136074833,-0.0167044412604238,-0.0470335528474903,0.000778807997017968,-0.00259226199818282,0.0634559213720654,-0.0259303389894746,99.0 206 | 0.00175052192322852,0.0506801187398187,0.0261284080806188,-0.00911348124867051,0.0245741444856101,0.038455977221052,-0.0213110188275045,0.0343088588777263,0.00943640914607987,0.00306440941436832,196.0 207 | 0.00901559882526763,-0.044641636506989,0.045529025410475,0.0287580963824284,0.0121905687618,-0.0138398158977999,0.0265502726256275,-0.0394933828740919,0.0461323310394148,0.036201264733046,202.0 208 | 0.030810829531385,-0.044641636506989,0.0401399650410705,0.076958286094736,0.0176943801946045,0.0378296802974729,-0.0286742944356786,0.0343088588777263,-0.00149858682029207,0.11904340302974,155.0 209 | 0.0380759064334241,0.0506801187398187,-0.0180618869484982,0.0666296740135272,-0.051103262715452,-0.0166581520539057,-0.0765355858888105,0.0343088588777263,-0.0119006848015081,-0.0135040182449705,77.0 210 | 0.00901559882526763,-0.044641636506989,0.0142724752679289,0.0149866136074833,0.054845107366035,0.0472241341511589,0.0707299262746723,-0.0394933828740919,-0.0332487872476258,-0.0590671943081523,191.0 211 | 0.0925639831987174,-0.044641636506989,0.0369065288194278,0.0218723549949558,-0.0249601584096305,-0.0166581520539057,0.000778807997017968,-0.0394933828740919,-0.0225121719296605,-0.0217882320746399,70.0 212 | 0.0671362140415805,-0.044641636506989,0.00349435452911985,0.0356438377699009,0.0493412959332305,0.0312535625998928,0.0707299262746723,-0.0394933828740919,-0.000609254186102297,0.0196328370737072,73.0 213 | 0.00175052192322852,-0.044641636506989,-0.0708746785686623,-0.0228849640236156,-0.00156895982021134,-0.00100072896442909,0.0265502726256275,-0.0394933828740919,-0.0225121719296605,0.00720651632920303,49.0 214 | 0.030810829531385,-0.044641636506989,-0.0331512559828308,-0.0228849640236156,-0.0469754041408486,-0.0811667351825494,0.103864666511456,-0.076394503750001,-0.0398095943643375,-0.0549250873933176,65.0 215 | 0.0271782910803654,0.0506801187398187,0.0940305687351156,0.0976155102571536,-0.0345918284170385,-0.0320024266815928,-0.0434008456520269,-0.00259226199818282,0.0366457977933988,0.106617082285236,263.0 216 | 0.0126481372762872,0.0506801187398187,0.0358287167455469,0.0494153205448459,0.0534691545078339,0.0741549018650587,-0.0691723102806364,0.145012221505454,0.0456008084141249,0.0486275854775501,248.0 217 | 0.0744012909436196,-0.044641636506989,0.0315174684500233,0.10105838095089,0.0465893902168282,0.0368902349121043,0.0155053592133662,-0.00259226199818282,0.0336568129023847,0.0444854785627154,296.0 218 | -0.0418399394890061,-0.044641636506989,-0.0654856181992578,-0.0400993174922969,-0.00569681839481472,0.014343545663258,-0.0434008456520269,0.0343088588777263,0.00702686254915195,-0.0135040182449705,214.0 219 | -0.0890629393522603,-0.044641636506989,-0.041773752573878,-0.0194420933298793,-0.0662387441556644,-0.0742774690231797,0.0081420836051921,-0.0394933828740919,0.00114379737951254,-0.0300724459043093,185.0 220 | 0.0235457526293458,0.0506801187398187,-0.0396181284261162,-0.00567061055493425,-0.0483513569990498,-0.0332550205287509,0.0118237214092792,-0.0394933828740919,-0.101643547945512,-0.0673514081378217,78.0 221 | -0.0454724779400257,-0.044641636506989,-0.0385403163522353,-0.0263278347173518,-0.0153284884022226,0.000878161806308105,-0.0323559322397657,-0.00259226199818282,0.00114379737951254,-0.0383566597339788,93.0 222 | -0.0236772472339084,0.0506801187398187,-0.0256065714656645,0.0425295791573734,-0.0538551684318543,-0.0476598497710694,-0.0213110188275045,-0.0394933828740919,0.00114379737951254,0.0196328370737072,252.0 223 | -0.099960554705319,-0.044641636506989,-0.0234509473179027,-0.0641994123484507,-0.0579830270064577,-0.0601857882426507,0.0118237214092792,-0.0394933828740919,-0.0181182673078967,-0.0507829804784829,150.0 224 | -0.0273097856849279,-0.044641636506989,-0.0665634302731387,-0.112399602060758,-0.0497273098572509,-0.0413968805352788,0.000778807997017968,-0.0394933828740919,-0.0358167281015492,-0.0093619113301358,77.0 225 | 0.030810829531385,0.0506801187398187,0.0325952805239042,0.0494153205448459,-0.040095639849843,-0.0435889197678055,-0.0691723102806364,0.0343088588777263,0.0630166151147464,0.00306440941436832,208.0 226 | -0.103593093156339,0.0506801187398187,-0.0460850008694016,-0.0263278347173518,-0.0249601584096305,-0.0248000120604336,0.0302319104297145,-0.0394933828740919,-0.0398095943643375,-0.0549250873933176,77.0 227 | 0.0671362140415805,0.0506801187398187,-0.0299178197611881,0.0574486853821349,-0.000193006962010205,-0.0157187066685371,0.0744115640787594,-0.0505637191368646,-0.0384591123013538,0.00720651632920303,108.0 228 | -0.0527375548420648,-0.044641636506989,-0.0126728265790937,-0.0607565416547144,-0.000193006962010205,0.00808057642746734,0.0118237214092792,-0.00259226199818282,-0.0271286455543265,-0.0507829804784829,160.0 229 | -0.0273097856849279,0.0506801187398187,-0.0159062628007364,-0.0297707054110881,0.00393485161259318,-0.000687580502639557,0.0412768238419757,-0.0394933828740919,-0.0236445575721341,0.0113486232440377,53.0 230 | -0.0382074010379866,0.0506801187398187,0.0713965151836166,-0.0573136709609782,0.153913713156516,0.155886650392127,0.000778807997017968,0.0719480021711535,0.0502764933899896,0.0693381200517237,220.0 231 | 0.00901559882526763,-0.044641636506989,-0.030995631835069,0.0218723549949558,0.00806271018719657,0.00870687335104641,0.00446044580110504,-0.00259226199818282,0.00943640914607987,0.0113486232440377,154.0 232 | 0.0126481372762872,0.0506801187398187,0.000260918307477141,-0.0114087283893043,0.0397096259258226,0.0572448849284239,-0.0397192078479398,0.0560805201945126,0.024052583226893,0.0320591578182113,259.0 233 | 0.0671362140415805,-0.044641636506989,0.0369065288194278,-0.0504279295735057,-0.0235842055514294,-0.034507614375909,0.0486400994501499,-0.0394933828740919,-0.0259524244351894,-0.0383566597339788,90.0 234 | 0.0453409833354632,-0.044641636506989,0.0390621529671896,0.0459724498511097,0.00668675732899544,-0.0241737151368545,0.0081420836051921,-0.0125555646346783,0.0643282330236709,0.0569117993072195,246.0 235 | 0.0671362140415805,0.0506801187398187,-0.0148284507268555,0.0585963091762383,-0.0593589798646588,-0.034507614375909,-0.0618090346724622,0.012906208769699,-0.00514530798026311,0.0486275854775501,124.0 236 | 0.0271782910803654,-0.044641636506989,0.00672779075076256,0.0356438377699009,0.0796122588136553,0.0707102687853738,0.0155053592133662,0.0343088588777263,0.0406722637144977,0.0113486232440377,67.0 237 | 0.0562385986885218,-0.044641636506989,-0.0687190544209005,-0.0687899065952895,-0.000193006962010205,-0.00100072896442909,0.0449584616460628,-0.0376483268302965,-0.0483617248028919,-0.00107769750046639,72.0 238 | 0.0344433679824045,0.0506801187398187,-0.00943939035745095,0.0597439326260547,-0.0359677812752396,-0.00757684666200928,-0.0765355858888105,0.0712099797536354,0.0110081010458725,-0.0217882320746399,257.0 239 | 0.0235457526293458,-0.044641636506989,0.0196615356373334,-0.0125563519424068,0.0837401173882587,0.0387691256828415,0.0633666506664982,-0.00259226199818282,0.0660482061630984,0.0486275854775501,262.0 240 | 0.0489735217864827,0.0506801187398187,0.0746299514052593,0.0666296740135272,-0.00982467696941811,-0.00225332281158722,-0.0434008456520269,0.0343088588777263,0.0336568129023847,0.0196328370737072,275.0 241 | 0.030810829531385,0.0506801187398187,-0.00836157828357004,0.00465800152627453,0.0149424744782022,0.0274957810584184,0.0081420836051921,-0.00812743012956918,-0.0295276227417736,0.0569117993072195,177.0 242 | -0.103593093156339,0.0506801187398187,-0.0234509473179027,-0.0228849640236156,-0.0868780370286814,-0.0677013513255995,-0.0176293810234174,-0.0394933828740919,-0.0781409106690696,-0.0714935150526564,71.0 243 | 0.0162806757273067,0.0506801187398187,-0.0460850008694016,0.0115437429137471,-0.0332158755588373,-0.0160318551303266,-0.0102661054152432,-0.00259226199818282,-0.0439854025655911,-0.0424987666488135,47.0 244 | -0.0600026317441039,0.0506801187398187,0.0541515220015222,-0.0194420933298793,-0.0497273098572509,-0.0489124436182275,0.0228686348215404,-0.0394933828740919,-0.0439854025655911,-0.0052198044153011,187.0 245 | -0.0273097856849279,-0.044641636506989,-0.0353068801305926,-0.0297707054110881,-0.0566070741482565,-0.058620045933703,0.0302319104297145,-0.0394933828740919,-0.0498684677352306,-0.129483011860342,125.0 246 | 0.0417084448844436,-0.044641636506989,-0.0320734439089499,-0.061904165207817,0.0796122588136553,0.0509819156926333,0.056003375058324,-0.00997248617336464,0.0450661683362615,-0.0590671943081523,78.0 247 | -0.0817978624502212,-0.044641636506989,-0.0816527993074713,-0.0400993174922969,0.00255889875439205,-0.0185370428246429,0.0707299262746723,-0.0394933828740919,-0.0109044358473771,-0.09220404962683,51.0 248 | -0.0418399394890061,-0.044641636506989,0.0476846495582368,0.0597439326260547,0.127770608850695,0.128016437292858,-0.0249926566315915,0.108111100629544,0.0638931206368394,0.0403433716478807,258.0 249 | -0.0127796318808497,-0.044641636506989,0.0606183944448076,0.0528581912385822,0.0479653430750293,0.0293746718291555,-0.0176293810234174,0.0343088588777263,0.0702112981933102,0.00720651632920303,215.0 250 | 0.0671362140415805,-0.044641636506989,0.056307146149284,0.0735154154009998,-0.0139525355440215,-0.039204841302752,-0.0323559322397657,-0.00259226199818282,0.0757375884575476,0.036201264733046,303.0 251 | -0.0527375548420648,0.0506801187398187,0.098341817030639,0.0872868981759448,0.0603489187988395,0.0487898764601065,-0.0581273968683752,0.108111100629544,0.0844952822124031,0.0403433716478807,243.0 252 | 0.00538306037424807,-0.044641636506989,0.0595405823709267,-0.0561660474078757,0.0245741444856101,0.0528608064633705,-0.0434008456520269,0.0509143632718854,-0.00421985970694603,-0.0300724459043093,91.0 253 | 0.0816663678456587,-0.044641636506989,0.0336730925977851,0.0081008722200108,0.0520932016496327,0.0566185880048449,-0.0176293810234174,0.0343088588777263,0.0348641930961596,0.0693381200517237,150.0 254 | 0.030810829531385,0.0506801187398187,0.056307146149284,0.076958286094736,0.0493412959332305,-0.0122740735888523,-0.0360375700438527,0.0712099797536354,0.120053382001538,0.0900486546258972,310.0 255 | 0.00175052192322852,-0.044641636506989,-0.0654856181992578,-0.00567061055493425,-0.00707277125301585,-0.0194764882100115,0.0412768238419757,-0.0394933828740919,-0.003303712578677,0.00720651632920303,153.0 256 | -0.0491050163910452,-0.044641636506989,0.160854917315731,-0.0469850588797694,-0.0290880169842339,-0.019789636671801,-0.0470824834561139,0.0343088588777263,0.028016506523264,0.0113486232440377,346.0 257 | -0.0273097856849279,0.0506801187398187,-0.0557853095343297,0.0253152256886921,-0.00707277125301585,-0.0235474182132754,0.052321737254237,-0.0394933828740919,-0.00514530798026311,-0.0507829804784829,63.0 258 | 0.0780338293946392,0.0506801187398187,-0.0245287593917836,-0.0423945646329306,0.00668675732899544,0.0528608064633705,-0.0691723102806364,0.0808042711813717,-0.0371283460104736,0.0569117993072195,89.0 259 | 0.0126481372762872,-0.044641636506989,-0.0363846922044735,0.0425295791573734,-0.0139525355440215,0.0129343775852051,-0.0268334755336351,0.00515697338575809,-0.0439854025655911,0.00720651632920303,50.0 260 | 0.0417084448844436,-0.044641636506989,-0.00836157828357004,-0.0573136709609782,0.00806271018719657,-0.0313761297580137,0.151725957964588,-0.076394503750001,-0.0802365402489018,-0.0176461251598052,39.0 261 | 0.0489735217864827,-0.044641636506989,-0.041773752573878,0.104501251644626,0.0355817673512192,-0.0257394574458021,0.177497422593197,-0.076394503750001,-0.0129079422541688,0.0154907301588724,103.0 262 | -0.0164121703318693,0.0506801187398187,0.127442743025423,0.0976155102571536,0.0163184273364034,0.0174750302811533,-0.0213110188275045,0.0343088588777263,0.0348641930961596,0.00306440941436832,308.0 263 | -0.0745327855481821,0.0506801187398187,-0.0773415510119477,-0.0469850588797694,-0.0469754041408486,-0.0326287236051719,0.00446044580110504,-0.0394933828740919,-0.072128454601956,-0.0176461251598052,116.0 264 | 0.0344433679824045,0.0506801187398187,0.0282840322283806,-0.0332135761048244,-0.0455994512826475,-0.00976888589453599,-0.050764121260201,-0.00259226199818282,-0.0594726974107223,-0.0217882320746399,145.0 265 | -0.034574862586967,0.0506801187398187,-0.0256065714656645,-0.0171468461892456,0.00118294589619092,-0.00287961973516629,0.0081420836051921,-0.015507654304751,0.0148227108412663,0.0403433716478807,74.0 266 | -0.0527375548420648,0.0506801187398187,-0.0622521819776151,0.0115437429137471,-0.00844872411121698,-0.0366996536084358,0.122272855531891,-0.076394503750001,-0.0868289932162924,0.00306440941436832,45.0 267 | 0.0598711371395414,-0.044641636506989,-0.000816893766403737,-0.0848566365108683,0.075484400239052,0.0794784257154807,0.00446044580110504,0.0343088588777263,0.0233748412798208,0.0279170509033766,115.0 268 | 0.063503675590561,0.0506801187398187,0.088641508365711,0.0700725447072635,0.0204462859110067,0.0375165318356834,-0.050764121260201,0.0712099797536354,0.0293004132685869,0.0734802269665584,264.0 269 | 0.00901559882526763,-0.044641636506989,-0.0320734439089499,-0.0263278347173518,0.0424615316422248,-0.0103951828181151,0.159089233572762,-0.076394503750001,-0.0119006848015081,-0.0383566597339788,87.0 270 | 0.00538306037424807,0.0506801187398187,0.0304396563761424,0.0838440274822086,-0.0373437341334407,-0.0473467013092799,0.0155053592133662,-0.0394933828740919,0.00864028293306308,0.0154907301588724,202.0 271 | 0.0380759064334241,0.0506801187398187,0.00888341489852436,0.0425295791573734,-0.0428475455662452,-0.0210422305189592,-0.0397192078479398,-0.00259226199818282,-0.0181182673078967,0.00720651632920303,127.0 272 | 0.0126481372762872,-0.044641636506989,0.00672779075076256,-0.0561660474078757,-0.0758704141630723,-0.0664487574784414,-0.0213110188275045,-0.0376483268302965,-0.0181182673078967,-0.09220404962683,182.0 273 | 0.0744012909436196,0.0506801187398187,-0.02021751109626,0.0459724498511097,0.0741084473808508,0.0328193049088404,-0.0360375700438527,0.0712099797536354,0.106354276741726,0.036201264733046,241.0 274 | 0.0162806757273067,-0.044641636506989,-0.0245287593917836,0.0356438377699009,-0.00707277125301585,-0.00319276819695581,-0.0139477432193303,-0.00259226199818282,0.0155668445407018,0.0154907301588724,66.0 275 | -0.00551455497881059,0.0506801187398187,-0.0115950145052127,0.0115437429137471,-0.0222082526932283,-0.0154055582067476,-0.0213110188275045,-0.00259226199818282,0.0110081010458725,0.0693381200517237,94.0 276 | 0.0126481372762872,-0.044641636506989,0.0261284080806188,0.063186803319791,0.125018703134293,0.0916912157252725,0.0633666506664982,-0.00259226199818282,0.057572856202426,-0.0217882320746399,283.0 277 | -0.034574862586967,-0.044641636506989,-0.0590187457559724,0.00121513083253827,-0.0538551684318543,-0.078035250564654,0.0670482884705852,-0.076394503750001,-0.02139368094036,0.0154907301588724,64.0 278 | 0.0671362140415805,0.0506801187398187,-0.0363846922044735,-0.0848566365108683,-0.00707277125301585,0.01966706951368,-0.0544457590642881,0.0343088588777263,0.00114379737951254,0.0320591578182113,102.0 279 | 0.0380759064334241,0.0506801187398187,-0.0245287593917836,0.00465800152627453,-0.0263361112678317,-0.0263657543693812,0.0155053592133662,-0.0394933828740919,-0.0159982677581387,-0.0259303389894746,200.0 280 | 0.00901559882526763,0.0506801187398187,0.0185837235634525,0.0390867084636372,0.0176943801946045,0.0105857641217836,0.0191869970174533,-0.00259226199818282,0.0163049527999418,-0.0176461251598052,265.0 281 | -0.0926954778032799,0.0506801187398187,-0.0902752958985185,-0.0573136709609782,-0.0249601584096305,-0.0304366843726451,-0.00658446761115617,-0.00259226199818282,0.024052583226893,0.00306440941436832,94.0 282 | 0.0707687524926,-0.044641636506989,-0.00512814206192736,-0.00567061055493425,0.0878679759628621,0.102964560349696,0.0118237214092792,0.0343088588777263,-0.0089440189577978,0.0279170509033766,230.0 283 | -0.0164121703318693,-0.044641636506989,-0.052551873312687,-0.0332135761048244,-0.0442234984244464,-0.0363865051466462,0.0191869970174533,-0.0394933828740919,-0.0683297436244215,-0.0300724459043093,181.0 284 | 0.0417084448844436,0.0506801187398187,-0.0223731352440218,0.0287580963824284,-0.0662387441556644,-0.0451546620767532,-0.0618090346724622,-0.00259226199818282,0.00286377051894013,-0.0549250873933176,156.0 285 | 0.0126481372762872,-0.044641636506989,-0.02021751109626,-0.015999222636143,0.0121905687618,0.0212328118226277,-0.0765355858888105,0.108111100629544,0.0598807230654812,-0.0217882320746399,233.0 286 | -0.0382074010379866,-0.044641636506989,-0.0547074974604488,-0.0779708951233958,-0.0332158755588373,-0.0864902590329714,0.140681044552327,-0.076394503750001,-0.0191970476139445,-0.0052198044153011,60.0 287 | 0.0453409833354632,-0.044641636506989,-0.00620595413580824,-0.015999222636143,0.125018703134293,0.125198101136752,0.0191869970174533,0.0343088588777263,0.0324332257796019,-0.0052198044153011,219.0 288 | 0.0707687524926,0.0506801187398187,-0.0169840748746173,0.0218723549949558,0.0438374845004259,0.0563054395430553,0.0375951860378887,-0.00259226199818282,-0.0702093127286876,-0.0176461251598052,80.0 289 | -0.0745327855481821,0.0506801187398187,0.0552293340754031,-0.0400993174922969,0.0534691545078339,0.05317395492516,-0.0434008456520269,0.0712099797536354,0.061237907519701,-0.0342145528191441,68.0 290 | 0.0598711371395414,0.0506801187398187,0.0767855755530211,0.0253152256886921,0.00118294589619092,0.0168487333575743,-0.0544457590642881,0.0343088588777263,0.0299356483965325,0.0444854785627154,332.0 291 | 0.0744012909436196,-0.044641636506989,0.0185837235634525,0.063186803319791,0.0617248716570406,0.0428400556861055,0.0081420836051921,-0.00259226199818282,0.0580391276638951,-0.0590671943081523,248.0 292 | 0.00901559882526763,-0.044641636506989,-0.0223731352440218,-0.0320659525517218,-0.0497273098572509,-0.0686407967109681,0.0780932018828464,-0.0708593356186146,-0.0629129499162512,-0.0383566597339788,84.0 293 | -0.0709002470971626,-0.044641636506989,0.0929527566612346,0.0126913664668496,0.0204462859110067,0.0425269072243159,0.000778807997017968,0.000359827671889909,-0.0545441527110952,-0.00107769750046639,200.0 294 | 0.0235457526293458,0.0506801187398187,-0.030995631835069,-0.00567061055493425,-0.0167044412604238,0.0177881787429428,-0.0323559322397657,-0.00259226199818282,-0.0740888714915354,-0.0342145528191441,55.0 295 | -0.0527375548420648,0.0506801187398187,0.0390621529671896,-0.0400993174922969,-0.00569681839481472,-0.0129003705124313,0.0118237214092792,-0.0394933828740919,0.0163049527999418,0.00306440941436832,85.0 296 | 0.0671362140415805,-0.044641636506989,-0.0611743699037342,-0.0400993174922969,-0.0263361112678317,-0.024486863598644,0.0339135482338016,-0.0394933828740919,-0.0561575730950062,-0.0590671943081523,89.0 297 | 0.00175052192322852,-0.044641636506989,-0.00836157828357004,-0.0641994123484507,-0.0387196869916418,-0.024486863598644,0.00446044580110504,-0.0394933828740919,-0.0646830224644503,-0.0549250873933176,31.0 298 | 0.0235457526293458,0.0506801187398187,-0.0374625042783544,-0.0469850588797694,-0.0910058956032848,-0.0755300628703378,-0.0323559322397657,-0.0394933828740919,-0.0307512098645563,-0.0135040182449705,129.0 299 | 0.0380759064334241,0.0506801187398187,-0.0137506386529745,-0.015999222636143,-0.0359677812752396,-0.0219816759043277,-0.0139477432193303,-0.00259226199818282,-0.0259524244351894,-0.00107769750046639,83.0 300 | 0.0162806757273067,-0.044641636506989,0.0735521393313785,-0.0412469410453994,-0.00432086553661359,-0.0135266674360104,-0.0139477432193303,-0.00111621716314646,0.0428956878925287,0.0444854785627154,275.0 301 | -0.00188201652779104,0.0506801187398187,-0.0245287593917836,0.0528581912385822,0.0273260502020124,0.0300009687527346,0.0302319104297145,-0.00259226199818282,-0.02139368094036,0.036201264733046,65.0 302 | 0.0126481372762872,-0.044641636506989,0.0336730925977851,0.0333485905259811,0.0300779559184146,0.0271826325966288,-0.0029028298070691,0.00884708547334898,0.0311929907028023,0.0279170509033766,198.0 303 | 0.0744012909436196,-0.044641636506989,0.034750904671666,0.0941726395634173,0.0575970130824372,0.0202933664372591,0.0228686348215404,-0.00259226199818282,0.0738021469200488,-0.0217882320746399,236.0 304 | 0.0417084448844436,0.0506801187398187,-0.0385403163522353,0.0528581912385822,0.0768603530972531,0.116429944206646,-0.0397192078479398,0.0712099797536354,-0.0225121719296605,-0.0135040182449705,253.0 305 | -0.00914709342983014,0.0506801187398187,-0.0396181284261162,-0.0400993174922969,-0.00844872411121698,0.0162224364339952,-0.0654906724765493,0.0712099797536354,0.0177634778671173,-0.0673514081378217,124.0 306 | 0.00901559882526763,0.0506801187398187,-0.00189470584028465,0.0218723549949558,-0.0387196869916418,-0.0248000120604336,-0.00658446761115617,-0.0394933828740919,-0.0398095943643375,-0.0135040182449705,44.0 307 | 0.0671362140415805,0.0506801187398187,-0.030995631835069,0.00465800152627453,0.0245741444856101,0.0356376410649462,-0.0286742944356786,0.0343088588777263,0.0233748412798208,0.0817644407962278,172.0 308 | 0.00175052192322852,-0.044641636506989,-0.0460850008694016,-0.0332135761048244,-0.07311850844667,-0.0814798836443389,0.0449584616460628,-0.0693832907835783,-0.0611765950943345,-0.0797777288823259,114.0 309 | -0.00914709342983014,0.0506801187398187,0.00133873038135806,-0.00222773986119799,0.0796122588136553,0.0700839718617947,0.0339135482338016,-0.00259226199818282,0.0267142576335128,0.0817644407962278,142.0 310 | -0.00551455497881059,-0.044641636506989,0.0649296427403312,0.0356438377699009,-0.00156895982021134,0.0149698425868371,-0.0139477432193303,0.000728838880648992,-0.0181182673078967,0.0320591578182113,109.0 311 | 0.096196521649737,-0.044641636506989,0.0401399650410705,-0.0573136709609782,0.0452134373586271,0.0606895180081088,-0.0213110188275045,0.0361539149215217,0.0125531528133893,0.0237749439885419,180.0 312 | -0.0745327855481821,-0.044641636506989,-0.0234509473179027,-0.00567061055493425,-0.0208322998350272,-0.0141529643595894,0.0155053592133662,-0.0394933828740919,-0.0384591123013538,-0.0300724459043093,144.0 313 | 0.0598711371395414,0.0506801187398187,0.0530737099276413,0.0528581912385822,0.0328298616348169,0.01966706951368,-0.0102661054152432,0.0343088588777263,0.0552050380896167,-0.00107769750046639,163.0 314 | -0.0236772472339084,-0.044641636506989,0.0401399650410705,-0.0125563519424068,-0.00982467696941811,-0.00100072896442909,-0.0029028298070691,-0.00259226199818282,-0.0119006848015081,-0.0383566597339788,147.0 315 | 0.00901559882526763,-0.044641636506989,-0.02021751109626,-0.0538708002672419,0.0314539087766158,0.0206065148990486,0.056003375058324,-0.0394933828740919,-0.0109044358473771,-0.00107769750046639,97.0 316 | 0.0162806757273067,0.0506801187398187,0.0142724752679289,0.00121513083253827,0.00118294589619092,-0.0213553789807487,-0.0323559322397657,0.0343088588777263,0.0749683360277342,0.0403433716478807,220.0 317 | 0.0199132141783263,-0.044641636506989,-0.0342290680567117,0.055153438482502,0.0672286830898452,0.0741549018650587,-0.00658446761115617,0.0328328140426899,0.0247253233428045,0.0693381200517237,190.0 318 | 0.0889314447476978,-0.044641636506989,0.00672779075076256,0.0253152256886921,0.0300779559184146,0.00870687335104641,0.0633666506664982,-0.0394933828740919,0.00943640914607987,0.0320591578182113,109.0 319 | 0.0199132141783263,-0.044641636506989,0.00457216660300077,0.0459724498511097,-0.0180803941186249,-0.0545491159304391,0.0633666506664982,-0.0394933828740919,0.0286607203138089,0.0610539062220542,191.0 320 | -0.0236772472339084,-0.044641636506989,0.0304396563761424,-0.00567061055493425,0.0823641645300576,0.092004364187062,-0.0176293810234174,0.0712099797536354,0.0330470723549341,0.00306440941436832,122.0 321 | 0.096196521649737,-0.044641636506989,0.0519958978537604,0.0792535333386559,0.054845107366035,0.0365770864503148,-0.0765355858888105,0.141322109417863,0.098646374304928,0.0610539062220542,230.0 322 | 0.0235457526293458,0.0506801187398187,0.0616962065186885,0.0620391798699746,0.0245741444856101,-0.0360733566848567,-0.0912621371051588,0.155344535350708,0.133395733837469,0.0817644407962278,242.0 323 | 0.0707687524926,0.0506801187398187,-0.00728376620968916,0.0494153205448459,0.0603489187988395,-0.00444536204411395,-0.0544457590642881,0.108111100629544,0.129019411600168,0.0569117993072195,248.0 324 | 0.030810829531385,-0.044641636506989,0.00564997867688165,0.0115437429137471,0.0782363059554542,0.077912683406533,-0.0434008456520269,0.108111100629544,0.0660482061630984,0.0196328370737072,249.0 325 | -0.00188201652779104,-0.044641636506989,0.0541515220015222,-0.0664946594890845,0.0727324945226497,0.0566185880048449,-0.0434008456520269,0.0848633944777217,0.0844952822124031,0.0486275854775501,192.0 326 | 0.0453409833354632,0.0506801187398187,-0.00836157828357004,-0.0332135761048244,-0.00707277125301585,0.00119131026809764,-0.0397192078479398,0.0343088588777263,0.0299356483965325,0.0279170509033766,131.0 327 | 0.0744012909436196,-0.044641636506989,0.114508998138853,0.0287580963824284,0.0245741444856101,0.0249905933641021,0.0191869970174533,-0.00259226199818282,-0.000609254186102297,-0.0052198044153011,237.0 328 | -0.0382074010379866,-0.044641636506989,0.067085266888093,-0.0607565416547144,-0.0290880169842339,-0.0232342697514859,-0.0102661054152432,-0.00259226199818282,-0.00149858682029207,0.0196328370737072,78.0 329 | -0.0127796318808497,0.0506801187398187,-0.0557853095343297,-0.00222773986119799,-0.0277120641260328,-0.029184090525487,0.0191869970174533,-0.0394933828740919,-0.0170521046047435,0.0444854785627154,135.0 330 | 0.00901559882526763,0.0506801187398187,0.0304396563761424,0.0425295791573734,-0.00294491267841247,0.0368902349121043,-0.0654906724765493,0.0712099797536354,-0.0236445575721341,0.0154907301588724,244.0 331 | 0.0816663678456587,0.0506801187398187,-0.0256065714656645,-0.0366564467985606,-0.0703666027302678,-0.0464072559239113,-0.0397192078479398,-0.00259226199818282,-0.0411803851880079,-0.0052198044153011,199.0 332 | 0.030810829531385,-0.044641636506989,0.104808689473925,0.076958286094736,-0.0112006298276192,-0.0113346282034837,-0.0581273968683752,0.0343088588777263,0.0571041874478439,0.036201264733046,270.0 333 | 0.0271782910803654,0.0506801187398187,-0.00620595413580824,0.0287580963824284,-0.0167044412604238,-0.00162702588800815,-0.0581273968683752,0.0343088588777263,0.0293004132685869,0.0320591578182113,164.0 334 | -0.0600026317441039,0.0506801187398187,-0.0471628129432825,-0.0228849640236156,-0.071742555588469,-0.0576806005483345,-0.00658446761115617,-0.0394933828740919,-0.0629129499162512,-0.0549250873933176,72.0 335 | 0.00538306037424807,-0.044641636506989,-0.0482406250171634,-0.0125563519424068,0.00118294589619092,-0.00663740127664067,0.0633666506664982,-0.0394933828740919,-0.0514005352605825,-0.0590671943081523,96.0 336 | -0.0200447087828888,-0.044641636506989,0.0854080721440683,-0.0366564467985606,0.0919958345374655,0.0894991764927457,-0.0618090346724622,0.145012221505454,0.0809479135112756,0.0527696923923848,306.0 337 | 0.0199132141783263,0.0506801187398187,-0.0126728265790937,0.0700725447072635,-0.0112006298276192,0.00714113104209875,-0.0397192078479398,0.0343088588777263,0.00538436996854573,0.00306440941436832,91.0 338 | -0.0636351701951234,-0.044641636506989,-0.0331512559828308,-0.0332135761048244,0.00118294589619092,0.0240511479787335,-0.0249926566315915,-0.00259226199818282,-0.0225121719296605,-0.0590671943081523,214.0 339 | 0.0271782910803654,-0.044641636506989,-0.00728376620968916,-0.0504279295735057,0.075484400239052,0.0566185880048449,0.0339135482338016,-0.00259226199818282,0.0434431722527813,0.0154907301588724,95.0 340 | -0.0164121703318693,-0.044641636506989,-0.0137506386529745,0.132044217194516,-0.00982467696941811,-0.00381906512053488,0.0191869970174533,-0.0394933828740919,-0.0358167281015492,-0.0300724459043093,216.0 341 | 0.030810829531385,0.0506801187398187,0.0595405823709267,0.0563010619323185,-0.0222082526932283,0.00119131026809764,-0.0323559322397657,-0.00259226199818282,-0.0247911874324607,-0.0176461251598052,263.0 342 | 0.0562385986885218,0.0506801187398187,0.0218171597850952,0.0563010619323185,-0.00707277125301585,0.0181013272047324,-0.0323559322397657,-0.00259226199818282,-0.0236445575721341,0.0237749439885419,178.0 343 | -0.0200447087828888,-0.044641636506989,0.0185837235634525,0.090729768869681,0.00393485161259318,0.00870687335104641,0.0375951860378887,-0.0394933828740919,-0.0578000656756125,0.00720651632920303,113.0 344 | -0.107225631607358,-0.044641636506989,-0.0115950145052127,-0.0400993174922969,0.0493412959332305,0.0644472995495832,-0.0139477432193303,0.0343088588777263,0.00702686254915195,-0.0300724459043093,200.0 345 | 0.0816663678456587,0.0506801187398187,-0.00297251791416553,-0.0332135761048244,0.0424615316422248,0.057871181852003,-0.0102661054152432,0.0343088588777263,-0.000609254186102297,-0.00107769750046639,139.0 346 | 0.00538306037424807,0.0506801187398187,0.0175059114895716,0.0322009670761646,0.127770608850695,0.127390140369279,-0.0213110188275045,0.0712099797536354,0.062575181458056,0.0154907301588724,139.0 347 | 0.0380759064334241,0.0506801187398187,-0.0299178197611881,-0.0745280244296595,-0.0125765826858204,-0.0125872220506418,0.00446044580110504,-0.00259226199818282,0.00371173823343597,-0.0300724459043093,88.0 348 | 0.030810829531385,-0.044641636506989,-0.02021751109626,-0.00567061055493425,-0.00432086553661359,-0.0294972389872765,0.0780932018828464,-0.0394933828740919,-0.0109044358473771,-0.00107769750046639,148.0 349 | 0.00175052192322852,0.0506801187398187,-0.0579409336820915,-0.0435421881860331,-0.0965097070360893,-0.0470335528474903,-0.098625412713333,0.0343088588777263,-0.0611765950943345,-0.0714935150526564,88.0 350 | -0.0273097856849279,0.0506801187398187,0.0606183944448076,0.107944122338362,0.0121905687618,-0.0175975974392743,-0.0029028298070691,-0.00259226199818282,0.0702112981933102,0.135611830689079,243.0 351 | -0.0854304009012408,0.0506801187398187,-0.0406959404999971,-0.0332135761048244,-0.0813742255958769,-0.0695802420963367,-0.00658446761115617,-0.0394933828740919,-0.0578000656756125,-0.0424987666488135,71.0 352 | 0.0126481372762872,0.0506801187398187,-0.0719524906425432,-0.0469850588797694,-0.051103262715452,-0.0971373067338155,0.118591217727804,-0.076394503750001,-0.0202887477516296,-0.0383566597339788,77.0 353 | -0.0527375548420648,-0.044641636506989,-0.0557853095343297,-0.0366564467985606,0.0892439288210632,-0.00319276819695581,0.0081420836051921,0.0343088588777263,0.132372649338676,0.00306440941436832,109.0 354 | -0.0236772472339084,0.0506801187398187,0.045529025410475,0.0218723549949558,0.10988322169408,0.0888728795691667,0.000778807997017968,0.0343088588777263,0.0741925366900307,0.0610539062220542,272.0 355 | -0.0745327855481821,0.0506801187398187,-0.00943939035745095,0.0149866136074833,-0.0373437341334407,-0.0216685274425382,-0.0139477432193303,-0.00259226199818282,-0.0332487872476258,0.0113486232440377,60.0 356 | -0.00551455497881059,0.0506801187398187,-0.0331512559828308,-0.015999222636143,0.00806271018719657,0.0162224364339952,0.0155053592133662,-0.00259226199818282,-0.0283202425479987,-0.0756356219674911,54.0 357 | -0.0600026317441039,0.0506801187398187,0.0498402737059986,0.0184294843012196,-0.0167044412604238,-0.0301235359108556,-0.0176293810234174,-0.00259226199818282,0.049768659920749,-0.0590671943081523,221.0 358 | -0.0200447087828888,-0.044641636506989,-0.084886235529114,-0.0263278347173518,-0.0359677812752396,-0.0341944659141195,0.0412768238419757,-0.0516707527631419,-0.0823814832581028,-0.0466408735636482,90.0 359 | 0.0380759064334241,0.0506801187398187,0.00564997867688165,0.0322009670761646,0.00668675732899544,0.0174750302811533,-0.0249926566315915,0.0343088588777263,0.0148227108412663,0.0610539062220542,311.0 360 | 0.0162806757273067,-0.044641636506989,0.0207393477112143,0.0218723549949558,-0.0139525355440215,-0.0132135189742209,-0.00658446761115617,-0.00259226199818282,0.0133159679089277,0.0403433716478807,281.0 361 | 0.0417084448844436,-0.044641636506989,-0.00728376620968916,0.0287580963824284,-0.0428475455662452,-0.0482861466946485,0.052321737254237,-0.076394503750001,-0.072128454601956,0.0237749439885419,182.0 362 | 0.0199132141783263,0.0506801187398187,0.104808689473925,0.0700725447072635,-0.0359677812752396,-0.0266789028311707,-0.0249926566315915,-0.00259226199818282,0.00371173823343597,0.0403433716478807,321.0 363 | -0.0491050163910452,0.0506801187398187,-0.0245287593917836,6.75072794357462e-05,-0.0469754041408486,-0.0282446451401184,-0.0654906724765493,0.0284046795375808,0.0191990330785671,0.0113486232440377,58.0 364 | 0.00175052192322852,0.0506801187398187,-0.00620595413580824,-0.0194420933298793,-0.00982467696941811,0.00494909180957202,-0.0397192078479398,0.0343088588777263,0.0148227108412663,0.0983328684555666,262.0 365 | 0.0344433679824045,-0.044641636506989,-0.0385403163522353,-0.0125563519424068,0.0094386630453977,0.00526224027136155,-0.00658446761115617,-0.00259226199818282,0.0311929907028023,0.0983328684555666,206.0 366 | -0.0454724779400257,0.0506801187398187,0.137143051690352,-0.015999222636143,0.0410855787840237,0.0318798595234718,-0.0434008456520269,0.0712099797536354,0.0710215779459822,0.0486275854775501,233.0 367 | -0.00914709342983014,0.0506801187398187,0.17055522598066,0.0149866136074833,0.0300779559184146,0.033758750294209,-0.0213110188275045,0.0343088588777263,0.0336568129023847,0.0320591578182113,242.0 368 | -0.0164121703318693,0.0506801187398187,0.00241654245523897,0.0149866136074833,0.0218222387692079,-0.0100820343563255,-0.0249926566315915,0.0343088588777263,0.085533121187439,0.0817644407962278,123.0 369 | -0.00914709342983014,-0.044641636506989,0.0379843408933087,-0.0400993174922969,-0.0249601584096305,-0.00381906512053488,-0.0434008456520269,0.0158582984397717,-0.00514530798026311,0.0279170509033766,167.0 370 | 0.0199132141783263,-0.044641636506989,-0.0579409336820915,-0.0573136709609782,-0.00156895982021134,-0.0125872220506418,0.0744115640787594,-0.0394933828740919,-0.0611765950943345,-0.0756356219674911,63.0 371 | 0.0526060602375023,0.0506801187398187,-0.00943939035745095,0.0494153205448459,0.0507172487914316,-0.019163339748222,-0.0139477432193303,0.0343088588777263,0.119343994203787,-0.0176461251598052,197.0 372 | -0.0273097856849279,0.0506801187398187,-0.0234509473179027,-0.015999222636143,0.0135665216200011,0.0127778033543103,0.0265502726256275,-0.00259226199818282,-0.0109044358473771,-0.0217882320746399,71.0 373 | -0.0745327855481821,-0.044641636506989,-0.0105172024313319,-0.00567061055493425,-0.0662387441556644,-0.0570543036247554,-0.0029028298070691,-0.0394933828740919,-0.0425721049227942,-0.00107769750046639,168.0 374 | -0.107225631607358,-0.044641636506989,-0.0342290680567117,-0.067642283042187,-0.0634868384392622,-0.0705196874817053,0.0081420836051921,-0.0394933828740919,-0.000609254186102297,-0.0797777288823259,140.0 375 | 0.0453409833354632,0.0506801187398187,-0.00297251791416553,0.107944122338362,0.0355817673512192,0.0224854056697859,0.0265502726256275,-0.00259226199818282,0.028016506523264,0.0196328370737072,217.0 376 | -0.00188201652779104,-0.044641636506989,0.068163078961974,-0.00567061055493425,0.119514891701488,0.130208476525385,-0.0249926566315915,0.0867084505215172,0.0461323310394148,-0.00107769750046639,121.0 377 | 0.0199132141783263,0.0506801187398187,0.00996122697240527,0.0184294843012196,0.0149424744782022,0.0447189464568426,-0.0618090346724622,0.0712099797536354,0.00943640914607987,-0.063209301222987,235.0 378 | 0.0162806757273067,0.0506801187398187,0.00241654245523897,-0.00567061055493425,-0.00569681839481472,0.0108989125835731,-0.050764121260201,0.0343088588777263,0.0226920225667445,-0.0383566597339788,245.0 379 | -0.00188201652779104,-0.044641636506989,-0.0385403163522353,0.0218723549949558,-0.108893282759899,-0.115613065979398,0.0228686348215404,-0.076394503750001,-0.0468794828442166,0.0237749439885419,40.0 380 | 0.0162806757273067,-0.044641636506989,0.0261284080806188,0.0585963091762383,-0.0607349327228599,-0.0442152166913845,-0.0139477432193303,-0.0339582147427055,-0.0514005352605825,-0.0259303389894746,52.0 381 | -0.0709002470971626,0.0506801187398187,-0.0891974838246376,-0.0745280244296595,-0.0428475455662452,-0.0257394574458021,-0.0323559322397657,-0.00259226199818282,-0.0129079422541688,-0.0549250873933176,104.0 382 | 0.0489735217864827,-0.044641636506989,0.0606183944448076,-0.0228849640236156,-0.0235842055514294,-0.072711726714232,-0.0434008456520269,-0.00259226199818282,0.104137611358979,0.036201264733046,132.0 383 | 0.00538306037424807,0.0506801187398187,-0.0288400076873072,-0.00911348124867051,-0.0318399227006362,-0.0288709420636975,0.0081420836051921,-0.0394933828740919,-0.0181182673078967,0.00720651632920303,88.0 384 | 0.0344433679824045,0.0506801187398187,-0.0299178197611881,0.00465800152627453,0.0933717873956666,0.0869939887984295,0.0339135482338016,-0.00259226199818282,0.024052583226893,-0.0383566597339788,69.0 385 | 0.0235457526293458,0.0506801187398187,-0.019139699022379,0.0494153205448459,-0.0634868384392622,-0.0611252336280193,0.00446044580110504,-0.0394933828740919,-0.0259524244351894,-0.0135040182449705,219.0 386 | 0.0199132141783263,-0.044641636506989,-0.0406959404999971,-0.015999222636143,-0.00844872411121698,-0.0175975974392743,0.052321737254237,-0.0394933828740919,-0.0307512098645563,0.00306440941436832,72.0 387 | -0.0454724779400257,-0.044641636506989,0.0153502873418098,-0.0745280244296595,-0.0497273098572509,-0.0172844489774848,-0.0286742944356786,-0.00259226199818282,-0.104364820832166,-0.0756356219674911,201.0 388 | 0.0526060602375023,0.0506801187398187,-0.0245287593917836,0.0563010619323185,-0.00707277125301585,-0.005071658967693,-0.0213110188275045,-0.00259226199818282,0.0267142576335128,-0.0383566597339788,110.0 389 | -0.00551455497881059,0.0506801187398187,0.00133873038135806,-0.0848566365108683,-0.0112006298276192,-0.0166581520539057,0.0486400994501499,-0.0394933828740919,-0.0411803851880079,-0.0880619427119953,51.0 390 | 0.00901559882526763,0.0506801187398187,0.0692408910358548,0.0597439326260547,0.0176943801946045,-0.0232342697514859,-0.0470824834561139,0.0343088588777263,0.103292264911524,0.0734802269665584,277.0 391 | -0.0236772472339084,-0.044641636506989,-0.0697968664947814,-0.0641994123484507,-0.0593589798646588,-0.0504781859271752,0.0191869970174533,-0.0394933828740919,-0.0891368600793477,-0.0507829804784829,63.0 392 | -0.0418399394890061,0.0506801187398187,-0.0299178197611881,-0.00222773986119799,0.0218222387692079,0.0365770864503148,0.0118237214092792,-0.00259226199818282,-0.0411803851880079,0.065196013136889,118.0 393 | -0.0745327855481821,-0.044641636506989,-0.0460850008694016,-0.0435421881860331,-0.0290880169842339,-0.0232342697514859,0.0155053592133662,-0.0394933828740919,-0.0398095943643375,-0.0217882320746399,69.0 394 | 0.0344433679824045,-0.044641636506989,0.0185837235634525,0.0563010619323185,0.0121905687618,-0.0545491159304391,-0.0691723102806364,0.0712099797536354,0.130080609521753,0.00720651632920303,273.0 395 | -0.0600026317441039,-0.044641636506989,0.00133873038135806,-0.0297707054110881,-0.00707277125301585,-0.0216685274425382,0.0118237214092792,-0.00259226199818282,0.0318152175007986,-0.0549250873933176,258.0 396 | -0.0854304009012408,0.0506801187398187,-0.030995631835069,-0.0228849640236156,-0.0634868384392622,-0.0542359674686496,0.0191869970174533,-0.0394933828740919,-0.096433222891784,-0.0342145528191441,43.0 397 | 0.0526060602375023,-0.044641636506989,-0.00405032998804645,-0.0309183289641906,-0.0469754041408486,-0.0583068974719135,-0.0139477432193303,-0.0258399681500055,0.0360557900898319,0.0237749439885419,198.0 398 | 0.0126481372762872,-0.044641636506989,0.0153502873418098,-0.0332135761048244,0.0410855787840237,0.0321930079852613,-0.0029028298070691,-0.00259226199818282,0.0450661683362615,-0.0673514081378217,242.0 399 | 0.0598711371395414,0.0506801187398187,0.0228949718589761,0.0494153205448459,0.0163184273364034,0.0118383579689417,-0.0139477432193303,-0.00259226199818282,0.0395398780720242,0.0196328370737072,232.0 400 | -0.0236772472339084,-0.044641636506989,0.045529025410475,0.090729768869681,-0.0180803941186249,-0.0354470597612776,0.0707299262746723,-0.0394933828740919,-0.0345237153303495,-0.0093619113301358,175.0 401 | 0.0162806757273067,-0.044641636506989,-0.0450071887955207,-0.0573136709609782,-0.0345918284170385,-0.05392281900686,0.0744115640787594,-0.076394503750001,-0.0425721049227942,0.0403433716478807,93.0 402 | 0.110726675453815,0.0506801187398187,-0.0331512559828308,-0.0228849640236156,-0.00432086553661359,0.0202933664372591,-0.0618090346724622,0.0712099797536354,0.0155668445407018,0.0444854785627154,168.0 403 | -0.0200447087828888,-0.044641636506989,0.0972640049567582,-0.00567061055493425,-0.00569681839481472,-0.0238605666750649,-0.0213110188275045,-0.00259226199818282,0.0616858488238662,0.0403433716478807,275.0 404 | -0.0164121703318693,-0.044641636506989,0.0541515220015222,0.0700725447072635,-0.0332158755588373,-0.0279314966783289,0.0081420836051921,-0.0394933828740919,-0.0271286455543265,-0.0093619113301358,293.0 405 | 0.0489735217864827,0.0506801187398187,0.1231314947299,0.0838440274822086,-0.104765424185296,-0.10089508827529,-0.0691723102806364,-0.00259226199818282,0.0366457977933988,-0.0300724459043093,281.0 406 | -0.0563700932930843,-0.044641636506989,-0.0805749872335904,-0.0848566365108683,-0.0373437341334407,-0.0370128020702253,0.0339135482338016,-0.0394933828740919,-0.0561575730950062,-0.137767225690012,72.0 407 | 0.0271782910803654,-0.044641636506989,0.0929527566612346,-0.0527231767141394,0.00806271018719657,0.0397085710682101,-0.0286742944356786,0.021024455362399,-0.0483617248028919,0.0196328370737072,140.0 408 | 0.063503675590561,-0.044641636506989,-0.0503962491649252,0.107944122338362,0.0314539087766158,0.0193539210518905,-0.0176293810234174,0.0236075338237126,0.0580391276638951,0.0403433716478807,189.0 409 | -0.0527375548420648,0.0506801187398187,-0.0115950145052127,0.0563010619323185,0.0562210602242361,0.0729023080179005,-0.0397192078479398,0.0712099797536354,0.0305664873984148,-0.0052198044153011,181.0 410 | -0.00914709342983014,0.0506801187398187,-0.0277621956134263,0.0081008722200108,0.0479653430750293,0.0372033833738938,-0.0286742944356786,0.0343088588777263,0.0660482061630984,-0.0424987666488135,209.0 411 | 0.00538306037424807,-0.044641636506989,0.0584627702970458,-0.0435421881860331,-0.07311850844667,-0.0723985782524425,0.0191869970174533,-0.076394503750001,-0.0514005352605825,-0.0259303389894746,136.0 412 | 0.0744012909436196,-0.044641636506989,0.0854080721440683,0.063186803319791,0.0149424744782022,0.0130909518160999,0.0155053592133662,-0.00259226199818282,0.0062093156165054,0.0859065477110625,261.0 413 | -0.0527375548420648,-0.044641636506989,-0.000816893766403737,-0.0263278347173518,0.0108146159035988,0.00714113104209875,0.0486400994501499,-0.0394933828740919,-0.0358167281015492,0.0196328370737072,113.0 414 | 0.0816663678456587,0.0506801187398187,0.00672779075076256,-0.00452298700183173,0.10988322169408,0.117056241130225,-0.0323559322397657,0.0918746074441444,0.0547240033481791,0.00720651632920303,131.0 415 | -0.00551455497881059,-0.044641636506989,0.00888341489852436,-0.0504279295735057,0.0259500973438113,0.0472241341511589,-0.0434008456520269,0.0712099797536354,0.0148227108412663,0.00306440941436832,174.0 416 | -0.0273097856849279,-0.044641636506989,0.0800190117746638,0.09876313370697,-0.00294491267841247,0.0181013272047324,-0.0176293810234174,0.00331191734196264,-0.0295276227417736,0.036201264733046,257.0 417 | -0.0527375548420648,-0.044641636506989,0.0713965151836166,-0.0745280244296595,-0.0153284884022226,-0.00131387742621863,0.00446044580110504,-0.0214118336448964,-0.0468794828442166,0.00306440941436832,55.0 418 | 0.00901559882526763,-0.044641636506989,-0.0245287593917836,-0.0263278347173518,0.0988755988284711,0.0941964034195887,0.0707299262746723,-0.00259226199818282,-0.02139368094036,0.00720651632920303,84.0 419 | -0.0200447087828888,-0.044641636506989,-0.0547074974604488,-0.0538708002672419,-0.0662387441556644,-0.0573674520865449,0.0118237214092792,-0.0394933828740919,-0.0740888714915354,-0.0052198044153011,42.0 420 | 0.0235457526293458,-0.044641636506989,-0.0363846922044735,6.75072794357462e-05,0.00118294589619092,0.0346981956795776,-0.0434008456520269,0.0343088588777263,-0.0332487872476258,0.0610539062220542,146.0 421 | 0.0380759064334241,0.0506801187398187,0.0164280994156907,0.0218723549949558,0.0397096259258226,0.0450320949186321,-0.0434008456520269,0.0712099797536354,0.049768659920749,0.0154907301588724,212.0 422 | -0.0781653239992017,0.0506801187398187,0.077863387626902,0.0528581912385822,0.0782363059554542,0.0644472995495832,0.0265502726256275,-0.00259226199818282,0.0406722637144977,-0.0093619113301358,233.0 423 | 0.00901559882526763,0.0506801187398187,-0.0396181284261162,0.0287580963824284,0.0383336730676214,0.0735286049414796,-0.0728539480847234,0.108111100629544,0.0155668445407018,-0.0466408735636482,91.0 424 | 0.00175052192322852,0.0506801187398187,0.0110390390462862,-0.0194420933298793,-0.0167044412604238,-0.00381906512053488,-0.0470824834561139,0.0343088588777263,0.024052583226893,0.0237749439885419,111.0 425 | -0.0781653239992017,-0.044641636506989,-0.0406959404999971,-0.081413765817132,-0.100637565610693,-0.112794729823292,0.0228686348215404,-0.076394503750001,-0.0202887477516296,-0.0507829804784829,152.0 426 | 0.030810829531385,0.0506801187398187,-0.0342290680567117,0.0436772026071898,0.0575970130824372,0.0688313780146366,-0.0323559322397657,0.057556565029549,0.0354619386607697,0.0859065477110625,120.0 427 | -0.034574862586967,0.0506801187398187,0.00564997867688165,-0.00567061055493425,-0.07311850844667,-0.062690975936967,-0.00658446761115617,-0.0394933828740919,-0.045420957777041,0.0320591578182113,67.0 428 | 0.0489735217864827,0.0506801187398187,0.088641508365711,0.0872868981759448,0.0355817673512192,0.0215459602844172,-0.0249926566315915,0.0343088588777263,0.0660482061630984,0.131469723774244,310.0 429 | -0.0418399394890061,-0.044641636506989,-0.0331512559828308,-0.0228849640236156,0.0465893902168282,0.0415874618389473,0.056003375058324,-0.0247329345237283,-0.0259524244351894,-0.0383566597339788,94.0 430 | -0.00914709342983014,-0.044641636506989,-0.0568631216082106,-0.0504279295735057,0.0218222387692079,0.0453452433804217,-0.0286742944356786,0.0343088588777263,-0.00991895736315477,-0.0176461251598052,183.0 431 | 0.0707687524926,0.0506801187398187,-0.030995631835069,0.0218723549949558,-0.0373437341334407,-0.0470335528474903,0.0339135482338016,-0.0394933828740919,-0.0149564750249113,-0.00107769750046639,66.0 432 | 0.00901559882526763,-0.044641636506989,0.0552293340754031,-0.00567061055493425,0.0575970130824372,0.0447189464568426,-0.0029028298070691,0.0232385226149535,0.0556835477026737,0.106617082285236,173.0 433 | -0.0273097856849279,-0.044641636506989,-0.0600965578298533,-0.0297707054110881,0.0465893902168282,0.0199802179754696,0.122272855531891,-0.0394933828740919,-0.0514005352605825,-0.0093619113301358,72.0 434 | 0.0162806757273067,-0.044641636506989,0.00133873038135806,0.0081008722200108,0.00531080447079431,0.0108989125835731,0.0302319104297145,-0.0394933828740919,-0.045420957777041,0.0320591578182113,49.0 435 | -0.0127796318808497,-0.044641636506989,-0.0234509473179027,-0.0400993174922969,-0.0167044412604238,0.0046359433477825,-0.0176293810234174,-0.00259226199818282,-0.0384591123013538,-0.0383566597339788,64.0 436 | -0.0563700932930843,-0.044641636506989,-0.074108114790305,-0.0504279295735057,-0.0249601584096305,-0.0470335528474903,0.0928197530991947,-0.076394503750001,-0.0611765950943345,-0.0466408735636482,48.0 437 | 0.0417084448844436,0.0506801187398187,0.0196615356373334,0.0597439326260547,-0.00569681839481472,-0.00256647127337676,-0.0286742944356786,-0.00259226199818282,0.0311929907028023,0.00720651632920303,178.0 438 | -0.00551455497881059,0.0506801187398187,-0.0159062628007364,-0.067642283042187,0.0493412959332305,0.0791652772536912,-0.0286742944356786,0.0343088588777263,-0.0181182673078967,0.0444854785627154,104.0 439 | 0.0417084448844436,0.0506801187398187,-0.0159062628007364,0.0172818607481171,-0.0373437341334407,-0.0138398158977999,-0.0249926566315915,-0.0110795197996419,-0.0468794828442166,0.0154907301588724,132.0 440 | -0.0454724779400257,-0.044641636506989,0.0390621529671896,0.00121513083253827,0.0163184273364034,0.0152829910486266,-0.0286742944356786,0.0265596234937854,0.0445283740214053,-0.0259303389894746,220.0 441 | -0.0454724779400257,-0.044641636506989,-0.0730303027164241,-0.081413765817132,0.0837401173882587,0.0278089295202079,0.17381578478911,-0.0394933828740919,-0.00421985970694603,0.00306440941436832,57.0 442 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/code/tests/feature_engineering_test.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from tempfile import TemporaryDirectory 3 | import databricks_test 4 | from pandas.testing import assert_frame_equal 5 | 6 | 7 | def test_feature_engineering(mocker): 8 | mocker.patch("azureml.core.Run") 9 | 10 | with databricks_test.session() as dbrickstest, \ 11 | TemporaryDirectory() as out_dir: 12 | 13 | # Provide input and output location as widgets to notebook 14 | switcher = { 15 | "training": "code/tests/diabetes_missing_values.csv", 16 | "feature_engineered": out_dir, 17 | } 18 | dbrickstest.dbutils.widgets.get = lambda x: switcher.get(x, "") 19 | 20 | capture_files = {} 21 | 22 | def mock_cp(src, dst, capture_files=capture_files): 23 | prefix = "file:" 24 | assert src.startswith(prefix) 25 | capture_files[dst] = pd.read_csv(src) 26 | return True 27 | 28 | dbrickstest.dbutils.fs.cp.side_effect = mock_cp 29 | 30 | # Run notebook 31 | dbrickstest.run_notebook("./code/prepare", "feature_engineering") 32 | 33 | expected_name = "engineered.csv" 34 | expected_file = "%s/%s" % (out_dir, expected_name) 35 | assert expected_file in capture_files 36 | resultDF = capture_files[expected_file] 37 | 38 | # Compare produced and expected CSV files 39 | expectedDF = pd.read_csv( 40 | "code/tests/feature_engineering_expected.csv") 41 | assert_frame_equal( 42 | expectedDF, resultDF, check_dtype=False, check_categorical=False) 43 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/docs/images/azureml-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/DataOps/5715cc7abec6fd71461b2b119faa45202019dcec/Python/azureml-pipeline-databricks/docs/images/azureml-pipeline.png -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/docs/images/output-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/DataOps/5715cc7abec6fd71461b2b119faa45202019dcec/Python/azureml-pipeline-databricks/docs/images/output-dataset.png -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM conda/miniconda3 2 | 3 | ARG SPARK_VERSION=2.4.3 4 | ARG HADOOP_VERSION=2.7 5 | 6 | # Install Spark runtime 7 | WORKDIR / 8 | RUN apt-get update \ 9 | && apt-get install -y \ 10 | wget \ 11 | apt-transport-https \ 12 | openjdk-8-jdk \ 13 | && wget -q https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \ 14 | && tar -zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \ 15 | && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz 16 | 17 | ENV SPARK_HOME /spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} 18 | 19 | COPY ci_dependencies.yml /setup/ 20 | 21 | # activate Conda environment 22 | ENV PATH /usr/local/envs/dataops_ci/bin:$PATH 23 | 24 | RUN conda update -n base -c defaults conda && \ 25 | conda env create -f /setup/ci_dependencies.yml && \ 26 | /bin/bash -c "source activate dataops_ci" && \ 27 | az --version && \ 28 | chmod -R 777 /usr/local/envs/dataops_ci/lib 29 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/docker-image-pipeline.yml: -------------------------------------------------------------------------------- 1 | name: docker-image 2 | 3 | # Pipeline that builds and pushes the microsoft/mlopspython image. 4 | resources: 5 | - repo: self 6 | 7 | pool: 8 | vmImage: 'ubuntu-latest' 9 | 10 | trigger: 11 | branches: 12 | include: 13 | - master 14 | 15 | paths: 16 | include: 17 | - Python/azureml-pipeline-databricks/environment_setup/Dockerfile 18 | 19 | variables: 20 | - template: ../variables.yml 21 | 22 | steps: 23 | - task: Docker@2 24 | displayName: Build and Push 25 | inputs: 26 | command: buildAndPush 27 | containerRegistry: '$(REGISTRY_SVC_CONNECTION)' 28 | repository: '$(AGENT_IMAGE_NAME)' 29 | tags: | 30 | $(Build.BuildNumber) 31 | latest 32 | buildContext: Python/azureml-pipeline-databricks 33 | dockerFile: Python/azureml-pipeline-databricks/environment_setup/Dockerfile 34 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/install_requirements.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eux 4 | 5 | conda env create -f diabetes_regression/ci_dependencies.yml 6 | 7 | conda activate mlopspython_ci 8 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/terraform-init-template.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | environment: test 3 | TerraformVersion: 0.12.21 4 | TerraformDirectory: 5 | TerraformBackendServiceConnection: $(ARM_SVC_CONNECTION) 6 | TerraformEnvironmentServiceConnection: $(ARM_SVC_CONNECTION) 7 | TerraformBackendResourceGroup: $(TERRAFORM_BACKEND_RG) 8 | TerraformBackendStorageAccount: $(TERRAFORM_BACKEND_STORAGE) 9 | TerraformBackendStorageContainer: terraformstate 10 | 11 | steps: 12 | 13 | - task: ms-devlabs.custom-terraform-tasks.custom-terraform-installer-task.TerraformInstaller@0 14 | displayName: Install Terraform 15 | inputs: 16 | terraformVersion: ${{ parameters.TerraformVersion }} 17 | 18 | - task: AzureCLI@1 19 | displayName: Terraform credentials 20 | inputs: 21 | azureSubscription: ${{ parameters.TerraformEnvironmentServiceConnection }} 22 | scriptLocation: inlineScript 23 | inlineScript: | 24 | set -eu 25 | subscriptionId=$(az account show --query id -o tsv) 26 | echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$servicePrincipalId" 27 | echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET;issecret=true]$servicePrincipalKey" 28 | echo "##vso[task.setvariable variable=ARM_SUBSCRIPTION_ID]$subscriptionId" 29 | echo "##vso[task.setvariable variable=ARM_TENANT_ID]$tenantId" 30 | addSpnToEnvironment: true 31 | 32 | # Using bash instead of Terraform extension because of following issue: 33 | # - https://github.com/microsoft/azure-pipelines-extensions/issues/738 34 | - task: AzureCLI@1 35 | displayName: Terraform init 36 | inputs: 37 | azureSubscription: ${{ parameters.TerraformBackendServiceConnection }} 38 | scriptLocation: inlineScript 39 | inlineScript: | 40 | set -eux # fail on error 41 | subscriptionId=$(az account show --query id -o tsv) 42 | terraform init \ 43 | -backend-config=storage_account_name=${{ parameters.TerraformBackendStorageAccount }} \ 44 | -backend-config=container_name=${{ parameters.TerraformBackendStorageContainer }} \ 45 | -backend-config=key=${{ parameters.environment }}.tfstate \ 46 | -backend-config=resource_group_name=${{ parameters.TerraformBackendResourceGroup }} \ 47 | -backend-config=subscription_id=$subscriptionId \ 48 | -backend-config=tenant_id=$tenantId \ 49 | -backend-config=client_id=$servicePrincipalId \ 50 | -backend-config=client_secret="$servicePrincipalKey" 51 | workingDirectory: ${{ parameters.TerraformDirectory }} 52 | addSpnToEnvironment: true 53 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/terraform-outputs-template.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | TerraformDirectory: 3 | 4 | steps: 5 | - bash: | 6 | set -eu 7 | 8 | echo "Setting job variables from Terraform outputs:" 9 | terraform output -json | jq -r ' 10 | . as $in 11 | | keys[] 12 | | ["- " + .] 13 | | @tsv' 14 | 15 | terraform output -json | jq -r ' 16 | . as $in 17 | | keys[] 18 | | ($in[.].value | tostring) as $value 19 | | ($in[.].sensitive | tostring) as $sensitive 20 | | [ 21 | "- " + . + ": " + if $in[.].sensitive then "(sensitive)" else $value end, # output name to console 22 | "##vso[task.setvariable variable=" + . + ";isSecret=" + $sensitive + "]" + $value, # set as ADO task variable 23 | "##vso[task.setvariable variable=" + . + ";isOutput=true;isSecret=" + $sensitive + "]" + $value # also set as ADO job variable 24 | ] 25 | | .[]' 26 | 27 | name: Outputs 28 | displayName: Read Terraform outputs 29 | workingDirectory: ${{ parameters.TerraformDirectory }} 30 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/terraform-pipeline.yml: -------------------------------------------------------------------------------- 1 | name: terraform 2 | 3 | pr: none 4 | trigger: none 5 | 6 | variables: 7 | - group: terraform 8 | - template: ../variables.yml 9 | 10 | pool: 11 | vmImage: ubuntu-latest 12 | 13 | stages: 14 | 15 | - template: terraform-stages-template.yml 16 | parameters: 17 | environment: test 18 | TerraformEnvVariables: 19 | TF_VAR_appname: $(BASE_NAME) 20 | TF_VAR_location: $(LOCATION) 21 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/terraform-stages-template.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | environment: test 3 | TerraformArguments: '' 4 | TerraformEnvVariables: 5 | 6 | stages: 7 | - stage: Terraform 8 | pool: 9 | vmImage: ubuntu-latest 10 | jobs: 11 | 12 | - job: Terraform 13 | displayName: Terraform 14 | # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file. 15 | condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM'])) 16 | steps: 17 | 18 | - template: terraform-init-template.yml 19 | parameters: 20 | TerraformDirectory: Python/azureml-pipeline-databricks/environment_setup/terraform 21 | environment: ${{ parameters.environment }} 22 | 23 | # Using bash instead of Terraform extension because of following issues: 24 | # - https://github.com/microsoft/azure-pipelines-extensions/issues/748 25 | # - https://github.com/microsoft/azure-pipelines-extensions/issues/725 26 | # - https://github.com/microsoft/azure-pipelines-extensions/issues/747 27 | - bash: | 28 | set -eu 29 | export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET) 30 | terraform apply -input=false -auto-approve -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }} 31 | displayName: Terraform apply 32 | workingDirectory: Python/azureml-pipeline-databricks/environment_setup/terraform 33 | env: 34 | ${{ parameters.TerraformEnvVariables }} 35 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/terraform/.artifactignore: -------------------------------------------------------------------------------- 1 | *.tmp 2 | -------------------------------------------------------------------------------- /Python/azureml-pipeline-databricks/environment_setup/terraform/azureml/main.tf: -------------------------------------------------------------------------------- 1 | # Storage Account 2 | 3 | resource "azurerm_storage_account" "aml" { 4 | name = "st${var.appname}${var.environment}" 5 | resource_group_name = var.resource_group_name 6 | location = var.location 7 | account_tier = "Standard" 8 | account_replication_type = "LRS" 9 | } 10 | 11 | # Key Vault 12 | 13 | resource "azurerm_key_vault" "aml" { 14 | name = "kv-${var.appname}-${var.environment}" 15 | location = var.location 16 | resource_group_name = var.resource_group_name 17 | tenant_id = var.tenant_id 18 | 19 | sku_name = "standard" 20 | } 21 | 22 | # Application Insights 23 | 24 | resource "azurerm_application_insights" "aml" { 25 | name = "appinsights-${var.appname}-${var.environment}" 26 | resource_group_name = var.resource_group_name 27 | location = var.location 28 | application_type = "web" 29 | } 30 | 31 | # Container Registry 32 | 33 | resource "azurerm_container_registry" "aml" { 34 | name = "acr${var.appname}${var.environment}" 35 | resource_group_name = var.resource_group_name 36 | location = var.location 37 | sku = "Standard" 38 | admin_enabled = true 39 | } 40 | 41 | 42 | # Azure ML Workspace 43 | 44 | resource "azurerm_template_deployment" "aml" { 45 | name = "aml-${var.appname}-${var.environment}-deploy" 46 | resource_group_name = var.resource_group_name 47 | 48 | template_body = <42") 17 | 18 | # COMMAND ---------- 19 | 20 | dbutils.fs.help() 21 | 22 | # COMMAND ---------- 23 | 24 | assert len(sc.range(1, 7, 2).collect()) == 3 25 | 26 | # COMMAND ---------- 27 | 28 | assert sqlContext.sql("SELECT 2").collect()[0][0] == 2 29 | 30 | # COMMAND ---------- 31 | 32 | spark.range(3).write.mode("overwrite").saveAsTable("mytesttable") 33 | 34 | # COMMAND ---------- 35 | 36 | assert table("mytesttable").collect()[2][0] == 2 37 | 38 | # COMMAND ---------- 39 | 40 | assert sql("SELECT 7").collect()[0][0] == 7 41 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/functions_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | 3 | 4 | def test_functions(): 5 | with databricks_test.session() as dbrickstest: 6 | # Run notebook 7 | dbrickstest.run_notebook(".", "functions_notebook") 8 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/library_notebook.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 4 | if 'dbutils' not in locals(): 5 | import databricks_test 6 | databricks_test.inject_variables() 7 | 8 | # COMMAND ---------- 9 | 10 | dbutils.fs.help() 11 | 12 | res = dbutils.library.install("path") 13 | assert isinstance(res, bool) and res 14 | 15 | res = dbutils.library.installPyPI("pypiPackage") 16 | assert isinstance(res, bool) and res 17 | res = dbutils.library.installPyPI( 18 | "pypiPackage", version="1", repo="repo", extras="extras") 19 | assert isinstance(res, bool) and res 20 | 21 | res = dbutils.library.list() 22 | assert isinstance(res, list) and res == [] 23 | 24 | dbutils.library.restartPython() 25 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/library_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | 3 | 4 | def test_library(): 5 | with databricks_test.session() as dbrickstest: 6 | # Run notebook 7 | dbrickstest.run_notebook(".", "library_notebook") 8 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/multiple_runs_notebook.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # COMMAND ---------- 4 | 5 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 6 | if 'dbutils' not in locals(): 7 | import databricks_test 8 | databricks_test.inject_variables() 9 | 10 | # COMMAND ---------- 11 | 12 | # Widgets for interactive development. 13 | dbutils.widgets.text("input", "") 14 | dbutils.widgets.text("output", "") 15 | 16 | # COMMAND ---------- 17 | 18 | with open(dbutils.widgets.get('output'), "w") as output_file: 19 | output_file.write(dbutils.widgets.get('input')) 20 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/multiple_runs_notebook2.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # COMMAND ---------- 4 | 5 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 6 | if 'dbutils' not in locals(): 7 | import databricks_test 8 | databricks_test.inject_variables() 9 | 10 | # COMMAND ---------- 11 | 12 | # Widgets for interactive development. 13 | dbutils.widgets.text("input", "") 14 | dbutils.widgets.text("output", "") 15 | 16 | # COMMAND ---------- 17 | 18 | with open(dbutils.widgets.get('output'), "w") as output_file: 19 | output_file.write(dbutils.widgets.get('input')) 20 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/multiple_runs_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | from tempfile import NamedTemporaryFile 3 | import uuid 4 | 5 | 6 | def run_notebook(notebook, run_num, dbrickstest): 7 | 8 | input = str(uuid.uuid4()) 9 | 10 | with NamedTemporaryFile() as tmp_dir: 11 | 12 | # Provide input and output location as widgets to notebook 13 | switch = { 14 | "input": input, 15 | "output": tmp_dir.name, 16 | } 17 | dbrickstest.dbutils.widgets.get.side_effect = lambda x: switch.get( 18 | x, "") 19 | 20 | # Run notebook 21 | dbrickstest.run_notebook(".", notebook) 22 | 23 | # Notebook writes the input parameter as output file 24 | with open(tmp_dir.name) as output_file: 25 | assert input == output_file.read(), f"Run #{run_num} output" 26 | 27 | 28 | def test_multiple_runs_in_same_session_1(): 29 | with databricks_test.session() as dbrickstest: 30 | run_notebook("multiple_runs_notebook", 1, dbrickstest) 31 | run_notebook("multiple_runs_notebook", 2, dbrickstest) 32 | 33 | with databricks_test.session() as dbrickstest: 34 | run_notebook("multiple_runs_notebook", 3, dbrickstest) 35 | 36 | 37 | def test_multiple_runs_in_same_session_and_run_other_session(): 38 | with databricks_test.session() as dbrickstest: 39 | run_notebook("multiple_runs_notebook", 4, dbrickstest) 40 | 41 | 42 | def test_multiple_runs_in_multiple_test_cases(): 43 | with databricks_test.session() as dbrickstest: 44 | run_notebook("multiple_runs_notebook2", 5, dbrickstest) 45 | 46 | 47 | def test_multiple_runs_in_multiple_test_cases2(): 48 | with databricks_test.session() as dbrickstest: 49 | run_notebook("multiple_runs_notebook2", 6, dbrickstest) 50 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/patch_notebook.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # COMMAND ---------- 4 | 5 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 6 | if 'dbutils' not in locals(): 7 | import databricks_test 8 | databricks_test.inject_variables() 9 | 10 | # COMMAND ---------- 11 | 12 | assert dbutils.widgets.get("input") == "input_value" 13 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/patch_notebook2.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # COMMAND ---------- 4 | 5 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 6 | if 'dbutils' not in locals(): 7 | import databricks_test 8 | databricks_test.inject_variables() 9 | 10 | # COMMAND ---------- 11 | 12 | assert dbutils.widgets.get("input") == "" 13 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/patch_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | 3 | 4 | def test_patch(): 5 | with databricks_test.session() as dbrickstest: 6 | # Provide input and output location as widgets to notebook 7 | switcher = { 8 | "input": "input_value", 9 | "output": "output_value", 10 | } 11 | dbrickstest.dbutils.widgets.get.side_effect = lambda x: switcher.get( 12 | x, "") 13 | 14 | # Run notebook 15 | dbrickstest.run_notebook(".", "patch_notebook") 16 | 17 | with databricks_test.session() as dbrickstest: 18 | dbrickstest.run_notebook(".", "patch_notebook2") 19 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/secrets_notebook.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 4 | if 'dbutils' not in locals(): 5 | import databricks_test 6 | databricks_test.inject_variables() 7 | 8 | # COMMAND ---------- 9 | 10 | res = dbutils.secrets.help() 11 | 12 | res = dbutils.secrets.get("scope", "key") 13 | assert isinstance(res, str) and res == "" 14 | 15 | res = dbutils.secrets.getBytes("scope", "key") 16 | assert isinstance(res, bytearray) and res == bytearray() 17 | 18 | res = dbutils.secrets.list("scope") 19 | assert isinstance(res, list) and res == [] 20 | 21 | res = dbutils.secrets.listScopes() 22 | assert isinstance(res, list) and res == [] 23 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/secrets_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | 3 | 4 | def test_secrets(): 5 | with databricks_test.session() as dbrickstest: 6 | # Run notebook 7 | dbrickstest.run_notebook(".", "secrets_notebook") 8 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/sqldw_expected.csv: -------------------------------------------------------------------------------- 1 | id,age 2 | 0,0 3 | 1,6 4 | 2,12 5 | 3,18 6 | 4,24 7 | 5,30 8 | 6,36 9 | 7,42 10 | 8,48 11 | 9,54 12 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/sqldw_notebook.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 4 | if 'dbutils' not in locals(): 5 | import databricks_test 6 | databricks_test.inject_variables() 7 | 8 | # COMMAND ---------- 9 | 10 | # Set up the Blob storage account access key in the notebook session conf. 11 | spark.conf.set( 12 | "fs.azure.account.key.myaccount.blob.core.windows.net", 13 | dbutils.secrets.get("dwsecrets", "accountkey")) 14 | 15 | # Get some data from a SQL DW table. 16 | df = spark.read \ 17 | .format("com.databricks.spark.sqldw") \ 18 | .option('url', 'jdbc:sqlserver://myserver.database.windows.net:1433;database=mydb;user=myuser;password=secret;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;')\ 19 | .option("tempDir", "wasbs://mycontainer@myaccount.blob.core.windows.net/") \ 20 | .option("forwardSparkAzureStorageCredentials", "true") \ 21 | .option("dbTable", "mytable") \ 22 | .load() 23 | 24 | # Apply some transformations to the data, then use the 25 | # Data Source API to write the data back to another table in SQL DW. 26 | 27 | dfres = df.select("id", "age") 28 | 29 | dfres.write \ 30 | .format("com.databricks.spark.sqldw") \ 31 | .option('url', 'jdbc:sqlserver://myserver.database.windows.net:1433;database=mydb;user=myuser;password=secret;encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;')\ 32 | .option("forwardSparkAzureStorageCredentials", "true") \ 33 | .option("dbTable", "my_table_in_dw_copy") \ 34 | .option("tempDir", "wasbs://mycontainer@myaccount.blob.core.windows.net/") \ 35 | .save() 36 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/sqldw_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | import pyspark 3 | import pyspark.sql.functions as F 4 | from tempfile import TemporaryDirectory 5 | from pandas.testing import assert_frame_equal 6 | import pandas as pd 7 | 8 | 9 | def test_sqldw(monkeypatch): 10 | with databricks_test.session() as dbrickstest, TemporaryDirectory() as tmp: 11 | 12 | out_dir = f"{tmp}/out" 13 | 14 | # Mock SQL DW loader, creating a Spark DataFrame instead 15 | def mock_load(reader): 16 | return ( 17 | dbrickstest.spark 18 | .range(10) 19 | .withColumn("age", F.col("id") * 6) 20 | .withColumn("salary", F.col("id") * 10000) 21 | ) 22 | 23 | monkeypatch.setattr( 24 | pyspark.sql.readwriter.DataFrameReader, "load", mock_load) 25 | 26 | # Mock SQL DW writer, writing to a local Parquet file instead 27 | def mock_save(writer): 28 | monkeypatch.undo() 29 | writer.format("parquet") 30 | writer.save(out_dir) 31 | 32 | monkeypatch.setattr( 33 | pyspark.sql.readwriter.DataFrameWriter, "save", mock_save) 34 | 35 | # Run notebook 36 | dbrickstest.run_notebook(".", "sqldw_notebook") 37 | 38 | # Notebook produces a Parquet file (directory) 39 | resultDF = pd.read_parquet(out_dir) 40 | 41 | # Compare produced Parquet file and expected CSV file 42 | expectedDF = pd.read_csv("tests/sqldw_expected.csv") 43 | assert_frame_equal(expectedDF, resultDF, check_dtype=False) 44 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/widgets_notebook.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 4 | if 'dbutils' not in locals(): 5 | import databricks_test 6 | databricks_test.inject_variables() 7 | 8 | # COMMAND ---------- 9 | 10 | dbutils.widgets.help() 11 | 12 | dbutils.widgets.combobox("name", "defaultValue", [ 13 | "choice1", "choice2"], "label") 14 | 15 | dbutils.widgets.dropdown("name", "defaultValue", [ 16 | "choice1", "choice2"], "label") 17 | 18 | res = dbutils.widgets.get("name") 19 | assert isinstance(res, str) and res == "" 20 | 21 | res = dbutils.widgets.getArgument("name") 22 | assert isinstance(res, str) and res == "" 23 | res = dbutils.widgets.getArgument("name", "optional") 24 | assert isinstance(res, str) and res == "" 25 | 26 | dbutils.widgets.multiselect("name", "defaultValue", [ 27 | "choice1", "choice2"], "label") 28 | 29 | dbutils.widgets.remove("name") 30 | 31 | dbutils.widgets.removeAll() 32 | 33 | dbutils.widgets.text("name", "defaultValue", "label") 34 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/widgets_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | 3 | 4 | def test_widgets(): 5 | with databricks_test.session() as dbrickstest: 6 | # Run notebook 7 | dbrickstest.run_notebook(".", "widgets_notebook") 8 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/workflow_notebook.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | 3 | # Instrument for unit tests. This is only executed in local unit tests, not in Databricks. 4 | if 'dbutils' not in locals(): 5 | import databricks_test 6 | databricks_test.inject_variables() 7 | 8 | # COMMAND ---------- 9 | 10 | res = dbutils.notebook.help() 11 | 12 | res = dbutils.notebook.run("value", 0, {"arg": "value"}) 13 | assert isinstance(res, str) and res == "" 14 | 15 | res = dbutils.notebook.exit("value") 16 | assert False, "should not be executed" 17 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tests/workflow_test.py: -------------------------------------------------------------------------------- 1 | import databricks_test 2 | 3 | 4 | def test_workflow(): 5 | with databricks_test.session() as dbrickstest: 6 | # Run notebook 7 | dbrickstest.run_notebook(".", "workflow_notebook") 8 | -------------------------------------------------------------------------------- /Python/packages/databricks-test/tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = True 3 | 4 | [flake8] 5 | # ignore obsolete warning 6 | ignore = W503 7 | exclude = .git,__pycache__,.venv,.tox 8 | per-file-ignores = 9 | # Ignore import and line length issues in Databricks notebooks 10 | tests/*notebook*.py:F821,F405,E402,E501 11 | 12 | [pytest] 13 | junit_family = legacy 14 | 15 | [testenv] 16 | whitelist_externals = 17 | bash 18 | python 19 | twine 20 | commands = 21 | bash -c "docker build -t databricks_test-test . && docker run -t -v $PWD/:/vol databricks_test-test bash -c 'cd /vol && flake8 && pytest'" 22 | python setup.py sdist bdist_wheel 23 | twine upload dist/* 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | languages: 4 | - csharp 5 | products: 6 | - dotnet 7 | description: "Add 150 character max description" 8 | urlFragment: "update-this-to-unique-url-stub" 9 | --- 10 | 11 | # Official Microsoft Sample 12 | 13 | 20 | 21 | Give a short description for your sample here. What does it do and why is it important? 22 | 23 | ## Contents 24 | 25 | Outline the file contents of the repository. It helps users navigate the codebase, build configuration and any related assets. 26 | 27 | | File/folder | Description | 28 | |-------------------|--------------------------------------------| 29 | | `src` | Sample source code. | 30 | | `.gitignore` | Define what to ignore at commit time. | 31 | | `CHANGELOG.md` | List of changes to the sample. | 32 | | `CONTRIBUTING.md` | Guidelines for contributing to the sample. | 33 | | `README.md` | This README file. | 34 | | `LICENSE` | The license for the sample. | 35 | 36 | ## Prerequisites 37 | 38 | Outline the required components and tools that a user might need to have on their machine in order to run the sample. This can be anything from frameworks, SDKs, OS versions or IDE releases. 39 | 40 | ## Setup 41 | 42 | Explain how to prepare the sample once the user clones or downloads the repository. The section should outline every step necessary to install dependencies and set up any settings (for example, API keys and output folders). 43 | 44 | ## Runnning the sample 45 | 46 | Outline step-by-step instructions to execute the sample and see its output. Include steps for executing the sample from the IDE, starting specific services in the Azure portal or anything related to the overall launch of the code. 47 | 48 | ## Key concepts 49 | 50 | Provide users with more context on the tools and services used in the sample. Explain some of the code that is being used and how services interact with each other. 51 | 52 | ## Contributing 53 | 54 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 55 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 56 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 57 | 58 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 59 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 60 | provided by the bot. You will only need to do this once across all repos using our CLA. 61 | 62 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 63 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 64 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 65 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets Microsoft's [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)) of a security vulnerability, please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /adf/utils/Invoke-ADFPipeline.ps1: -------------------------------------------------------------------------------- 1 | param( 2 | [parameter(Mandatory=$true)] 3 | $ResourceGroupName, 4 | [parameter(Mandatory=$true)] 5 | $DataFactoryName, 6 | [parameter(Mandatory=$true)] 7 | $PipelineName 8 | ) 9 | 10 | Import-Module "$($PSScriptRoot)\Utilities.psm1" 11 | 12 | $runId = Invoke-AzDataFactoryV2Pipeline -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -PipelineName $PipelineName 13 | 14 | Write-Host "##vso[task.setvariable variable=AdfPipeline.RunId]$($runId)" 15 | 16 | $runDetails = Get-AzDataFactoryV2PipelineRun -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -PipelineRunId $runId 17 | 18 | while ($runDetails.Status -eq "InProgress") { 19 | LogInfo "Waiting for the ADF Pipeline $($runDetails.PipelineName) to finish..." 20 | 21 | Start-Sleep -Seconds 5 22 | 23 | $runDetails = Get-AzDataFactoryV2PipelineRun -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -PipelineRunId $runId 24 | } 25 | 26 | if ($runDetails.Status -eq "Succeeded") { 27 | LogInfo "Data Factory Pipeline $($runDetails.PipelineName) succeeded!" 28 | } 29 | else { 30 | LogError "Data Factory Pipeline $($runDetails.PipelineName) failed: $($runDetails.Message)" 31 | throw "Error executing the ADF Pipeline. Check the message above for details." 32 | } -------------------------------------------------------------------------------- /adf/utils/New-DataFactory.ps1: -------------------------------------------------------------------------------- 1 | param( 2 | [string]$DataFactoryName, 3 | [string]$ResourceGroupName, 4 | [string]$Location 5 | ) 6 | 7 | Import-Module "$($PSScriptRoot)\Utilities.psm1" 8 | 9 | LogInfo "Creating Data Factory [$($DataFactoryName)] in Resource Group [$($ResourceGroupName)]..." 10 | if ((Get-AzDataFactoryV2 -Name $DataFactoryName -ResourceGroupName $ResourceGroupName -ErrorAction SilentlyContinue) -eq $null) 11 | { 12 | LogInfo "`tData Factory [$($DataFactoryName)] Does Not Exist. Creating..." 13 | 14 | Set-AzDataFactoryV2 ` 15 | -Name $DataFactoryName ` 16 | -Location $Location ` 17 | -ResourceGroupName $ResourceGroupName 18 | 19 | LogInfo "`tData Factory [$($DataFactoryName)] Created!" 20 | 21 | } 22 | else 23 | { 24 | LogInfo "`tData Factory [$($DataFactoryName)] Exists. Moving On!" 25 | } 26 | -------------------------------------------------------------------------------- /adf/utils/New-StorageAccount.ps1: -------------------------------------------------------------------------------- 1 | param( 2 | [string]$AccountName, 3 | [ValidateSet('Standard_LRS', 'Standard_GRS', 'Standard_RAGRS')] $SKU, 4 | [string]$ResourceGroupName, 5 | [string]$Location, 6 | [bool]$ADLSGen2=$false, 7 | [bool]$AssignIdentity=$true, 8 | [parameter(Mandatory=$true)] 9 | [string]$ContainerName 10 | ) 11 | 12 | Import-Module "$($PSScriptRoot)\Utilities.psm1" 13 | 14 | LogInfo "Creating Storage Account [$($AccountName)] in Resource Group [$($ResourceGroupName)]..." 15 | 16 | $storageAccount = Get-AzStorageAccount -Name $AccountName -ResourceGroupName $ResourceGroupName -ErrorAction SilentlyContinue 17 | 18 | if ($storageAccount -eq $null) 19 | { 20 | LogInfo "`tStorage Account [$($AccountName)] Does Not Exist. Creating..." 21 | 22 | $storageAccount = New-AzStorageAccount ` 23 | -Name $AccountName ` 24 | -Location $Location ` 25 | -ResourceGroupName $ResourceGroupName ` 26 | -SkuName $SKU ` 27 | -AccessTier Hot ` 28 | -EnableHttpsTrafficOnly $true ` 29 | -EnableHierarchicalNamespace $ADLSGen2 ` 30 | -Kind StorageV2 ` 31 | -AssignIdentity:$AssignIdentity 32 | 33 | LogInfo "`tStorage Account [$($AccountName)] Created!" 34 | } 35 | else 36 | { 37 | LogInfo "`tStorage Account [$($AccountName)] Exists. Moving On!" 38 | } 39 | 40 | $storageAccountContext = $storageAccount.Context 41 | 42 | LogInfo "Checking if the input blob container exists and, if not, we will create it..." 43 | if((Get-AzRmStorageContainer -ResourceGroupName $ResourceGroupName -StorageAccountName $AccountName -Name $ContainerName -ErrorAction SilentlyContinue) -eq $null) 44 | { 45 | LogInfo "`tBlob Container $($ContainerName) does not exist. Creating..." 46 | 47 | New-AzStorageContainer ` 48 | -Name $ContainerName ` 49 | -Context $storageAccountContext ` 50 | -Permission blob 51 | 52 | LogInfo "`tBlob Container $($ContainerName) created!" 53 | 54 | LogInfo "`tUploading Data Source files to the $($ContainerName) container..." 55 | } 56 | else 57 | { 58 | LogInfo "`tBlob Container $($ContainerName) exists. Moving on!" 59 | } 60 | 61 | LogInfo "`tUploading data source files to the $($ContainerName)" 62 | 63 | $sourceFiles = Get-ChildItem "$($PSScriptRoot)\DataSourceFiles" -Filter *.csv 64 | 65 | foreach ($item in $sourceFiles) { 66 | Set-AzStorageBlobContent ` 67 | -Container $ContainerName ` 68 | -File $item.FullName ` 69 | -Blob "input/$($item.Name)" ` 70 | -Context $storageAccountContext ` 71 | -ErrorAction SilentlyContinue 72 | } 73 | 74 | $storageAccountKeys = Get-AzStorageAccountKey -ResourceGroupName $ResourceGroupName -Name $AccountName 75 | 76 | $storageAccountKey = $storageAccountKeys[0].Value 77 | 78 | Write-Host "##vso[task.setvariable variable=StorageAccount.Key;issecret=true]$($storageAccountKey)" -------------------------------------------------------------------------------- /adf/utils/Utilities.psm1: -------------------------------------------------------------------------------- 1 | function LogInfo { 2 | param ($Text) 3 | 4 | WriteToConsole -Text $Text -LogLevel "Info" -ForegroundColor White 5 | } 6 | 7 | function LogError { 8 | param ($Text) 9 | 10 | WriteToConsole -Text $Text -LogLevel "Error" -ForegroundColor Red 11 | } 12 | 13 | function LogWarning { 14 | param ($Text) 15 | 16 | WriteToConsole -Text $Text -LogLevel "Warn" -ForegroundColor Yellow 17 | } 18 | 19 | function WriteToConsole { 20 | param ($Text, $LogLevel, $ForegroundColor) 21 | 22 | Write-Host -ForegroundColor $ForegroundColor "[$(Get-Date)] [$($LogLevel)]`t$($Text)" 23 | } 24 | 25 | function Get-MDWResourceGroupName { 26 | param([parameter(Mandatory=$true)][string]$baseName) 27 | 28 | $resourceGroupName = "$($baseName)-RG" 29 | 30 | return $resourceGroupName 31 | } 32 | 33 | function Get-MDWVnetName { 34 | param([parameter(Mandatory=$true)][string]$baseName) 35 | 36 | $vnetName = "$($baseName)-VNet" 37 | 38 | return $vnetName 39 | } 40 | 41 | function Get-MDWKeyVaultName { 42 | param( 43 | [parameter(Mandatory=$true)][string]$baseName, 44 | [bool]$encrypted=$false 45 | ) 46 | 47 | $keyVaultName = "" 48 | 49 | if(!$encrypted) { 50 | $keyVaultName = "$($baseName)-KV" 51 | } else { 52 | $keyVaultName = "$($baseName)-KV-Enc" 53 | } 54 | 55 | return $keyVaultName 56 | } 57 | 58 | function Get-MDWDataFactoryName{ 59 | param([parameter(Mandatory=$true)][string]$baseName) 60 | 61 | $dataFactoryName = "$($baseName)-ADF" 62 | 63 | return $dataFactoryName 64 | } 65 | 66 | function Get-MDWNetworkSecurityGroupName { 67 | param([parameter(Mandatory=$true)][string]$baseName) 68 | 69 | $nsgName = "$($baseName)-ADB-NSG" 70 | 71 | return $nsgName 72 | } --------------------------------------------------------------------------------