├── .github ├── workflows │ ├── container-update │ │ ├── .install-test │ │ └── .install-all │ ├── docker-lint.yml │ ├── update-version.yml │ ├── pypi.yml │ ├── ossar-analysis.yml │ ├── draft-release.yml │ ├── pypi-test-publish.yml │ ├── docker-publish.yml │ ├── github-container.yml │ ├── container-update.yml │ └── codeql-analysis.yml └── dependabot.yml ├── MANIFEST.in ├── .pypirc ├── src └── az-python │ └── __init__.py ├── CODE_OF_CONDUCT.md ├── LICENSE ├── pyproject.toml ├── SUPPORT.md ├── README.md ├── SECURITY.md └── .gitignore /.github/workflows/container-update/.install-test: -------------------------------------------------------------------------------- 1 | ai-python[test] 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include VERSION 2 | 3 | include piptools.txt 4 | include requirements.txt 5 | include test/requirements.txt 6 | 7 | recursive-include .depends/ *.txt 8 | -------------------------------------------------------------------------------- /.github/workflows/container-update/.install-all: -------------------------------------------------------------------------------- 1 | ai-python[core,ama,reco,msft-utils,retail,energy-ca,energy-km,fclib,fsi-nlp,fsi-risk,fsi,gem-ent,gem-reco,gem,sc-ro,sc-ip,sc-ecr,sc,cdm,maro,all] 2 | -------------------------------------------------------------------------------- /.pypirc: -------------------------------------------------------------------------------- 1 | [distutils] 2 | index-servers = 3 | pypi 4 | testpypi 5 | 6 | [pypi] 7 | repository = https://upload.pypi.org/legacy/ 8 | 9 | [testpypi] 10 | repository = https://test.pypi.org/legacy/ 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "." 5 | schedule: 6 | interval: daily 7 | time: "13:00" 8 | open-pull-requests-limit: 50 9 | assignees: 10 | - dciborow 11 | - mattchansky 12 | allow: 13 | - dependency-type: direct 14 | -------------------------------------------------------------------------------- /src/az-python/__init__.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # Licensed under the MIT License. See LICENSE in project root for information. 4 | # ------------------------------------------------------------- 5 | """Collection of Python Libraries for use on Azure""" 6 | __version__="1.0.2" 7 | -------------------------------------------------------------------------------- /.github/workflows/docker-lint.yml: -------------------------------------------------------------------------------- 1 | 2 | name: Docker Linting 3 | 4 | on: 5 | pull_request: 6 | branches: [ main ] 7 | paths: 8 | - 'Dockerfile' 9 | 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | 19 | - name: Docker Lint 20 | uses: luke142367/Docker-Lint-Action@v1.1.1 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /.github/workflows/update-version.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Update Version 4 | 5 | # Controls when the workflow will run 6 | on: 7 | # Allows you to run this workflow manually from the Actions tab 8 | workflow_dispatch: 9 | 10 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 11 | jobs: 12 | # This workflow contains a single job called "build" 13 | build: 14 | # The type of runner that the job will run on 15 | runs-on: ubuntu-latest 16 | 17 | # Steps represent a sequence of tasks that will be executed as part of the job 18 | steps: 19 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 20 | - name: Bump version 21 | uses: remorses/bump-version@js 22 | with: 23 | version_file: ./VERSION 24 | github_token: ${{ secrets.GITHUB_TOKEN }} 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "azpy" 7 | authors = [{name = "Daniel Ciborowski", email = "dciborow@microsoft.com"}] 8 | readme = "README.md" 9 | license = {file = "LICENSE"} 10 | classifiers = ["License :: OSI Approved :: MIT License"] 11 | dynamic = ["version", "description"] 12 | 13 | [project.urls] 14 | Home = "https://github.com/microsoft/az-python" 15 | 16 | [project.optional-dependencies] 17 | audio-speech = [ 18 | "torchaudio<=0.7.2", 19 | "soundfile<=0.10.3.post1", 20 | "pydub<=0.25.1", 21 | "librosa<=0.7.0", 22 | "pedalboard<=0.3.6" 23 | ] 24 | ama = [ 25 | "azure-mgmt-deploymentmanager", 26 | "azure-mgmt-resource", 27 | "azure-storage-blob", 28 | "azure-identity>=1.2.0", 29 | "urllib3>=1.15", 30 | "six>=1.10", 31 | "certifi", 32 | "python-dateutil", 33 | "requests", 34 | "pyyaml", 35 | "wget", 36 | "adal", 37 | "cryptography>=3.3.1" 38 | ] 39 | msft-utils = [ 40 | "azureml-core==1.33.0", 41 | "azure-identity>=1.4.1", 42 | "azure-keyvault==4.1.0", 43 | "msrestazure>=0.6.4", 44 | "azure-mgmt-resource>=12.1.0", 45 | "python-dotenv>=0.14.0,<0.20.0" 46 | ] 47 | 48 | [tool.flit.module] 49 | name = "az-python" 50 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: Python CI 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | release: 8 | types: [created] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | publish: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | 17 | - if: ${{ github.event_name == 'release' }} 18 | name: Publish Release to PyPi 19 | uses: dciborow/pyaction@0.0.28 20 | with: 21 | pypi_publish: true 22 | pypi_password: ${{ secrets.PYPI_PASSWORD }} 23 | 24 | - if: ${{ github.event_name == 'push' }} 25 | name: Publish RC to PyPi 26 | uses: dciborow/pyaction@0.0.28 27 | with: 28 | pypi_publish: true 29 | pypi_password: ${{ secrets.PYPI_PASSWORD }} 30 | version_suffix: -rc${{ github.run_number }}-post${{ github.run_attempt }} 31 | 32 | - if: ${{ github.event_name == 'pull_request' }} 33 | name: Publish Snapshot to TestPyPi 34 | uses: dciborow/pyaction@0.0.28 35 | with: 36 | pypi_publish: true 37 | pypi_password: ${{ secrets.TEST_PYPI_PASSWORD }} 38 | pypi_repo: testpypi 39 | version_suffix: -post${{ github.run_number }}-dev${{ github.run_attempt }} 40 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/spot](https://aka.ms/spot). CSS will work with/help you to determine next steps. More details also available at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). 7 | - **Not sure?** Fill out a SPOT intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /.github/workflows/ossar-analysis.yml: -------------------------------------------------------------------------------- 1 | # This workflow integrates a collection of open source static analysis tools 2 | # with GitHub code scanning. For documentation, or to provide feedback, visit 3 | # https://github.com/github/ossar-action 4 | name: OSSAR 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | # The branches below must be a subset of the branches above 11 | branches: [ main ] 12 | schedule: 13 | - cron: '18 12 * * 3' 14 | 15 | jobs: 16 | OSSAR-Scan: 17 | # OSSAR runs on windows-latest. 18 | # ubuntu-latest and macos-latest support coming soon 19 | runs-on: windows-latest 20 | 21 | steps: 22 | - name: Checkout repository 23 | uses: actions/checkout@v2 24 | 25 | # Ensure a compatible version of dotnet is installed. 26 | # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201. 27 | # A version greater than or equal to v3.1.201 of dotnet must be installed on the agent in order to run this action. 28 | # GitHub hosted runners already have a compatible version of dotnet installed and this step may be skipped. 29 | # For self-hosted runners, ensure dotnet version 3.1.201 or later is installed by including this action: 30 | # - name: Install .NET 31 | # uses: actions/setup-dotnet@v1 32 | # with: 33 | # dotnet-version: '3.1.x' 34 | 35 | # Run open source static analysis tools 36 | - name: Run OSSAR 37 | uses: github/ossar-action@v1 38 | id: ossar 39 | 40 | # Upload results to the Security tab 41 | - name: Upload OSSAR results 42 | uses: github/codeql-action/upload-sarif@v1 43 | with: 44 | sarif_file: ${{ steps.ossar.outputs.sarifFile }} 45 | -------------------------------------------------------------------------------- /.github/workflows/draft-release.yml: -------------------------------------------------------------------------------- 1 | name: Create Draft Release on Version Update 2 | on: 3 | push: 4 | branches: [ main ] 5 | paths: 6 | - 'VERSION' 7 | workflow_dispatch: 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Setup Python 15 | uses: actions/setup-python@v2.2.2 16 | with: 17 | python-version: 3.7 18 | - name: Install dependencies 19 | run: | 20 | python3.7 -m pip install --upgrade pip 21 | python3.7 -m pip install setuptools wheel twine 22 | python3.7 -m pip install virtualenv pip-tools 23 | sudo add-apt-repository -y ppa:ubuntugis/ppa \ 24 | && sudo apt install -y \ 25 | gdal-bin libgdal-dev 26 | mv ./.depends/requirements.txt requirements.in 27 | touch requirements.txt 28 | echo "" >> requirements.in 29 | cat requirements.linked.in >> requirements.in 30 | pip-compile -v requirements.in 31 | cd .depends 32 | requirements=$(find . -type f -name "*.requirements*" | paste -sd " ") 33 | for req in $requirements 34 | do 35 | echo "Solving for Requirements : $req" 36 | mv $req temp.in 37 | echo "-c ../requirements.txt" >> temp.in 38 | touch $req 39 | pip-compile temp.in -o $req 40 | done 41 | cd .. 42 | mv requirements.txt ./.depends/requirements.txt 43 | python setup.py sdist bdist_wheel 44 | 45 | - name: Get version from ref 46 | id: get-version 47 | run: echo ::set-output name=VERSION::$(cat VERSION) 48 | - uses: ncipollo/release-action@v1 49 | with: 50 | artifacts: "*.tar.gz,*.whl" 51 | bodyFile: "body.md" 52 | token: ${{ secrets.GITHUB_TOKEN }} 53 | -------------------------------------------------------------------------------- /.github/workflows/pypi-test-publish.yml: -------------------------------------------------------------------------------- 1 | name: Test Upload Python Package 2 | 3 | on: 4 | workflow_dispatch 5 | 6 | jobs: 7 | deploy: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | with: 12 | python-version: '3.7' 13 | - name: Setup Python 14 | uses: actions/setup-python@v2.2.2 15 | with: 16 | python-version: 3.7 17 | - name: Install dependencies 18 | run: | 19 | python3.7 -m pip install --upgrade pip 20 | python3.7 -m pip install virtualenv pip-tools 21 | sudo add-apt-repository -y ppa:ubuntugis/ppa \ 22 | && sudo apt install -y \ 23 | gdal-bin libgdal-dev 24 | 25 | mv requirements.txt requirements.in 26 | touch requirements.txt 27 | echo "" >> requirements.in 28 | cat requirements.linked.in >> requirements.in 29 | pip-compile -v requirements.in 30 | 31 | cd ai-python 32 | 33 | requirements=$(find . -type f -name "*requirements*" | paste -sd " ") 34 | 35 | for req in $requirements 36 | do 37 | mv $req temp.in 38 | echo "-c ../requirements.txt" >> temp.in 39 | touch $req 40 | pip-compile temp.in -o $req 41 | done 42 | 43 | cd .. 44 | 45 | python setup.py sdist bdist_wheel 46 | - name: Publish package 47 | uses: pypa/gh-action-pypi-publish@release/v1 48 | with: 49 | user: __token__ 50 | password: ${{ secrets.TEST_PYPI_PASSWORD }} 51 | repository_url: https://test.pypi.org/legacy/ 52 | - name: Install dependencies 53 | run: | 54 | python3.7 -m pip install --upgrade pip 55 | python3.7 -m pip install virtualenv pip-tools 56 | pip-compile --extra-index-url https://test.pypi.org/simple \ 57 | ai-python[core,ama,reco,msft-utils,retail,energy-ca,energy-km,fclib,fsi-nlp,fsi-risk,fsi,gem-ent,gem-reco,gem,sc-ro,sc-ip,sc-ecr,sc,cdm,maro,all] 58 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | # This workflow uses actions that are not certified by GitHub. 4 | # They are provided by a third-party and are governed by 5 | # separate terms of service, privacy policy, and support 6 | # documentation. 7 | 8 | on: 9 | workflow_dispatch 10 | 11 | env: 12 | # github.repository as / 13 | IMAGE_NAME: ${{ github.repository }} 14 | 15 | 16 | jobs: 17 | build: 18 | 19 | runs-on: ubuntu-latest 20 | permissions: 21 | contents: read 22 | packages: write 23 | 24 | steps: 25 | - name: Checkout repository 26 | uses: actions/checkout@v2 27 | 28 | # Login against a Docker registry except on PR 29 | # https://github.com/docker/login-action 30 | - name: Log into registry ${{ env.REGISTRY }} 31 | if: github.event_name != 'pull_request' 32 | uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c 33 | with: 34 | registry: ${{ env.REGISTRY }} 35 | username: ${{ github.actor }} 36 | password: ${{ secrets.DOCKERHUB_TOKEN }} 37 | 38 | # Extract metadata (tags, labels) for Docker 39 | # https://github.com/docker/metadata-action 40 | - name: Extract Docker metadata 41 | id: meta 42 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 43 | with: 44 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 45 | 46 | # Build and push Docker image with Buildx (don't push on PR) 47 | # https://github.com/docker/build-push-action 48 | - name: Build and push Docker image 49 | uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc 50 | with: 51 | context: . 52 | push: ${{ github.event_name != 'pull_request' }} 53 | tags: "dciborow/ai-python:latest" 54 | labels: ${{ steps.meta.outputs.labels }} 55 | secrets: | 56 | MSAZURE_ADO_PAT=${{ secrets.MSAZURE_ADO_PAT }} 57 | -------------------------------------------------------------------------------- /.github/workflows/github-container.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Package - Docker Image 2 | 3 | # This workflow uses actions that are not certified by GitHub. 4 | # They are provided by a third-party and are governed by 5 | # separate terms of service, privacy policy, and support 6 | # documentation. 7 | 8 | on: 9 | workflow_dispatch 10 | 11 | env: 12 | # github.repository as / 13 | IMAGE_NAME: ${{ github.repository }} 14 | REGISTRY: https://ghcr.io 15 | 16 | 17 | jobs: 18 | build: 19 | 20 | runs-on: ubuntu-latest 21 | permissions: 22 | contents: read 23 | packages: write 24 | 25 | steps: 26 | - name: Checkout repository 27 | uses: actions/checkout@v2 28 | 29 | # Login against a Docker registry except on PR 30 | # https://github.com/docker/login-action 31 | - name: Log into registry ${{ env.REGISTRY }} 32 | if: github.event_name != 'pull_request' 33 | uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c 34 | with: 35 | registry: ${{ env.REGISTRY }} 36 | username: ${{ github.actor }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | # Extract metadata (tags, labels) for Docker 40 | # https://github.com/docker/metadata-action 41 | - name: Extract Docker metadata 42 | id: meta 43 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 44 | with: 45 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 46 | 47 | # Build and push Docker image with Buildx (don't push on PR) 48 | # https://github.com/docker/build-push-action 49 | - name: Build and push Docker image 50 | uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc 51 | with: 52 | context: . 53 | push: ${{ github.event_name != 'pull_request' }} 54 | tags: "microsoft/ai-python:latest" 55 | labels: ${{ steps.meta.outputs.labels }} 56 | secrets: | 57 | MSAZURE_ADO_PAT=${{ secrets.MSAZURE_ADO_PAT }} 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Python by Microsoft 2 | 3 | This is a collection of Python packages scanned using various open source and internal tools to provide up-to-date, and secury dependencies. 4 | 5 | ## Installation 6 | Currently Python3.7 is supported. 7 | 8 | ### Ubuntu 18.04 9 | The aiubuntu library is provided to simplify installation of required Ubuntu dependancies. 10 | 11 | shell 12 | 13 | add-apt-repository -y ppa:deadsnakes/ppa \ 14 | && apt-get update \ 15 | && apt-get install -y python3.7 python3.7-dev 16 | 17 | curl -fSsLO https://bootstrap.pypa.io/get-pip.py \ 18 | && /usr/bin/python3.7 get-pip.py 'pip==20.3.3' 19 | 20 | add-apt-repository -y ppa:dciborow/ppa \ 21 | && apt-get update \ 22 | && apt-get install -y aiubuntu 23 | 24 | # For Core Libaries (this will install nearly everything) 25 | python3.7 -m pip install ai-python[core] 26 | 27 | # For Libraries needed for testing 28 | python3.7 -m pip install ai-python[tests] 29 | 30 | # For Libraries needed for a single package 31 | python3.7 -m pip install ai-python[retail] 32 | 33 | ## Contributing 34 | 35 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 36 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 37 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 38 | 39 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 40 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 41 | provided by the bot. You will only need to do this once across all repos using our CLA. 42 | 43 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 44 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 45 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 46 | 47 | ## Trademarks 48 | 49 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 50 | trademarks or logos is subject to and must follow 51 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 52 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 53 | Any use of third-party trademarks or logos are subject to those third-party's policies. 54 | -------------------------------------------------------------------------------- /.github/workflows/container-update.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow that is manually triggered 2 | 3 | name: Publish Container Update 4 | 5 | # Controls when the action will run. Workflow runs when manually triggered using the UI 6 | # or API. 7 | on: 8 | workflow_dispatch: 9 | workflow_run: 10 | workflows: ["Upload Python Package"] 11 | types: [completed] 12 | 13 | jobs: 14 | update_container: 15 | runs-on: ubuntu-latest 16 | # if: ${{ github.event.workflow_run.conclusion == 'success' }} 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: '3.7' 23 | - name: Install dependencies 24 | run: | 25 | python3.7 -m pip install --upgrade pip 26 | python3.7 -m pip install virtualenv 27 | 28 | sudo add-apt-repository -y ppa:ubuntugis/ppa \ 29 | && sudo apt install -y \ 30 | gdal-bin libgdal-dev 31 | - name: Update dependencies 32 | run: | 33 | version=$(cat ./VERSION) 34 | cd .. 35 | git clone https://${{ secrets.ADO_USER }}:${{ secrets.ADO_PASSWORD }}@cloudes.visualstudio.com/CDPX/_git/CDPX-Linux-Images 36 | cd CDPX-Linux-Images 37 | 38 | cd src/user/agai/agai-template-ubuntu-1804/latest/requirements 39 | 40 | install_all="ai-python[core,ama,reco,msft-utils,retail,energy-ca,energy-km,fclib,fsi-nlp,fsi-risk,fsi,gem-ent,gem-reco,gem,sc-ro,sc-ip,sc-ecr,sc,cdm,maro,all]==$version" 41 | install_test="ai-python[test]==$version" 42 | 43 | echo "$install_all" > core.in 44 | echo "$install_test" > test.in 45 | 46 | ./generate.sh -s . 47 | 48 | git config --global user.email "dciborow@microsoft.com" 49 | git config --global user.name "Daniel Ciborowski" 50 | 51 | git add -A 52 | git commit -m "Updating Container" 53 | 54 | if git rev-parse --quiet --verify dciborow/updateContainer > /dev/null; then 55 | git fetch 56 | git merge -X ours dciborow/updateContainer 57 | fi 58 | 59 | git push origin --force master:dciborow/updateContainer 60 | - name: HTTP Request Action 61 | uses: fjogeleit/http-request-action@v1.8.0 62 | continue-on-error: true 63 | with: 64 | url: 'https://cloudes.visualstudio.com/CDPX/_apis/git/repositories/CDPX-Linux-Images/pullrequests?api-version=6.0' 65 | method: 'POST' 66 | username: "dciborow" 67 | password: ${{ secrets.ADO_PAT }} 68 | data: '{"targetRefName": "refs/heads/master", "sourceRefName": "refs/heads/dciborow/updateContainer", "title": "Dependancy Update", "autoCompleteSetBy": "dciborow@microsoft.com"}' 69 | ignoreStatusCodes: '409' 70 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '27 23 * * 1' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'python' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | 69 | # - name: Python Safety Check 70 | # uses: aufdenpunkt/python-safety-check@v1.0.5 71 | # with: 72 | # scan_requirements_file_only: 'true' 73 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Aa][Rr][Mm]/ 27 | [Aa][Rr][Mm]64/ 28 | bld/ 29 | [Bb]in/ 30 | [Oo]bj/ 31 | [Ll]og/ 32 | [Ll]ogs/ 33 | 34 | # Visual Studio 2015/2017 cache/options directory 35 | .vs/ 36 | # Uncomment if you have tasks that create the project's static files in wwwroot 37 | #wwwroot/ 38 | 39 | # Visual Studio 2017 auto generated files 40 | Generated\ Files/ 41 | 42 | # MSTest test Results 43 | [Tt]est[Rr]esult*/ 44 | [Bb]uild[Ll]og.* 45 | 46 | # NUnit 47 | *.VisualState.xml 48 | TestResult.xml 49 | nunit-*.xml 50 | 51 | # Build Results of an ATL Project 52 | [Dd]ebugPS/ 53 | [Rr]eleasePS/ 54 | dlldata.c 55 | 56 | # Benchmark Results 57 | BenchmarkDotNet.Artifacts/ 58 | 59 | # .NET Core 60 | project.lock.json 61 | project.fragment.lock.json 62 | artifacts/ 63 | 64 | # StyleCop 65 | StyleCopReport.xml 66 | 67 | # Files built by Visual Studio 68 | *_i.c 69 | *_p.c 70 | *_h.h 71 | *.ilk 72 | *.meta 73 | *.obj 74 | *.iobj 75 | *.pch 76 | *.pdb 77 | *.ipdb 78 | *.pgc 79 | *.pgd 80 | *.rsp 81 | *.sbr 82 | *.tlb 83 | *.tli 84 | *.tlh 85 | *.tmp 86 | *.tmp_proj 87 | *_wpftmp.csproj 88 | *.log 89 | *.vspscc 90 | *.vssscc 91 | .builds 92 | *.pidb 93 | *.svclog 94 | *.scc 95 | 96 | # Chutzpah Test files 97 | _Chutzpah* 98 | 99 | # Visual C++ cache files 100 | ipch/ 101 | *.aps 102 | *.ncb 103 | *.opendb 104 | *.opensdf 105 | *.sdf 106 | *.cachefile 107 | *.VC.db 108 | *.VC.VC.opendb 109 | 110 | # Visual Studio profiler 111 | *.psess 112 | *.vsp 113 | *.vspx 114 | *.sap 115 | 116 | # Visual Studio Trace Files 117 | *.e2e 118 | 119 | # TFS 2012 Local Workspace 120 | $tf/ 121 | 122 | # Guidance Automation Toolkit 123 | *.gpState 124 | 125 | # ReSharper is a .NET coding add-in 126 | _ReSharper*/ 127 | *.[Rr]e[Ss]harper 128 | *.DotSettings.user 129 | 130 | # TeamCity is a build add-in 131 | _TeamCity* 132 | 133 | # DotCover is a Code Coverage Tool 134 | *.dotCover 135 | 136 | # AxoCover is a Code Coverage Tool 137 | .axoCover/* 138 | !.axoCover/settings.json 139 | 140 | # Visual Studio code coverage results 141 | *.coverage 142 | *.coveragexml 143 | 144 | # NCrunch 145 | _NCrunch_* 146 | .*crunch*.local.xml 147 | nCrunchTemp_* 148 | 149 | # MightyMoose 150 | *.mm.* 151 | AutoTest.Net/ 152 | 153 | # Web workbench (sass) 154 | .sass-cache/ 155 | 156 | # Installshield output folder 157 | [Ee]xpress/ 158 | 159 | # DocProject is a documentation generator add-in 160 | DocProject/buildhelp/ 161 | DocProject/Help/*.HxT 162 | DocProject/Help/*.HxC 163 | DocProject/Help/*.hhc 164 | DocProject/Help/*.hhk 165 | DocProject/Help/*.hhp 166 | DocProject/Help/Html2 167 | DocProject/Help/html 168 | 169 | # Click-Once directory 170 | publish/ 171 | 172 | # Publish Web Output 173 | *.[Pp]ublish.xml 174 | *.azurePubxml 175 | # Note: Comment the next line if you want to checkin your web deploy settings, 176 | # but database connection strings (with potential passwords) will be unencrypted 177 | *.pubxml 178 | *.publishproj 179 | 180 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 181 | # checkin your Azure Web App publish settings, but sensitive information contained 182 | # in these scripts will be unencrypted 183 | PublishScripts/ 184 | 185 | # NuGet Packages 186 | *.nupkg 187 | # NuGet Symbol Packages 188 | *.snupkg 189 | # The packages folder can be ignored because of Package Restore 190 | **/[Pp]ackages/* 191 | # except build/, which is used as an MSBuild target. 192 | !**/[Pp]ackages/build/ 193 | # Uncomment if necessary however generally it will be regenerated when needed 194 | #!**/[Pp]ackages/repositories.config 195 | # NuGet v3's project.json files produces more ignorable files 196 | *.nuget.props 197 | *.nuget.targets 198 | 199 | # Microsoft Azure Build Output 200 | csx/ 201 | *.build.csdef 202 | 203 | # Microsoft Azure Emulator 204 | ecf/ 205 | rcf/ 206 | 207 | # Windows Store app package directories and files 208 | AppPackages/ 209 | BundleArtifacts/ 210 | Package.StoreAssociation.xml 211 | _pkginfo.txt 212 | *.appx 213 | *.appxbundle 214 | *.appxupload 215 | 216 | # Visual Studio cache files 217 | # files ending in .cache can be ignored 218 | *.[Cc]ache 219 | # but keep track of directories ending in .cache 220 | !?*.[Cc]ache/ 221 | 222 | # Others 223 | ClientBin/ 224 | ~$* 225 | *~ 226 | *.dbmdl 227 | *.dbproj.schemaview 228 | *.jfm 229 | *.pfx 230 | *.publishsettings 231 | orleans.codegen.cs 232 | 233 | # Including strong name files can present a security risk 234 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 235 | #*.snk 236 | 237 | # Since there are multiple workflows, uncomment next line to ignore bower_components 238 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 239 | #bower_components/ 240 | 241 | # RIA/Silverlight projects 242 | Generated_Code/ 243 | 244 | # Backup & report files from converting an old project file 245 | # to a newer Visual Studio version. Backup files are not needed, 246 | # because we have git ;-) 247 | _UpgradeReport_Files/ 248 | Backup*/ 249 | UpgradeLog*.XML 250 | UpgradeLog*.htm 251 | ServiceFabricBackup/ 252 | *.rptproj.bak 253 | 254 | # SQL Server files 255 | *.mdf 256 | *.ldf 257 | *.ndf 258 | 259 | # Business Intelligence projects 260 | *.rdl.data 261 | *.bim.layout 262 | *.bim_*.settings 263 | *.rptproj.rsuser 264 | *- [Bb]ackup.rdl 265 | *- [Bb]ackup ([0-9]).rdl 266 | *- [Bb]ackup ([0-9][0-9]).rdl 267 | 268 | # Microsoft Fakes 269 | FakesAssemblies/ 270 | 271 | # GhostDoc plugin setting file 272 | *.GhostDoc.xml 273 | 274 | # Node.js Tools for Visual Studio 275 | .ntvs_analysis.dat 276 | node_modules/ 277 | 278 | # Visual Studio 6 build log 279 | *.plg 280 | 281 | # Visual Studio 6 workspace options file 282 | *.opt 283 | 284 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 285 | *.vbw 286 | 287 | # Visual Studio LightSwitch build output 288 | **/*.HTMLClient/GeneratedArtifacts 289 | **/*.DesktopClient/GeneratedArtifacts 290 | **/*.DesktopClient/ModelManifest.xml 291 | **/*.Server/GeneratedArtifacts 292 | **/*.Server/ModelManifest.xml 293 | _Pvt_Extensions 294 | 295 | # Paket dependency manager 296 | .paket/paket.exe 297 | paket-files/ 298 | 299 | # FAKE - F# Make 300 | .fake/ 301 | 302 | # CodeRush personal settings 303 | .cr/personal 304 | 305 | # Python Tools for Visual Studio (PTVS) 306 | __pycache__/ 307 | *.pyc 308 | 309 | # Cake - Uncomment if you are using it 310 | # tools/** 311 | # !tools/packages.config 312 | 313 | # Tabs Studio 314 | *.tss 315 | 316 | # Telerik's JustMock configuration file 317 | *.jmconfig 318 | 319 | # BizTalk build output 320 | *.btp.cs 321 | *.btm.cs 322 | *.odx.cs 323 | *.xsd.cs 324 | 325 | # OpenCover UI analysis results 326 | OpenCover/ 327 | 328 | # Azure Stream Analytics local run output 329 | ASALocalRun/ 330 | 331 | # MSBuild Binary and Structured Log 332 | *.binlog 333 | 334 | # NVidia Nsight GPU debugger configuration file 335 | *.nvuser 336 | 337 | # MFractors (Xamarin productivity tool) working folder 338 | .mfractor/ 339 | 340 | # Local History for Visual Studio 341 | .localhistory/ 342 | 343 | # BeatPulse healthcheck temp database 344 | healthchecksdb 345 | 346 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 347 | MigrationBackup/ 348 | 349 | # Ionide (cross platform F# VS Code tools) working folder 350 | .ionide/ 351 | --------------------------------------------------------------------------------