├── .gitattributes ├── .github └── workflows │ ├── check-header.yml │ ├── smoke_test.yml │ ├── sonarcloud_scan_app.yml │ └── sonarcloud_scan_service.yml ├── .gitignore ├── BUILD.md ├── CODE-OF-CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── RUN-WITH-DOCKER.md ├── active-learning-service ├── Dockerfile ├── config │ ├── __init__.py │ ├── app_os.py │ ├── config.py │ └── constant.py ├── logs │ ├── main.log │ └── main_debug.log ├── manage.py ├── models │ └── porojectName_model.pkl ├── mysite │ ├── __init__.py │ ├── asgi.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── requirements.txt └── src │ ├── __init__.py │ ├── admin.py │ ├── al │ ├── __init__.py │ ├── activelearning.py │ ├── embeddings.py │ ├── get_vectors.py │ ├── project_service.py │ ├── query_instance.py │ ├── retry_service.py │ ├── separate_data.py │ ├── sr_service.py │ ├── teach_model.py │ ├── train_model.py │ └── vector_sr.py │ ├── apps.py │ ├── aws │ ├── __init__.py │ ├── s3.py │ └── sts.py │ ├── db │ ├── __init__.py │ └── mongo_connect.py │ ├── exceptions │ ├── __init__.py │ └── base_exceptions.py │ ├── migrations │ └── __init__.py │ ├── models.py │ ├── tests.py │ ├── urls.py │ ├── utils │ ├── JSONEncoder.py │ ├── __init__.py │ ├── common.py │ ├── fileSystem.py │ ├── jwtVerification.py │ └── modules.py │ └── views.py ├── annotation-app ├── .browserslistrc ├── .prettierignore ├── .prettierrc ├── Dockerfile ├── angular.json ├── build-script.js ├── docker-entrypoint.sh ├── nginx.conf ├── package-lock.json ├── package.json ├── proxy.conf.json ├── src │ ├── app │ │ ├── app-routing.module.ts │ │ ├── app.component.html │ │ ├── app.component.ts │ │ ├── app.module.ts │ │ ├── component │ │ │ ├── datasets │ │ │ │ ├── create-new-dataset-modal │ │ │ │ │ ├── create-new-dataset-modal.component.html │ │ │ │ │ ├── create-new-dataset-modal.component.scss │ │ │ │ │ └── create-new-dataset-modal.component.ts │ │ │ │ ├── create-new-dataset-page │ │ │ │ │ ├── create-new-dataset-page.component.html │ │ │ │ │ ├── create-new-dataset-page.component.scss │ │ │ │ │ └── create-new-dataset-page.component.ts │ │ │ │ ├── create-new-dataset │ │ │ │ │ ├── create-new-dataset.component.html │ │ │ │ │ ├── create-new-dataset.component.scss │ │ │ │ │ └── create-new-dataset.component.ts │ │ │ │ ├── dataset-analyze │ │ │ │ │ ├── analyze-datagrid │ │ │ │ │ │ ├── analyze-datagrid.component.html │ │ │ │ │ │ ├── analyze-datagrid.component.scss │ │ │ │ │ │ └── analyze-datagrid.component.ts │ │ │ │ │ ├── dataset-analyze.component.html │ │ │ │ │ ├── dataset-analyze.component.scss │ │ │ │ │ └── dataset-analyze.component.ts │ │ │ │ ├── datasets-routing.module.ts │ │ │ │ ├── datasets.module.ts │ │ │ │ ├── dnd.directive.ts │ │ │ │ ├── modal-datagrid │ │ │ │ │ ├── modal-datagrid.component.html │ │ │ │ │ └── modal-datagrid.component.ts │ │ │ │ ├── my-datasets │ │ │ │ │ ├── my-datasets.component.html │ │ │ │ │ ├── my-datasets.component.scss │ │ │ │ │ └── my-datasets.component.ts │ │ │ │ └── upload-file │ │ │ │ │ ├── upload-file.component.html │ │ │ │ │ ├── upload-file.component.scss │ │ │ │ │ └── upload-file.component.ts │ │ │ ├── faq │ │ │ │ ├── faq.component.html │ │ │ │ ├── faq.component.scss │ │ │ │ └── faq.component.ts │ │ │ ├── header │ │ │ │ ├── header.component.html │ │ │ │ ├── header.component.scss │ │ │ │ └── header.component.ts │ │ │ ├── home │ │ │ │ ├── home.component.html │ │ │ │ ├── home.component.scss │ │ │ │ └── home.component.ts │ │ │ ├── login │ │ │ │ ├── basic-login │ │ │ │ │ ├── basic-login.component.html │ │ │ │ │ ├── basic-login.component.scss │ │ │ │ │ └── basic-login.component.ts │ │ │ │ ├── login-routing.module.ts │ │ │ │ ├── login.module.ts │ │ │ │ └── login │ │ │ │ │ ├── login.component.html │ │ │ │ │ ├── login.component.scss │ │ │ │ │ └── login.component.ts │ │ │ ├── page-not-found │ │ │ │ ├── page-not-found.component.html │ │ │ │ ├── page-not-found.component.scss │ │ │ │ └── page-not-found.component.ts │ │ │ ├── permission │ │ │ │ ├── permissions-routing.module.ts │ │ │ │ ├── permissions.module.ts │ │ │ │ └── permissions │ │ │ │ │ ├── permissions.component.html │ │ │ │ │ ├── permissions.component.scss │ │ │ │ │ └── permissions.component.ts │ │ │ ├── projects │ │ │ │ ├── annotate-progress-board │ │ │ │ │ ├── annotate-progress-board.component.html │ │ │ │ │ ├── annotate-progress-board.component.scss │ │ │ │ │ └── annotate-progress-board.component.ts │ │ │ │ ├── create-project │ │ │ │ │ ├── create-project.component.html │ │ │ │ │ ├── create-project.component.scss │ │ │ │ │ └── create-project.component.ts │ │ │ │ ├── download │ │ │ │ │ ├── download.component.html │ │ │ │ │ ├── download.component.scss │ │ │ │ │ └── download.component.ts │ │ │ │ ├── generate │ │ │ │ │ ├── generate.component.html │ │ │ │ │ ├── generate.component.scss │ │ │ │ │ └── generate.component.ts │ │ │ │ ├── labeling-task-list │ │ │ │ │ ├── edit-project │ │ │ │ │ │ ├── edit-project.component.html │ │ │ │ │ │ ├── edit-project.component.scss │ │ │ │ │ │ └── edit-project.component.ts │ │ │ │ │ ├── projects.component.html │ │ │ │ │ ├── projects.component.scss │ │ │ │ │ ├── projects.component.ts │ │ │ │ │ └── task-datagrid │ │ │ │ │ │ ├── task-datagrid.component.html │ │ │ │ │ │ ├── task-datagrid.component.scss │ │ │ │ │ │ └── task-datagrid.component.ts │ │ │ │ ├── project-analyze │ │ │ │ │ ├── append │ │ │ │ │ │ ├── append.component.html │ │ │ │ │ │ ├── append.component.scss │ │ │ │ │ │ └── append.component.ts │ │ │ │ │ ├── latest-annotation-data │ │ │ │ │ │ ├── latest-annotation-data.component.html │ │ │ │ │ │ ├── latest-annotation-data.component.scss │ │ │ │ │ │ └── latest-annotation-data.component.ts │ │ │ │ │ ├── project-analyze.component.html │ │ │ │ │ ├── project-analyze.component.scss │ │ │ │ │ ├── project-analyze.component.ts │ │ │ │ │ └── user-category-d3 │ │ │ │ │ │ ├── user-category-d3.component.html │ │ │ │ │ │ ├── user-category-d3.component.scss │ │ │ │ │ │ └── user-category-d3.component.ts │ │ │ │ ├── projects-routing.module.ts │ │ │ │ ├── projects.module.ts │ │ │ │ └── treeview-modal │ │ │ │ │ ├── treeview-modal.component.html │ │ │ │ │ ├── treeview-modal.component.scss │ │ │ │ │ └── treeview-modal.component.ts │ │ │ └── sidenav │ │ │ │ ├── sidenav.component.html │ │ │ │ ├── sidenav.component.scss │ │ │ │ └── sidenav.component.ts │ │ ├── core.module.ts │ │ ├── guards │ │ │ └── auth.guard.ts │ │ ├── model │ │ │ ├── authentication.ts │ │ │ ├── constant.ts │ │ │ ├── dataset.ts │ │ │ ├── env.ts │ │ │ ├── index.ts │ │ │ ├── sr.ts │ │ │ └── user.ts │ │ ├── pipes │ │ │ ├── full-name.pipe.ts │ │ │ ├── math-round.pipe.ts │ │ │ └── sliceText.pipe.ts │ │ ├── services │ │ │ ├── api.service.ts │ │ │ ├── cds-icon.service.ts │ │ │ ├── common │ │ │ │ ├── common.service.ts │ │ │ │ ├── dom.service.ts │ │ │ │ ├── download.service.ts │ │ │ │ ├── email.service.ts │ │ │ │ ├── markdown-parser.service.ts │ │ │ │ ├── s3.service.ts │ │ │ │ ├── tool.service.ts │ │ │ │ ├── tool.utils.js │ │ │ │ └── up-zip.service.ts │ │ │ ├── environments.service.ts │ │ │ ├── internal-api.service.ts │ │ │ ├── label-studio.service.ts │ │ │ ├── user-auth.service.ts │ │ │ └── web-analytics.service.ts │ │ └── shared │ │ │ ├── clr-filter │ │ │ ├── datagridFilter.component.html │ │ │ ├── datagridFilter.component.scss │ │ │ └── datagridFilter.component.ts │ │ │ ├── form-validators │ │ │ ├── dataset-validator.ts │ │ │ └── form-validator-util.ts │ │ │ ├── modal-confirm │ │ │ ├── modal-confirm.component.html │ │ │ └── modal-confirm.component.ts │ │ │ ├── routeReuseStrategy.ts │ │ │ ├── shared.module.ts │ │ │ └── utils │ │ │ ├── html.ts │ │ │ ├── index.es5.ts │ │ │ └── treeView.ts │ ├── assets │ │ ├── .gitkeep │ │ ├── files │ │ │ ├── taxonomy_sample.json │ │ │ └── taxonomy_sample.yaml │ │ └── images │ │ │ ├── ava-logo.png │ │ │ ├── ava-small.png │ │ │ ├── demo.png │ │ │ ├── favicon.ico │ │ │ ├── polygon.svg │ │ │ ├── rect.svg │ │ │ ├── refresh.svg │ │ │ ├── trash.svg │ │ │ └── vm-logo.png │ ├── dev-platform │ │ ├── dev-platform.module.ts │ │ └── dev-platform │ │ │ ├── dev-platform.component.css │ │ │ ├── dev-platform.component.html │ │ │ └── dev-platform.component.ts │ ├── environments │ │ ├── environment.prod.ts │ │ └── environment.ts │ ├── favicon.ico │ ├── index.html │ ├── libs │ │ ├── categoryChart.js │ │ ├── hierarchicalChart.js │ │ ├── modelChart.js │ │ └── userChart.js │ ├── main.ts │ ├── micro-frontend │ │ ├── entry.component.ts │ │ └── micro-frontend.module.ts │ ├── polyfills.ts │ └── styles.scss ├── tsconfig.app.json ├── tsconfig.json └── tslint.json ├── annotation-service ├── Dockerfile ├── config │ ├── app-os.js │ ├── code_msg.js │ ├── config.js │ └── constant.js ├── db │ ├── db-connect.js │ └── mongo.db.js ├── middlewares │ └── jwt.middleware.js ├── package-lock.json ├── package.json ├── resources │ ├── APIs.js │ ├── template-annotator.html │ ├── template-generater.html │ ├── template-notFinish.html │ ├── template-notStart.html │ └── template-owner.html ├── routers │ ├── auth-controller.js │ ├── community-controller.js │ ├── db-controller.js │ ├── email-controller.js │ ├── file-controller.js │ ├── integration-controller.js │ ├── project-controller.js │ ├── scripts-controller.js │ ├── slack-controller.js │ ├── srs-controller.js │ └── user-controller.js ├── scripts │ └── reveiwInfoMigration.js ├── server.js ├── services │ ├── activelearning.service.js │ ├── auth.service.js │ ├── authForNoe.service.js │ ├── community.service.js │ ├── dataSet-service.js │ ├── email-service.js │ ├── file-service.js │ ├── integration.service.js │ ├── localFileSys.service.js │ ├── project.service.js │ ├── s3.service.js │ ├── slack │ │ ├── slack.js │ │ ├── slackAnnotateModal.service.js │ │ ├── slackBuildAppHome.service.js │ │ ├── slackChat.service.js │ │ ├── slackConversations.service.js │ │ ├── slackUsers.service.js │ │ └── utils │ │ │ ├── accessToken.service.js │ │ │ └── slack.utils.js │ ├── sqs.service.js │ ├── srs-service.js │ ├── supercollider.service.js │ └── user-service.js ├── swagger2.json └── utils │ ├── DB.OPERATIONS.js │ ├── ImportDataset.util.js │ ├── common.utils.js │ ├── fileSystem.utils.js │ ├── imgImporter.js │ ├── logImporter.js │ ├── mongoModel.utils.js │ ├── s3.js │ ├── sqs.js │ ├── srsImporter.js │ ├── sts.js │ ├── taskSchedule.js │ └── validator.js ├── docker-compose-host-db.yml ├── docker-compose.yml └── docs ├── AWS-step-by-step-config-with-chart.pdf ├── manifest.yml └── tutorial ├── annotator-screen.png ├── imdb-500.csv └── project-screen.png /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior, 2 | # in case people don't have core.autocrlf set. 3 | * text=auto 4 | # Declares that files will always have CRLF line ends 5 | *.sh text eol=lf 6 | docker*.yml text eol=lf 7 | *Dockerfile text eol=lf 8 | *nginx.conf text eol=lf -------------------------------------------------------------------------------- /.github/workflows/check-header.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019-2024 VMware, Inc. 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | name: check-header 7 | 8 | on: 9 | push: 10 | branches: ["**"] 11 | pull_request: 12 | branches: ['**'] 13 | 14 | jobs: 15 | check-header: 16 | runs-on: ubuntu-latest # runs a test on Ubuntu 17 | 18 | env: 19 | GITHUB_CONTEXT: ${{ toJson(github) }} 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v3 24 | with: 25 | fetch-depth: 3 26 | 27 | - run: wget https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/devops/check_headers.py 28 | 29 | - name: Check Header 30 | run: | 31 | chmod +x ./check_headers.py 32 | python ./check_headers.py -f "$(git diff --name-only --diff-filter=d $GITHUB_CONTEXT)" 33 | -------------------------------------------------------------------------------- /.github/workflows/sonarcloud_scan_app.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019-2024 VMware, Inc. 3 | # SPDX-License-Identifier: Apache-2.0 4 | # 5 | 6 | name: sonarcloud_scan_app 7 | 8 | on: 9 | push: 10 | branches: ["**"] 11 | pull_request: 12 | branches: ['**'] 13 | 14 | jobs: 15 | #sonar cloud scan job for annotation app 16 | scan_annotation_app: 17 | runs-on: ubuntu-latest 18 | 19 | #define variables to be used for SonarCloud scan 20 | env: 21 | SONAR_ORG: ${{ secrets.SONAR_ORG }} 22 | 23 | steps: 24 | 25 | - name: Checkout 26 | uses: actions/checkout@v3 27 | with: 28 | fetch-depth: 0 29 | 30 | - name: Checkout devops branch 31 | uses: actions/checkout@v3 32 | with: 33 | fetch-depth: 0 34 | ref: 'devops' 35 | path: 'devops' 36 | 37 | - name: SonarCloud setup for annotation-app 38 | working-directory: ./devops/sonar 39 | run: python ./config_sonar_project.py -ProjectName ${{ env.SONAR_ORG }}-annotation-app -ProjectKey ${{ env.SONAR_ORG }}-annotation-app -OrgKey ${{ env.SONAR_ORG }} -QualityGateName angular-client-gate -QualityGateConditions ./quality_gates/angular-client.json -SonarToken ${{ secrets.SONAR_TOKEN }} 40 | 41 | - name: SonarCloud Scan annotation-app 42 | uses: sonarsource/sonarcloud-github-action@master 43 | with: 44 | projectBaseDir: ./annotation-app 45 | args: > 46 | -Dsonar.organization=${{ env.SONAR_ORG }} 47 | -Dsonar.projectKey=${{ env.SONAR_ORG }}-annotation-app 48 | -Dsonar.test.exclusions=**/node_modules/**/*,*.md,*.txt,*.yml 49 | -Dsonar.coverage.exclusions=** 50 | #Set below secrets in your Github Actions secrets 51 | env: 52 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 53 | GITHUB_TOKEN: ${{ secrets.DATA_GITHUB_TOKEN }} 54 | 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store 3 | 4 | # annotation-app 5 | annotation-app/.angular 6 | annotation-app/node_modules 7 | annotation-app/.npmrc 8 | annotation-app/chart 9 | annotation-app/src/environments/environment.stg.ts 10 | annotation-app/src/environments/environment.test.ts 11 | # annotation-app/package-lock.json 12 | annotation-app/tsconfig.spec.json 13 | annotation-app/dist 14 | 15 | 16 | # annotation-service 17 | annotation-service/node_modules 18 | annotation-service/FILE_SYS 19 | annotation-service/config/app-local.js 20 | annotation-service/config/app-poc.js 21 | 22 | 23 | # active-learning-service 24 | *__pycache__* 25 | active-learning-service/venv/ 26 | active-learning-service/.venv/ 27 | active-learning-service/.python-version 28 | active-learning-service/.idea 29 | active-learning-service/logs 30 | active-learning-service/models 31 | active-learning-service/config/app_local.py 32 | active-learning-service/config/app_poc.py -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Data Annotator for Machine Learning 2 | Copyright 2021 VMware, Inc. 3 | 4 | This product is licensed to you under the Apache 2.0 license (the "License"). You may not use this product except in compliance with the Apache 2.0 License. 5 | 6 | This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 7 | 8 | -------------------------------------------------------------------------------- /RUN-WITH-DOCKER.md: -------------------------------------------------------------------------------- 1 | # Data Annotator for Machine Learning (DAML) run with docker 2 | 3 | 4 | ## Tools used 5 | Please go to [https://docs.docker.com/get-docker/](https://docs.docker.com/get-docker/) to download the proper docker version for your machine. 6 | - If your machine is **Windows / Mac** you can only use docker desktop, to run the application. 7 | - If your machine is **Linux** you also need to download docker-compose. [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/) 8 | 9 | ## Start/End application 10 | In active-learning-service we use _en_core_web_md_ as the default spacy module if you want to replace it. you can directly replace it in the docker-compose file. eg. _- SPACY_MODEL=zh_core_web_md_ for spacy models ref: [https://spacy.io/models](https://spacy.io/models) 11 | 12 | - If your machine already installed MongoDB, and you want to use that as the database. you need to download the [docker-compose-host-db.yml](docker-compose-host-db.yml) file and use it to run the DAML application. 13 | 14 | Start the containers: 15 | ```bash 16 | docker compose -f docker-compose-host-db.yml up 17 | ``` 18 | End the containers: 19 | ```bash 20 | docker compose -f docker-compose-host-db.yml down 21 | ``` 22 | 23 | - Else you need to download the [docker-compose.yml](docker-compose.yml) file and use it to run the DAML application. 24 | 25 | Start the containers: 26 | ```bash 27 | docker compose -f docker-compose.yml up 28 | ``` 29 | End the containers: 30 | ```bash 31 | docker compose -f docker-compose.yml down 32 | ``` 33 | 34 | ## How to use 35 | Open `http://localhost:4200` with your browser, now you can use full of the DAML application functions. 36 | 37 | ## More commands 38 | For more commands you can reference here [docker compose commands](https://docs.docker.com/engine/reference/commandline/compose/#child-commands). 39 | 40 | ## DAML application docker images 41 | [https://hub.docker.com/search?q=vmware-daml-&type=image](https://hub.docker.com/search?q=vmware-daml-&type=image) -------------------------------------------------------------------------------- /active-learning-service/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | FROM python:3.7.10 5 | 6 | WORKDIR /app 7 | 8 | COPY requirements.txt ./ 9 | RUN pip install --no-cache-dir -r requirements.txt 10 | RUN python -m spacy download en_core_web_md 11 | 12 | COPY . /app 13 | 14 | EXPOSE 8000 15 | 16 | CMD [ "python", "manage.py","runserver","0.0.0.0:8000" ] -------------------------------------------------------------------------------- /active-learning-service/config/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/config/app_os.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | app = { 6 | 7 | "MONGODB_URL": os.getenv("MONGODB_URL", "mongodb://localhost:27017/daml"), 8 | "MONGODB_COLLECTION": os.getenv("MONGODB_COLLECTION", "daml"), 9 | #If True will save datasets to local, If set USE_AWS=true set it to false 10 | "USE_LOCAL_FILE_SYS": os.getenv("USE_LOCAL_FILE_SYS", True), 11 | 12 | #If True, will save datasets to S3, shoud set USE_LOCAL_FILE_SYS=False 13 | "USE_AWS": os.getenv("USE_AWS", False), 14 | # AWS IAM 15 | "REGION": os.getenv("REGION", None), 16 | "AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID", None), 17 | "AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY", None), 18 | # AWS S3 19 | "S3_BUCKET_NAME": os.getenv("S3_BUCKET_NAME", None), 20 | "S3_ROLE_ARN": os.getenv("S3_ROLE_ARN", None), 21 | 22 | # spacy model default is en_core_web_md, NOTE: [sm: no word vectors] more: https://spacy.io/models 23 | "SPACY_MODEL": os.getenv("SPACY_MODEL", "en_core_web_md"), 24 | 25 | # optional token config 26 | "TOKEN_ALGORITHM": os.getenv("TOKEN_ALGORITHM", "HS256"), 27 | # !!! important 121ba6ff-64d6-4c1a-b6ef-dd6b95433064 just a random value get the value from environment or replace by yourslef 28 | # generate the key yourself. should keep the same with annotation-service TOKEN_SECRET_OR_PRIVATE_KEY 29 | "TOKEN_SECRET_OR_PRIVATE_KEY": os.getenv("TOKEN_SECRET_OR_PRIVATE_KEY", "121ba6ff-64d6-4c1a-b6ef-dd6b95433064"), 30 | # 01a406f3-e5f4-45b8-a722-cff17e9d8cf9 just a random value 31 | # generate the key yourself. django SECRET_KEY a random generated value https://docs.djangoproject.com/en/3.2/ref/settings/#std:setting-SECRET_KEY 32 | "DJANGO_SECRET_KEY": os.getenv("DJANGO_SECRET_KEY", "01a406f3-e5f4-45b8-a722-cff17e9d8cf9"), 33 | 34 | } -------------------------------------------------------------------------------- /active-learning-service/config/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import os 5 | import importlib 6 | import logging 7 | 8 | # logging 9 | log = logging.getLogger('loop_al') 10 | 11 | # import modules 12 | env = os.getenv('SYS_ENV', 'os') 13 | path_file = f"config.app_{env}" 14 | app = importlib.import_module(path_file) 15 | 16 | config = app.app 17 | config["ENV"] = env 18 | log.info(f'[ ENV ] = {env}') 19 | log.info(f'[ CONFIG ] = {config}') 20 | 21 | -------------------------------------------------------------------------------- /active-learning-service/config/constant.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | cst={ 5 | "QUERY_STRATEGY": { 6 | "POOL_BASED_SAMPLING": { 7 | "PB_UNS": "PB_UNS", 8 | "PB_MS": "PB_MS", 9 | "PB_ES": "PB_ES", 10 | }, 11 | "RANKED_BATCH_MODE": { 12 | "RBM_UNBS": "RBM_UNBS", 13 | }, 14 | }, 15 | "ESTIMATOR": { 16 | "KNC": "KNC", 17 | "GBC": "GBC", 18 | "RFC": "RFC", 19 | }, 20 | } -------------------------------------------------------------------------------- /active-learning-service/logs/main.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/active-learning-service/logs/main.log -------------------------------------------------------------------------------- /active-learning-service/logs/main_debug.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/active-learning-service/logs/main_debug.log -------------------------------------------------------------------------------- /active-learning-service/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019-2021 VMware, Inc. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | """Django's command-line utility for administrative tasks.""" 8 | import os 9 | import sys 10 | from src.utils.modules import download_spacy_module 11 | 12 | 13 | def main(): 14 | """Run administrative tasks.""" 15 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') 16 | try: 17 | from django.core.management import execute_from_command_line 18 | except ImportError as exc: 19 | raise ImportError( 20 | "Couldn't import Django. Are you sure it's installed and " 21 | "available on your PYTHONPATH environment variable? Did you " 22 | "forget to activate a virtual environment?" 23 | ) from exc 24 | 25 | download_spacy_module() 26 | execute_from_command_line(sys.argv) 27 | 28 | if __name__ == '__main__': 29 | main() 30 | -------------------------------------------------------------------------------- /active-learning-service/models/porojectName_model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/active-learning-service/models/porojectName_model.pkl -------------------------------------------------------------------------------- /active-learning-service/mysite/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/mysite/asgi.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | """ 5 | ASGI config for mysite project. 6 | 7 | It exposes the ASGI callable as a module-level variable named ``application``. 8 | 9 | For more information on this file, see 10 | https://docs.djangoproject.com/en/dev/howto/deployment/asgi/ 11 | """ 12 | 13 | import os 14 | 15 | from django.core.asgi import get_asgi_application 16 | 17 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') 18 | 19 | application = get_asgi_application() 20 | -------------------------------------------------------------------------------- /active-learning-service/mysite/urls.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | """mysite URL Configuration 5 | 6 | The `urlpatterns` list routes URLs to views. For more information please see: 7 | https://docs.djangoproject.com/en/dev/topics/http/urls/ 8 | Examples: 9 | Function views 10 | 1. Add an import: from my_app import views 11 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 12 | Class-based views 13 | 1. Add an import: from other_app.views import Home 14 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 15 | Including another URLconf 16 | 1. Import the include() function: from django.urls import include, path 17 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 18 | """ 19 | from django.contrib import admin 20 | from django.urls import path,include 21 | from src import views 22 | 23 | urlpatterns = [ 24 | path('api/poc', views.api_poc), 25 | path('api/al/sr/vector', views.al_srs_vector), 26 | path('api/al/model/train', views.al_train_model), 27 | path('api/al/model/query', views.al_query_instance), 28 | path('api/al/model/teach', views.al_teach_model), 29 | path('health', views.health, name='health'), 30 | path('', views.health, name='health'), 31 | 32 | ] 33 | -------------------------------------------------------------------------------- /active-learning-service/mysite/wsgi.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | """ 5 | WSGI config for mysite project. 6 | 7 | It exposes the WSGI callable as a module-level variable named ``application``. 8 | 9 | For more information on this file, see 10 | https://docs.djangoproject.com/en/dev/howto/deployment/wsgi/ 11 | """ 12 | 13 | import os 14 | 15 | from django.core.wsgi import get_wsgi_application 16 | 17 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') 18 | 19 | application = get_wsgi_application() 20 | -------------------------------------------------------------------------------- /active-learning-service/requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo~=3.11.3 2 | mongoengine~=0.23.0 3 | Django~=3.2 4 | 5 | numpy~=1.19.5 6 | pandas~=1.0.3 7 | scikit-learn~=0.24.2 8 | modAL-python~=0.3.6 9 | PyYAML~=5.4.1 10 | boto3~=1.14.7 11 | botocore~=1.17.7 12 | requests~=2.24.0 13 | fastai~=1.0.61 14 | spacy~=3.0.6 15 | 16 | PyJWT~=2.0.1 17 | cryptography~=3.4.7 -------------------------------------------------------------------------------- /active-learning-service/src/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/src/admin.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from django.contrib import admin 5 | 6 | # Register your models here. 7 | -------------------------------------------------------------------------------- /active-learning-service/src/al/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/src/al/get_vectors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import time 6 | 7 | from config.config import config 8 | import numpy as np 9 | import spacy 10 | 11 | from src.exceptions.base_exceptions import NetWorkException 12 | 13 | nlp = spacy.load(config['SPACY_MODEL']) 14 | log = logging.getLogger('loop_al') 15 | 16 | 17 | def os_text_vectors(sr_text): 18 | 19 | vectors = [] 20 | for text in sr_text: 21 | average = [] 22 | for token in nlp(text): 23 | average.append(nlp.vocab[token.text].vector) 24 | vectors.append(np.mean(average, axis=0)) 25 | return vectors 26 | 27 | 28 | def request_text_vectors(sr_text): 29 | star_time = time.time() 30 | vectors = os_text_vectors(sr_text) 31 | log.info(f'Response Time(in secs): {int(time.time() - star_time)}') 32 | return vectors 33 | -------------------------------------------------------------------------------- /active-learning-service/src/al/project_service.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from src.db.mongo_connect import mongo_client 5 | 6 | 7 | def pro_collection(): 8 | db = mongo_client() 9 | return db["projects"] 10 | 11 | 12 | def find_project_by_name(project_name): 13 | projects = pro_collection() 14 | return projects.find({"projectName": project_name}) 15 | 16 | 17 | def update_project(conditions, update): 18 | projects = pro_collection() 19 | projects.find_one_and_update(conditions, update) 20 | 21 | 22 | def update_end_condition(sr_ids, project_name): 23 | if len(sr_ids) < 10: 24 | # add a stop condition 25 | conditions = {"projectName": project_name} 26 | update = {"$set": {"al.alFailed": True}} 27 | update_project(conditions, update) 28 | return True 29 | -------------------------------------------------------------------------------- /active-learning-service/src/al/query_instance.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import os 6 | import time 7 | 8 | import numpy as np 9 | import pickle 10 | from config.constant import cst 11 | 12 | log = logging.getLogger('loop_al') 13 | 14 | 15 | def query_instance(df_pool, model_name, n_query, query_strategy): 16 | log.info(f'query instance') 17 | start_time = time.time() 18 | 19 | if model_name == None: 20 | print("No dataset file specified!") 21 | return 22 | elif not os.path.exists(model_name): 23 | print("Source file does not exist:", model_name) 24 | return 25 | 26 | # load the model from disk 27 | with open(model_name, 'rb') as model_al: 28 | learner = pickle.load(model_al) 29 | x_pool = np.array(df_pool['sr_text']) 30 | y_pool = np.array(df_pool['ids']) 31 | 32 | query_result = [] 33 | if query_strategy == cst['QUERY_STRATEGY']['RANKED_BATCH_MODE']['RBM_UNBS']: 34 | query_idx, vector = learner.query(x_pool) 35 | sr_id = y_pool[query_idx] 36 | query_result = list(sr_id) 37 | else: 38 | for i in range(n_query): 39 | query_idx, vector = learner.query(x_pool) 40 | sr_id = y_pool[query_idx] 41 | query_result.append(sr_id[0]) 42 | 43 | x_pool = np.delete(x_pool, query_idx, axis=0) 44 | y_pool = np.delete(y_pool, query_idx, axis=0) 45 | 46 | log.info(f'query instance end (in secs): {int(time.time() - start_time)}') 47 | return query_result 48 | 49 | -------------------------------------------------------------------------------- /active-learning-service/src/al/retry_service.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import json 5 | 6 | from src.al.project_service import update_project 7 | 8 | 9 | def support_al_retry(request, function_name): 10 | 11 | if request.method == "POST" and request.body: 12 | req = json.loads(request.body) 13 | else: 14 | return 15 | 16 | if function_name == "active_learning_train": 17 | update = {"$set": {"al.training": False}} 18 | elif function_name == "active_learning_query": 19 | update = {"$set": {"al.querying": False}} 20 | elif function_name == "active_learning_teach": 21 | update = {"$set": {"al.teaching": False}} 22 | else: 23 | return 24 | 25 | conditions = {"projectName": req['projectName']} 26 | update_project(conditions, update) -------------------------------------------------------------------------------- /active-learning-service/src/al/separate_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import pickle 5 | import random 6 | import numpy as np 7 | import pandas as pd 8 | 9 | 10 | # common function to separate sr data 80% for train and 20% for test 11 | def separate_data(request): 12 | 13 | sr_ids = request['ids'] 14 | 15 | id_test, x_test, y_test, x_teach, y_teach = [], [], [], [], [] 16 | sr_len = int(len(sr_ids) * 0.8) 17 | 18 | sr_selected = random.sample(range(0, len(sr_ids)), sr_len) 19 | 20 | for index, sr_id in enumerate(sr_ids): 21 | if index in sr_selected: 22 | # x_teach.append(request['sr_vectors'][index]) 23 | x_teach.append(request['sr_text'][index]) 24 | y_teach.append(request['labels'][index]) 25 | else: 26 | id_test.append(sr_id) 27 | # x_test.append(request['sr_vectors'][index]) 28 | x_test.append(request['sr_text'][index]) 29 | y_test.append(request['labels'][index]) 30 | 31 | return { 32 | "x_teach": np.array(x_teach), 33 | "y_teach": np.array(y_teach), 34 | "x_test": np.array(x_test), 35 | "y_test": np.array(y_test), 36 | "id_test": id_test, 37 | "sr_len": sr_len 38 | } 39 | 40 | 41 | # load vector model return sr vectors 42 | def vector_sr(sr_array, model_file, project_type, obj_col, num_col): 43 | # use for tabular one-hot-encodding 44 | if project_type == 'tabular': 45 | num_sr, obj_sr = [], [] 46 | if num_col: 47 | num_sr = pd.DataFrame(sr_array, columns=num_col).replace("", 0).replace(np.nan, 0) 48 | if not obj_col: 49 | sr_array = np.array(num_sr) 50 | if obj_col: 51 | obj_sr = pd.DataFrame(sr_array, columns=obj_col, dtype=np.str) 52 | with open(model_file, 'rb') as model_vector: 53 | vaporizer = pickle.load(model_vector) 54 | obj_sr = vaporizer.transform(obj_sr).toarray() 55 | if not num_col: 56 | sr_array = np.array(obj_sr) 57 | if num_col and obj_col: 58 | # combine number column and categorical column vector together 59 | sr_array = np.concatenate((obj_sr, num_sr), axis=1) 60 | else: 61 | # text project not used now 62 | with open(model_file, 'rb') as model_vector: 63 | vaporizer = pickle.load(model_vector) 64 | sr_array = vaporizer.transform(sr_array).toarray() 65 | return sr_array 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /active-learning-service/src/al/teach_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import os 6 | import time 7 | 8 | import numpy as np 9 | import pickle 10 | 11 | log = logging.getLogger('loop_al') 12 | 13 | 14 | def teach_model(model_name, data, test_sr): 15 | log.info(f'teach model') 16 | start_time = time.time() 17 | 18 | if model_name == None: 19 | print("No dataset file specified!") 20 | return 21 | elif not os.path.exists(model_name): 22 | print("Source file does not exist:", model_name) 23 | return 24 | 25 | test_sr_x = test_sr['sr_text'] 26 | sr_len = data['sr_len'] 27 | x_test = np.concatenate((data['x_test'], test_sr_x), axis=0) 28 | y_test = np.concatenate((data['y_test'], test_sr['labels']), axis=0) 29 | 30 | with open(model_name, 'rb') as model_al: 31 | learner = pickle.load(model_al) 32 | 33 | # we can tech several records once 34 | x_row = data['x_teach'] 35 | y_row = data['y_teach'] 36 | learner.teach(X=x_row, y=y_row) 37 | accuracy = learner.score(x_test, y_test) 38 | log.info(f'Accuracy after teach with {sr_len} tickets {accuracy}') 39 | 40 | with open(model_name, 'wb') as model_al: 41 | pickle.dump(learner, model_al) 42 | 43 | log.info(f'teach model end (in secs): {int(time.time() - start_time)}') 44 | return {"accuracy": accuracy, "test": data['id_test']} 45 | 46 | 47 | -------------------------------------------------------------------------------- /active-learning-service/src/al/train_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import time 6 | import numpy as np 7 | import pickle 8 | 9 | from config.constant import cst 10 | from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier 11 | from sklearn.feature_extraction.text import TfidfVectorizer 12 | from sklearn.neighbors import KNeighborsClassifier 13 | from modAL.models import ActiveLearner 14 | from functools import partial 15 | from modAL.batch import uncertainty_batch_sampling 16 | from modAL.uncertainty import uncertainty_sampling, margin_sampling, entropy_sampling 17 | 18 | modelDir = "models/" 19 | log = logging.getLogger('loop_al') 20 | 21 | 22 | def train_model(data, project_id, estimator, query_strategy): 23 | log.info(f'train model') 24 | start_time = time.time() 25 | 26 | x_test = data['x_test'] 27 | y_test = data['y_test'] 28 | 29 | # initial model train model 30 | if estimator == cst['ESTIMATOR']['KNC']: 31 | estimator = KNeighborsClassifier() 32 | elif estimator == cst['ESTIMATOR']['GBC']: 33 | estimator = GradientBoostingClassifier() 34 | elif estimator == cst['ESTIMATOR']['RFC']: 35 | estimator = RandomForestClassifier() 36 | 37 | if query_strategy == cst['QUERY_STRATEGY']['POOL_BASED_SAMPLING']['PB_UNS']: 38 | query_strategy = uncertainty_sampling 39 | elif query_strategy == cst['QUERY_STRATEGY']['POOL_BASED_SAMPLING']['PB_MS']: 40 | query_strategy = margin_sampling 41 | elif query_strategy == cst['QUERY_STRATEGY']['POOL_BASED_SAMPLING']['PB_ES']: 42 | query_strategy = entropy_sampling 43 | elif query_strategy == cst['QUERY_STRATEGY']['RANKED_BATCH_MODE']['RBM_UNBS']: 44 | batch_size = 10 45 | query_strategy = partial(uncertainty_batch_sampling, n_instances=batch_size) 46 | 47 | learner = ActiveLearner(estimator=estimator, query_strategy=query_strategy, X_training=data['x_teach'], y_training=data['y_teach']) 48 | 49 | # save the model to disk 50 | model_name = project_id + "_model.pkl" 51 | with open('./' + modelDir + model_name, 'wb') as knn_pickle: 52 | pickle.dump(learner, knn_pickle) 53 | 54 | initial_accuracy = learner.score(x_test, y_test) 55 | 56 | log.info(f'Initial accuracy: { initial_accuracy } train model end (in secs): {int(time.time() - start_time)}') 57 | return {"accuracy": initial_accuracy, "model": model_name, "test": data['id_test']} 58 | -------------------------------------------------------------------------------- /active-learning-service/src/apps.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from django.apps import AppConfig 5 | 6 | 7 | class SrcConfig(AppConfig): 8 | name = 'src' 9 | -------------------------------------------------------------------------------- /active-learning-service/src/aws/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/src/aws/s3.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import logging 5 | import boto3 6 | import botocore 7 | from botocore.exceptions import ClientError 8 | from .sts import aws_credentials 9 | from config.config import config 10 | 11 | 12 | def s3_client(token): 13 | cred = aws_credentials(token) 14 | 15 | return boto3.client( 16 | 's3', 17 | aws_access_key_id=cred['accessKeyId'], 18 | aws_secret_access_key=cred['secretAccessKey'], 19 | aws_session_token=cred['sessionToken'] 20 | ) 21 | 22 | 23 | def upload_file_to_s3(file_name, object_name, token): 24 | if not check_aws_config(): 25 | return False 26 | try: 27 | s3_client(token).upload_file(file_name, config["S3_BUCKET_NAME"], object_name) 28 | except ClientError as e: 29 | logging.error(e) 30 | return False 31 | return True 32 | 33 | 34 | def download_file_from_s3(key, local_name, token): 35 | if not check_aws_config(): 36 | return False 37 | try: 38 | s3_client(token).download_file(config["S3_BUCKET_NAME"], key, local_name) 39 | except botocore.exceptions.ClientError as e: 40 | if e.response['Error']['Code'] == "404": 41 | print("The object does not exist.") 42 | else: 43 | logging.error(e) 44 | return False 45 | return True 46 | 47 | 48 | def check_aws_config(): 49 | aws_config = False 50 | if "ESP" in config and config["ESP"] or config["USE_AWS"]: 51 | aws_config = True 52 | elif config["USE_LOCAL_FILE_SYS"]: 53 | aws_config = False 54 | return aws_config 55 | 56 | -------------------------------------------------------------------------------- /active-learning-service/src/aws/sts.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | 5 | import boto3 6 | import requests 7 | import json 8 | 9 | from config.config import config 10 | 11 | 12 | def sts_role(arn, session_name): 13 | 14 | sts = boto3.client( 15 | 'sts', 16 | region_name=config["REGION"], 17 | aws_access_key_id=config["AWS_ACCESS_KEY_ID"], 18 | aws_secret_access_key=config["AWS_SECRET_ACCESS_KEY"] 19 | ) 20 | 21 | data = sts.assume_role( 22 | DurationSeconds=1800, 23 | RoleArn=arn, 24 | RoleSessionName=session_name 25 | ) 26 | data = data['Credentials'] 27 | 28 | return { 29 | 'accessKeyId': data['AccessKeyId'], 30 | 'secretAccessKey': data['SecretAccessKey'], 31 | 'sessionToken': data['SessionToken'] 32 | } 33 | 34 | 35 | def esp_sts_credentials(token): 36 | data = { 37 | "iamRoleArn": config["S3_ROLE_ARN"], 38 | "externalId": config["S3_EXTERNAL"], 39 | "region": config["REGION"], 40 | "durationInSeconds": 1800 41 | } 42 | headers = {"Authorization": token} 43 | 44 | credentials = requests.get(config["ESP_AUTHORIZE_URL"], params=data, headers=headers) 45 | credentials = json.loads(credentials.content) 46 | 47 | return credentials 48 | 49 | 50 | def aws_credentials(token): 51 | if "ESP" in config and config["ESP"]: 52 | credentials = esp_sts_credentials(token) 53 | else: 54 | credentials = sts_role(config["S3_ROLE_ARN"], 'loop_al') 55 | 56 | return credentials 57 | -------------------------------------------------------------------------------- /active-learning-service/src/db/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/src/db/mongo_connect.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import pymongo 5 | 6 | from config.config import config 7 | 8 | 9 | def mongo_client(): 10 | client = pymongo.MongoClient(config["MONGODB_URL"]) 11 | db = client[config["MONGODB_COLLECTION"]] 12 | return db 13 | -------------------------------------------------------------------------------- /active-learning-service/src/exceptions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/active-learning-service/src/exceptions/__init__.py -------------------------------------------------------------------------------- /active-learning-service/src/exceptions/base_exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | 5 | class NetWorkException(Exception): 6 | def __init__(self, code, message): 7 | self.code = code 8 | self.message = message 9 | 10 | 11 | class AuthException(Exception): 12 | def __init__(self, code, message): 13 | self.code = code 14 | self.message = message -------------------------------------------------------------------------------- /active-learning-service/src/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/src/models.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import mongoengine 5 | 6 | 7 | class Srs(mongoengine.Document): 8 | id = mongoengine.ObjectIdField(required=True) 9 | problemCategory=mongoengine.StringField(required=True) 10 | projectName=mongoengine.StringField(required=True) 11 | userInputsLength=mongoengine.StringField(required=True) 12 | -------------------------------------------------------------------------------- /active-learning-service/src/tests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Create your tests here. 5 | from config.config import config 6 | 7 | 8 | def api_poc(req): 9 | return config 10 | -------------------------------------------------------------------------------- /active-learning-service/src/urls.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | from django.urls import path 5 | from . import views 6 | 7 | -------------------------------------------------------------------------------- /active-learning-service/src/utils/JSONEncoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import json 5 | from bson import ObjectId 6 | import datetime 7 | 8 | 9 | # to handle the objectID and datetime 10 | class JSONEncoder(json.JSONEncoder): 11 | def default(self, o): 12 | if isinstance(o, ObjectId): 13 | return str(o) 14 | if isinstance(o, datetime.datetime): 15 | return o.strftime('%Y-%m-%d %H:%M:%S') 16 | return json.JSONEncoder.default(self, o) 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /active-learning-service/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | -------------------------------------------------------------------------------- /active-learning-service/src/utils/common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import json 5 | import logging 6 | import time 7 | import traceback 8 | 9 | from django.http import JsonResponse 10 | 11 | from src.al.retry_service import support_al_retry 12 | from src.exceptions.base_exceptions import AuthException 13 | from src.utils.jwtVerification import verification_token 14 | 15 | from config.config import config 16 | 17 | log = logging.getLogger('loop_al') 18 | log.info(f'[ config ]={config}') 19 | 20 | # common function to handle request 21 | def request_handle(request, fun): 22 | start_time = time.time() 23 | log.info(f'[ {request.method}{request.path} ] request') 24 | 25 | try: 26 | # validate user and token 27 | verification_token(request) 28 | # business logic function 29 | res = fun(request) 30 | 31 | except AuthException as e: 32 | JsonResponse.status_code = 401 33 | res = {"status": "ERROR", "data": str(e)} 34 | log.error(f'{e}, {traceback.format_exc()}') 35 | except Exception as e: 36 | # support active learning retry 37 | support_al_retry(request, fun.__name__) 38 | JsonResponse.status_code = 500 39 | res = {"status": "ERROR", "data": str(e)} 40 | log.error(f'{e}, {traceback.format_exc()}') 41 | 42 | log.info(f'[ {request.method}{request.path} ] response time (in secs): {int(time.time() - start_time)}') 43 | return JsonResponse(res) -------------------------------------------------------------------------------- /active-learning-service/src/utils/fileSystem.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2019-2021 VMware, Inc. 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | import logging 6 | import os 7 | import src.aws.s3 as S3 8 | 9 | 10 | log = logging.getLogger('loop_al') 11 | 12 | 13 | def upload_file(remote_file, local_file, token): 14 | file_path = remote_file 15 | if not os.path.exists(local_file): 16 | log.error(f"LOCAL FILE NOT EXIST {local_file}") 17 | raise Exception(500, f"LOCAL FILE NOT EXIST {local_file}") 18 | 19 | if not S3.check_aws_config(): 20 | file_path = local_file 21 | else: 22 | S3.upload_file_to_s3(local_file, remote_file, token) 23 | 24 | return file_path 25 | 26 | 27 | def download_file(condition, remote_file, local_file, token): 28 | if not S3.check_aws_config(): 29 | if not os.path.exists(local_file): 30 | log.error(f"LOCAL FILE NOT EXIST {local_file}") 31 | raise Exception(500, f"LOCAL FILE NOT EXIST {local_file}") 32 | elif condition: 33 | if not os.path.exists(local_file): 34 | S3.download_file_from_s3(remote_file, local_file, token) 35 | else: 36 | S3.download_file_from_s3(remote_file, local_file, token) 37 | -------------------------------------------------------------------------------- /active-learning-service/src/utils/jwtVerification.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | import jwt 5 | from config.config import config 6 | import json 7 | import requests 8 | import logging 9 | 10 | from src.exceptions.base_exceptions import NetWorkException, AuthException 11 | 12 | log = logging.getLogger('loop_al') 13 | public_key = None 14 | 15 | 16 | def verification_token(request): 17 | 18 | if "ESP" in config and config["ESP"]: 19 | pk = obtain_public_key() 20 | key = bytes(pk['key'].replace("RSA ", ""), encoding="utf8") 21 | algorithms = pk['alg'] 22 | else: 23 | key = config["TOKEN_SECRET_OR_PRIVATE_KEY"] 24 | algorithms = config["TOKEN_ALGORITHM"] 25 | 26 | if "Authorization" not in request.headers: 27 | raise AuthException(401, f"missing token in header['Authorization']") 28 | 29 | token = request.headers["Authorization"] 30 | 31 | try: 32 | decode = jwt.decode(token, key, algorithms=[algorithms]) 33 | except Exception as e: 34 | raise AuthException(401, f"invalid token: {e}") 35 | 36 | if request.method == "POST" and request.body: 37 | req = json.loads(request.body) 38 | if "user" in req and req["user"] != decode["email"]: 39 | raise AuthException(401, "invalid user or token") 40 | 41 | 42 | def obtain_public_key(): 43 | global public_key 44 | 45 | if public_key == None: 46 | res = requests.get(config['TOKEN_SECRET_OR_PRIVATE_KEY']) 47 | if res.status_code == 200: 48 | public_key = res.json() 49 | else: 50 | raise NetWorkException(res.status_code, f"Request token public key fail: {res}") 51 | 52 | return public_key -------------------------------------------------------------------------------- /active-learning-service/src/utils/modules.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019-2021 VMware, Inc. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import importlib 8 | from config.config import config 9 | 10 | 11 | def download_spacy_module(): 12 | 13 | print(f'[ MODULE ] {config["SPACY_MODEL"]}') 14 | is_model_exist = is_setup_module(config['SPACY_MODEL']) 15 | if is_model_exist: 16 | return 17 | else: 18 | module = f"python -m spacy download {config['SPACY_MODEL']}" 19 | try: 20 | print(f'[ MODULE ] [ DOWNLOAD ]: {module}') 21 | os.system(module) 22 | except: 23 | # if error occurs will try again 24 | print(f'[ ERROR ] [ DOWNLOAD ]: {module}') 25 | print(f'[ RETRY ] [ DOWNLOAD ]: {module}') 26 | os.system(module) 27 | 28 | 29 | def is_setup_module(name): 30 | try: 31 | importlib.import_module(name) 32 | return True 33 | except: 34 | return False -------------------------------------------------------------------------------- /active-learning-service/src/views.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2019-2021 VMware, Inc. 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | import json 6 | from django.http import JsonResponse 7 | from django.views.decorators.csrf import csrf_exempt 8 | from django.views.decorators.http import require_GET, require_POST 9 | 10 | from .al.activelearning import active_learning_query, active_learning_train, active_learning_teach 11 | 12 | from .al.vector_sr import srs_vector 13 | from .utils.common import request_handle 14 | import src.tests as poc 15 | 16 | 17 | # vector all sr data 18 | @require_POST 19 | def al_srs_vector(request): 20 | return request_handle(request, srs_vector) 21 | 22 | 23 | # active learning train a model 24 | @require_POST 25 | def al_train_model(request): 26 | return request_handle(request, active_learning_train) 27 | 28 | 29 | # active learning query instance from model 30 | @require_POST 31 | def al_query_instance(request): 32 | return request_handle(request, active_learning_query) 33 | 34 | 35 | # active learning teach a model 36 | @require_POST 37 | def al_teach_model(request): 38 | return request_handle(request, active_learning_teach) 39 | 40 | 41 | # health check 42 | @require_GET 43 | def health(request): 44 | return JsonResponse({"status": "OK"}) 45 | 46 | 47 | # api_poc 48 | @require_GET 49 | def api_poc(request): 50 | return request_handle(request, poc.api_poc) 51 | -------------------------------------------------------------------------------- /annotation-app/.browserslistrc: -------------------------------------------------------------------------------- 1 | # This file is used by the build system to adjust CSS and JS output to support the specified browsers below. 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | 5 | # For the full list of supported browsers by the Angular framework, please see: 6 | # https://angular.io/guide/browser-support 7 | 8 | # You can see what browsers were selected by your queries by running: 9 | # npx browserslist 10 | 11 | last 1 Chrome version 12 | last 1 Firefox version 13 | last 2 Edge major versions 14 | last 2 Safari major versions 15 | last 2 iOS major versions 16 | Firefox ESR 17 | -------------------------------------------------------------------------------- /annotation-app/.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | package-lock.json 3 | node_modules 4 | -------------------------------------------------------------------------------- /annotation-app/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 120, 3 | "singleQuote": true, 4 | "trailingComma": "all", 5 | "tabWidth": 2 6 | } -------------------------------------------------------------------------------- /annotation-app/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2022 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | ### STAGE 1: Build ### 5 | FROM node:16-alpine as builder 6 | COPY package*.json . 7 | RUN npm config set registry http://registry.npmjs.org/ && npm i && mkdir /app && cp -R ./node_modules ./app 8 | WORKDIR /app 9 | COPY . . 10 | #ARG BUILD_ENV=production 11 | #RUN npm run build -- -c $BUILD_ENV 12 | RUN npm run build 13 | 14 | ### STAGE 2: Setup ### 15 | FROM nginx:1.13.5-alpine 16 | COPY nginx.conf /etc/nginx/conf.d/default.conf 17 | COPY --from=builder /app/docker-entrypoint.sh / 18 | COPY --from=builder /app/dist /usr/share/nginx/html 19 | EXPOSE 4200 20 | ENTRYPOINT ["/docker-entrypoint.sh"] 21 | -------------------------------------------------------------------------------- /annotation-app/build-script.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2024 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | const fs = require('fs'); 9 | const fse = require('fs-extra'); 10 | const concat = require('concat'); 11 | 12 | const bundleFileName = 'micro-bundle-loop.js'; 13 | const path = './dist/'; 14 | const filesToConcat = []; 15 | 16 | const buildFiles = fs.readdirSync(path); 17 | for (const file of buildFiles) { 18 | if ((file.includes('main') || file.includes('src_app_') || file.includes('scripts') || file.includes('vendors-')) && !(file.includes('.map'))) { 19 | filesToConcat.push(path + file); 20 | } 21 | } 22 | 23 | (async function build() { 24 | await fse.ensureDir('elements'); 25 | await concat(filesToConcat, 'elements/' + bundleFileName); 26 | })(); -------------------------------------------------------------------------------- /annotation-app/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2019-2022 VMware, Inc. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | ## Substitutes env variables in main.*.js bundle 7 | sed -i 's/${APP_CONFIG}/'"${APP_CONFIG}"'/' $(ls /usr/share/nginx/html/main*.js) 8 | sed -i 's/${CLIENT_ID}/'"${CLIENT_ID}"'/' $(ls /usr/share/nginx/html/main*.js) 9 | ## Starts the application 10 | nginx -g 'daemon off;' -------------------------------------------------------------------------------- /annotation-app/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 4200; 3 | root /usr/share/nginx/html; 4 | location / { 5 | try_files $uri $uri/ /index.html =404; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /annotation-app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "annotation-app", 3 | "version": "0.0.0", 4 | "scripts": { 5 | "ng": "ng", 6 | "start": "ng serve", 7 | "build": "ng build", 8 | "bundle": "ng run annotation-app:mfe && node build-script.js", 9 | "bundle_dev": "ng run annotation-app:mfe:dev && node build-script.js", 10 | "bundle_stg": "ng run annotation-app:mfe:stg && node build-script.js", 11 | "bundle_prod": "ng run annotation-app:mfe:prod && node build-script.js" 12 | }, 13 | "private": true, 14 | "dependencies": { 15 | "@angular-devkit/build-angular": "^14.2.9", 16 | "@angular-slider/ngx-slider": "^2.0.3", 17 | "@angular/animations": "^14.2.0", 18 | "@angular/common": "^14.2.0", 19 | "@angular/compiler": "^14.2.0", 20 | "@angular/core": "^14.2.0", 21 | "@angular/elements": "^14.2.11", 22 | "@angular/forms": "^14.2.0", 23 | "@angular/platform-browser": "^14.2.0", 24 | "@angular/platform-browser-dynamic": "^14.2.0", 25 | "@angular/router": "^14.2.0", 26 | "@auth0/angular-jwt": "^3.0.1", 27 | "@cds/angular": "6.2.2", 28 | "@cds/core": "6.2.2", 29 | "@clr/angular": "13.10.1", 30 | "@clr/ui": "13.10.1", 31 | "@ng-select/ng-select": "^9.1.0", 32 | "@types/lodash": "^4.14.189", 33 | "aws-sdk": "^2.1261.0", 34 | "buffer": "^5.5.0", 35 | "concat": "^1.0.3", 36 | "d3-v6-tip": "^1.0.9", 37 | "fs-extra": "^10.1.0", 38 | "js-untar": "^2.0.0", 39 | "jszip": "^3.7.1", 40 | "lodash": "^4.17.21", 41 | "marked": "^2.1.3", 42 | "ngx-build-plus": "^14.0.0", 43 | "ngx-papaparse": "^5.1.0", 44 | "pako": "^2.0.3", 45 | "rxjs": "~7.5.0", 46 | "tslib": "^2.3.0", 47 | "windowed-observable": "^1.0.0", 48 | "zone.js": "~0.11.4" 49 | }, 50 | "devDependencies": { 51 | "@angular/cli": "~14.2.5", 52 | "@angular/compiler-cli": "^14.2.0", 53 | "typescript": "~4.7.2" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /annotation-app/proxy.conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "/api/*": { 3 | "target": "http://localhost:3000", 4 | "secure": false, 5 | "logLevel": "debug", 6 | "changeOrigin": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /annotation-app/src/app/app-routing.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { RouterModule, Routes } from '@angular/router'; 8 | import { FAQComponent } from './component/faq/faq.component'; 9 | import { HomeComponent } from './component/home/home.component'; 10 | import { PageNotFoundComponent } from './component/page-not-found/page-not-found.component'; 11 | import { AuthGuard } from './guards/auth.guard'; 12 | 13 | const routes: Routes = [ 14 | { path: '', redirectTo: 'loop-home', pathMatch: 'full' }, 15 | { path: 'loop-home', component: HomeComponent }, 16 | { 17 | path: 'login', 18 | loadChildren: () => import('./component/login/login.module').then((m) => m.LoginsModule), 19 | }, 20 | { 21 | path: 'loop/datasets', 22 | loadChildren: () => import('./component/datasets/datasets.module').then((m) => m.DatasetsModule), 23 | canActivate: [AuthGuard], 24 | }, 25 | { 26 | path: 'loop/project', 27 | loadChildren: () => import('./component/projects/projects.module').then((m) => m.ProjectsModule), 28 | canActivate: [AuthGuard], 29 | }, 30 | { 31 | path: 'loop/permissions', 32 | loadChildren: () => import('./component/permission/permissions.module').then((m) => m.PermissionsModule), 33 | }, 34 | { path: 'loop/faq', component: FAQComponent }, 35 | { path: '404', component: PageNotFoundComponent }, 36 | { path: '**', redirectTo: '404' }, 37 | ]; 38 | 39 | @NgModule({ 40 | imports: [RouterModule.forRoot(routes)], 41 | exports: [RouterModule], 42 | }) 43 | export class AppRoutingModule {} 44 | -------------------------------------------------------------------------------- /annotation-app/src/app/app.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 | 6 |
7 |
8 | 9 |
10 |
11 | 12 |
13 |
14 |
15 | -------------------------------------------------------------------------------- /annotation-app/src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component } from '@angular/core'; 7 | 8 | @Component({ 9 | selector: 'mf-loop', 10 | templateUrl: './app.component.html', 11 | }) 12 | export class AppComponent { 13 | title = 'annotation-app'; 14 | } 15 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/create-new-dataset-modal/create-new-dataset-modal.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 | 47 |
48 |
49 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/create-new-dataset-modal/create-new-dataset-modal.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .modal { 6 | .modal-body { 7 | height: 70vh; 8 | overflow-x: hidden; 9 | padding-left: 5px; 10 | } 11 | 12 | .modal-dialog { 13 | width: 60vw; 14 | } 15 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/create-new-dataset-page/create-new-dataset-page.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 |
6 |
7 |

Upload New Dataset

8 | 13 | 14 |
15 | 18 | 21 | 22 |
23 |
24 |
25 |
26 |
27 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/create-new-dataset-page/create-new-dataset-page.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .h2Title { 6 | margin-top: unset; 7 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/create-new-dataset-page/create-new-dataset-page.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Component, OnInit } from '@angular/core'; 6 | import { EnvironmentsService } from 'src/app/services/environments.service'; 7 | 8 | @Component({ 9 | selector: 'app-create-new-dataset-page', 10 | templateUrl: './create-new-dataset-page.component.html', 11 | styleUrls: ['./create-new-dataset-page.component.scss'], 12 | }) 13 | export class CreateNewDatasetPageComponent implements OnInit { 14 | msg; 15 | uploading = false; 16 | createBtnDisable = true; 17 | 18 | constructor(public env: EnvironmentsService) {} 19 | 20 | ngOnInit(): void { 21 | this.msg = { page: 'createDataset', createDataBtn: 0 }; 22 | } 23 | 24 | receiveOutFormData(formdata) { 25 | if (formdata) { 26 | this.msg.type = formdata.fileFormat; 27 | this.createBtnDisable = false; 28 | } else { 29 | this.createBtnDisable = true; 30 | } 31 | } 32 | 33 | clickCreate() { 34 | this.uploading = true; 35 | this.msg.createDataBtn = this.msg.createDataBtn + 1; 36 | this.msg = JSON.parse(JSON.stringify(this.msg)); 37 | } 38 | 39 | receiveUploadDone(uploading) { 40 | this.uploading = false; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/create-new-dataset/create-new-dataset.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .sectionDivision { 7 | margin-top: 1rem; 8 | } 9 | 10 | .divisionLine { 11 | border-bottom: 1px solid rgb(197, 197, 197); 12 | margin-top: 2rem; 13 | } 14 | 15 | .divisionWidth { 16 | width: 99.7%; 17 | margin-left: 0; 18 | } 19 | 20 | 21 | 22 | .radioRow { 23 | margin-left: unset; 24 | margin-right: unset; 25 | margin-top: 1rem; 26 | 27 | .clr-radio-wrapper input[type=radio]+label::before { 28 | left: unset; 29 | right: 0; 30 | } 31 | 32 | .clr-radio-wrapper { 33 | padding: 0.5rem 0.5rem 0.5rem; 34 | border-radius: 5px; 35 | box-shadow: 1px 1px 4px 2px #e9e9e9; 36 | width: 49%; 37 | 38 | label { 39 | padding-left: unset; 40 | width: 100%; 41 | } 42 | } 43 | 44 | .clr-radio-wrapper:hover { 45 | box-shadow: 1px 1px 5px 1px rgb(128 168 196) 46 | } 47 | 48 | .selectedRadio { 49 | border: 1px solid rgb(0, 104, 150); 50 | } 51 | } 52 | 53 | .disabledRadio { 54 | opacity: 0.4; 55 | pointer-events: none; 56 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/dataset-analyze/analyze-datagrid/analyze-datagrid.component.html: -------------------------------------------------------------------------------- 1 | 3 |
4 | 5 | 6 | {{ column.label }} 7 | 8 | 9 | 10 | 11 | 18 | 19 | 20 | {{ itemData[column.prop] }} 21 | 22 | 23 | 24 | 25 |
26 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/dataset-analyze/analyze-datagrid/analyze-datagrid.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .columnWidth{ 6 | width: 16rem; 7 | } 8 | .imgLimit{ 9 | max-width: 15rem; 10 | max-height: 15rem 11 | } 12 | .showImg{ 13 | display: none; 14 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/dataset-analyze/analyze-datagrid/analyze-datagrid.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core'; 6 | 7 | @Component({ 8 | selector: 'app-analyze-datagrid', 9 | templateUrl: './analyze-datagrid.component.html', 10 | styleUrls: ['./analyze-datagrid.component.scss'], 11 | }) 12 | export class AnalyzeDatagridComponent implements OnInit { 13 | @Input() configData: any; 14 | @Input() gridloading: false; 15 | 16 | errorImg = false; 17 | 18 | constructor() { 19 | this.configData = { 20 | columnData: [], 21 | tableData: [], 22 | pageSizeOption: [10, 20, 50], 23 | type: '', 24 | }; 25 | } 26 | 27 | ngOnInit(): void {} 28 | 29 | setDefaultImage() { 30 | this.errorImg = true; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/dataset-analyze/dataset-analyze.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .h2Title { 6 | overflow: hidden; 7 | text-overflow: ellipsis; 8 | max-width: 85%; 9 | font-size: 2em; 10 | padding-top: 0.2rem; 11 | } 12 | 13 | 14 | 15 | cds-icon[shape="arrow"] { 16 | transform: rotate(-90deg); 17 | width: 1.5rem; 18 | height: 1.5rem; 19 | margin-right: 0.5rem; 20 | cursor: pointer; 21 | } 22 | 23 | .properties { 24 | border-bottom: 1px solid rgb(197, 197, 197); 25 | padding-bottom: 2rem; 26 | } 27 | 28 | .createEntrance { 29 | padding-left: 1.5rem; 30 | 31 | p { 32 | margin-top: 0.5rem; 33 | margin-bottom: 0.5rem; 34 | } 35 | 36 | div { 37 | padding-bottom: 1rem; 38 | } 39 | 40 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/datasets-routing.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { Routes, RouterModule } from '@angular/router'; 8 | import { CreateNewDatasetModalComponent } from './create-new-dataset-modal/create-new-dataset-modal.component'; 9 | import { CreateNewDatasetPageComponent } from './create-new-dataset-page/create-new-dataset-page.component'; 10 | import { DatasetAnalyzeComponent } from './dataset-analyze/dataset-analyze.component'; 11 | import { MyDatasetsComponent } from './my-datasets/my-datasets.component'; 12 | 13 | const routes: Routes = [ 14 | { 15 | path: 'list', 16 | component: MyDatasetsComponent, 17 | data: { 18 | title: 'loopDatasets', 19 | }, 20 | }, 21 | { 22 | path: 'create', 23 | component: CreateNewDatasetPageComponent, 24 | data: { 25 | title: 'loopDatasetsCreate', 26 | }, 27 | }, 28 | { 29 | path: 'create/modal', 30 | component: CreateNewDatasetModalComponent, 31 | data: { 32 | title: 'loopDatasetsCreate', 33 | }, 34 | }, 35 | { 36 | path: 'analyze', 37 | component: DatasetAnalyzeComponent, 38 | data: { 39 | title: 'loopDatasetsAnalyze', 40 | }, 41 | }, 42 | ]; 43 | 44 | @NgModule({ 45 | imports: [RouterModule.forChild(routes)], 46 | exports: [RouterModule], 47 | declarations: [], 48 | }) 49 | export class DatasetsRoutingModule {} 50 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/datasets.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { CommonModule } from '@angular/common'; 8 | import { SharedModule } from 'src/app/shared/shared.module'; 9 | import { DatasetsRoutingModule } from './datasets-routing.module'; 10 | import { MyDatasetsComponent } from './my-datasets/my-datasets.component'; 11 | import { ModalDatagridComponent } from './modal-datagrid/modal-datagrid.component'; 12 | import { CreateNewDatasetPageComponent } from './create-new-dataset-page/create-new-dataset-page.component'; 13 | import { DatasetAnalyzeComponent } from './dataset-analyze/dataset-analyze.component'; 14 | import { ProjectsModule } from '../projects/projects.module'; 15 | import { AnalyzeDatagridComponent } from './dataset-analyze/analyze-datagrid/analyze-datagrid.component'; 16 | 17 | @NgModule({ 18 | declarations: [ 19 | MyDatasetsComponent, 20 | ModalDatagridComponent, 21 | CreateNewDatasetPageComponent, 22 | DatasetAnalyzeComponent, 23 | AnalyzeDatagridComponent, 24 | ], 25 | imports: [CommonModule, DatasetsRoutingModule, SharedModule, ProjectsModule], 26 | }) 27 | export class DatasetsModule {} 28 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/dnd.directive.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Directive, Output, EventEmitter, HostBinding, HostListener } from '@angular/core'; 6 | 7 | @Directive({ 8 | selector: '[appDnd]', 9 | }) 10 | export class DndDirective { 11 | constructor() {} 12 | @HostBinding('class.fileover') fileOver: boolean; 13 | @Output() fileDropped = new EventEmitter(); 14 | 15 | // Dragover listener 16 | @HostListener('dragover', ['$event']) onDragOver(evt) { 17 | evt.preventDefault(); 18 | evt.stopPropagation(); 19 | this.fileOver = true; 20 | } 21 | 22 | // Dragleave listener 23 | @HostListener('dragleave', ['$event']) public onDragLeave(evt) { 24 | evt.preventDefault(); 25 | evt.stopPropagation(); 26 | this.fileOver = false; 27 | } 28 | 29 | // Drop listener 30 | @HostListener('drop', ['$event']) public ondrop(evt) { 31 | evt.preventDefault(); 32 | evt.stopPropagation(); 33 | this.fileOver = false; 34 | let files = evt.dataTransfer.files; 35 | if (files.length > 0) { 36 | this.fileDropped.emit(files); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/modal-datagrid/modal-datagrid.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core'; 6 | import { EnvironmentsService } from 'src/app/services/environments.service'; 7 | 8 | @Component({ 9 | selector: 'app-modal-datagrid', 10 | templateUrl: './modal-datagrid.component.html', 11 | }) 12 | export class ModalDatagridComponent implements OnInit { 13 | @Input() msgPreview: any; 14 | 15 | @Output('onClosePreviewDialog') 16 | onClosePreviewDialogEmitter = new EventEmitter(); 17 | 18 | previewDatasetDialog = true; 19 | topRowHeader: any = []; 20 | topRowContent: any = []; 21 | 22 | constructor(public env: EnvironmentsService) {} 23 | 24 | ngOnInit(): void { 25 | this.toShowPreview(); 26 | } 27 | 28 | toShowPreview() { 29 | if (this.msgPreview.format == 'image') { 30 | this.topRowContent = []; 31 | let flag = JSON.parse(JSON.stringify(this.msgPreview.topReview)); 32 | flag.forEach((element) => { 33 | element.fileSize = (element.fileSize / 1024).toFixed(2); 34 | if (!this.env.config.enableAWSS3) { 35 | element.location = `${this.env.config.annotationService}/api/v1.0/datasets/set-data?file=${ 36 | element.location 37 | }&token=${JSON.parse(localStorage.getItem(this.env.config.sessionKey)).token.access_token}`; 38 | } 39 | }); 40 | this.topRowContent = flag; 41 | this.topRowHeader = ['Id', 'ImageName', 'ImageSize(KB)', 'Image']; 42 | } else if (this.msgPreview.format == 'txt') { 43 | this.topRowHeader = ['FileName', 'FileContent']; 44 | this.topRowContent = this.msgPreview.topReview; 45 | } else { 46 | this.topRowHeader = this.msgPreview.topReview.header == null ? [] : this.msgPreview.topReview.header; 47 | this.topRowContent = this.msgPreview.topReview.topRows == null ? [] : this.msgPreview.topReview.topRows; 48 | } 49 | } 50 | 51 | onClosePreviewDialog() { 52 | this.onClosePreviewDialogEmitter.emit(true); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/my-datasets/my-datasets.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .pageTitle { 6 | margin-left: 0.2rem; 7 | align-items: baseline; 8 | justify-content: space-between; 9 | margin-bottom: 1rem; 10 | } 11 | 12 | .h2Title { 13 | margin-top: unset; 14 | } 15 | 16 | .floatRight { 17 | float: right; 18 | margin-right: unset !important; 19 | } 20 | 21 | clr-dg-cell { 22 | a { 23 | cursor: pointer; 24 | text-decoration: underline; 25 | color: rgb(0, 112, 172); 26 | } 27 | } 28 | 29 | .cellWordBreak { 30 | word-break: break-all; 31 | display: -webkit-box; 32 | -webkit-box-orient: vertical; 33 | -webkit-line-clamp: 2; 34 | overflow: hidden; 35 | height: 1.8rem; 36 | line-height: 0.7rem; 37 | } 38 | 39 | .actionClass { 40 | padding: 0.1rem 0.2rem; 41 | height: 1rem; 42 | line-height: 0.6rem; 43 | min-width: 1.5rem; 44 | } 45 | 46 | .greenBtn { 47 | opacity: 0.7; 48 | } 49 | 50 | .disableTree { 51 | ::ng-deep .clr-checkbox-wrapper .clr-control-label { 52 | pointer-events: none; 53 | opacity: 0.5; 54 | } 55 | 56 | ::ng-deep .clr-checkbox-wrapper input[type='checkbox'] { 57 | pointer-events: none; 58 | } 59 | } 60 | 61 | 62 | .ellipsisMore { 63 | max-width: 30vw; 64 | 65 | a { 66 | padding-right: 0.5rem; 67 | cursor: pointer; 68 | } 69 | 70 | a:last-child { 71 | padding-right: 0.5rem; 72 | } 73 | 74 | div:nth-child(1) { 75 | overflow: hidden; 76 | text-overflow: ellipsis; 77 | white-space: nowrap; 78 | max-width: 100%; 79 | } 80 | 81 | div:nth-child(2) { 82 | text-decoration: underline; 83 | color: rgb(0, 103, 159); 84 | cursor: pointer; 85 | } 86 | 87 | .openMore { 88 | display: block; 89 | margin-bottom: 0.3rem; 90 | } 91 | 92 | ::ng-deep .spinner.spinner-sm { 93 | height: 0.6rem; 94 | width: 0.6rem; 95 | min-height: 0.6rem; 96 | min-width: 0.6rem; 97 | } 98 | 99 | } 100 | 101 | ::ng-deep input.disabled-checkbox+label::before { 102 | background-color: #eee; 103 | cursor: not-allowed; 104 | opacity: 0.3; 105 | } 106 | 107 | .refreshBtn { 108 | display: inline-block; 109 | margin-right: 2rem; 110 | color: #0079b8; 111 | font-weight: 600; 112 | font-size: 0.7rem; 113 | font-style: normal; 114 | cursor: pointer; 115 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/datasets/upload-file/upload-file.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .progressFile { 6 | margin-left: -0.2rem; 7 | } 8 | 9 | .progressBlock { 10 | line-height: 1.7rem; 11 | margin-top: 1rem; 12 | } 13 | 14 | .uploadBox { 15 | border: 1px dashed #e9e9e9; 16 | text-align: center; 17 | margin-top: 1rem; 18 | height: 3rem; 19 | background-color: white; 20 | line-height: 3rem; 21 | position: relative; 22 | 23 | 24 | input { 25 | opacity: 0; 26 | position: absolute; 27 | z-index: 2; 28 | width: 100%; 29 | height: 100%; 30 | top: 0; 31 | left: 0; 32 | } 33 | 34 | label { 35 | width: 183px; 36 | height: 44px; 37 | text-decoration: underline; 38 | color: rgb(68, 137, 188); 39 | 40 | } 41 | } 42 | 43 | .fileover { 44 | border: 1px dashed rgb(0, 104, 150); 45 | background-color: rgb(222, 244, 251); 46 | } 47 | 48 | .file-icon { 49 | 50 | .delete { 51 | display: flex; 52 | margin-left: 0.5rem; 53 | cursor: pointer; 54 | align-self: flex-end; 55 | } 56 | 57 | display: flex; 58 | flex-grow: 1; 59 | } 60 | 61 | .uploadInfo { 62 | margin-top: 0; 63 | font-size: 0.8em; 64 | color: #999; 65 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/faq/faq.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .title { 7 | margin-top: 2rem; 8 | margin-bottom: 2rem; 9 | } 10 | 11 | ::ng-deep .clr-accordion-header-button { 12 | height: 2.5rem; 13 | } 14 | 15 | ::ng-deep .clr-accordion-inner-content { 16 | padding: 0.8rem 1.5rem !important; 17 | } 18 | 19 | ::ng-deep .clr-accordion-content { 20 | border: unset !important; 21 | } 22 | 23 | ::ng-deep .clr-accordion-header { 24 | border: unset !important; 25 | border-bottom: 1px solid #ededed !important; 26 | } 27 | 28 | span { 29 | font-size: 0.7rem; 30 | padding-right: 0.3rem; 31 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/faq/faq.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component } from '@angular/core'; 7 | import { EnvironmentsService } from 'src/app/services/environments.service'; 8 | @Component({ 9 | styleUrls: ['./faq.component.scss'], 10 | templateUrl: './faq.component.html', 11 | }) 12 | export class FAQComponent { 13 | constructor(public env: EnvironmentsService) {} 14 | } 15 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/header/header.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 |
6 | 7 | 8 | {{ env.config.serviceTitle }} 9 | 10 |
11 |
12 | Sign In 13 | 14 | 18 | 19 | Log out 20 | 21 | 22 |
23 |
24 | 34 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/header/header.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .flaskIcon { 6 | color: white; 7 | width: 1.2rem !important; 8 | height: 1.2rem !important; 9 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/header/header.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Component, OnInit } from '@angular/core'; 6 | import { EnvironmentsService } from 'src/app/services/environments.service'; 7 | import { Router } from '@angular/router'; 8 | import { UserAuthService } from 'src/app/services/user-auth.service'; 9 | 10 | @Component({ 11 | selector: 'app-header', 12 | templateUrl: './header.component.html', 13 | styleUrls: ['./header.component.scss'], 14 | }) 15 | export class HeaderComponent implements OnInit { 16 | loggedUser; 17 | errMessage: string; 18 | role: string; 19 | 20 | constructor(private router: Router, private userAuthService: UserAuthService, public env: EnvironmentsService) {} 21 | 22 | ngOnInit(): void { 23 | if (!this.env.config.embedded) { 24 | this.userAuthService.loggedUserListener().subscribe((res) => { 25 | if (res) { 26 | this.loggedUser = res; 27 | } 28 | }); 29 | } 30 | } 31 | 32 | login() { 33 | if (!this.env.config.embedded) { 34 | if (this.env.config.authUrl) { 35 | this.userAuthService.redirectToLogin(); 36 | } else { 37 | this.router.navigateByUrl('/login/basic'); 38 | } 39 | } 40 | } 41 | 42 | logOut() { 43 | this.userAuthService.logout(); 44 | this.router.navigateByUrl('/loop-home'); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/home/home.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Component, OnInit } from '@angular/core'; 6 | import { ActivatedRoute } from '@angular/router'; 7 | import { EnvironmentsService } from 'src/app/services/environments.service'; 8 | import { UserAuthService } from 'src/app/services/user-auth.service'; 9 | 10 | @Component({ 11 | selector: 'app-home', 12 | templateUrl: './home.component.html', 13 | styleUrls: ['./home.component.scss'], 14 | }) 15 | export class HomeComponent implements OnInit { 16 | unsubscribe: boolean; 17 | isUnsubscribe: boolean; 18 | 19 | constructor( 20 | private route: ActivatedRoute, 21 | private userAuthService: UserAuthService, 22 | public env: EnvironmentsService, 23 | ) {} 24 | 25 | ngOnInit(): void { 26 | this.route.queryParams.subscribe((params) => { 27 | const isUserLandingFromOutside = !params['hash'] || String(params['hash']).length == 0; 28 | if (params['o'] == 'email') { 29 | this.isUnsubscribe = true; 30 | this.unsubscribe = String(params['s']) == '1' ? true : false; 31 | } else { 32 | this.isUnsubscribe = false; 33 | } 34 | // if (isUserLandingFromOutside || this.unsubscribe) { 35 | // const user = this.userAuthService.loggedUser().user; 36 | // } 37 | if (this.isUnsubscribe) { 38 | setTimeout(() => { 39 | this.isUnsubscribe = false; 40 | }, 10000); 41 | } 42 | }); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/login/basic-login/basic-login.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .loading { 7 | background-color: rgba(0, 119, 184, 0.2); 8 | border-color: rgba(0, 119, 184, 0.2); 9 | } 10 | 11 | .login-wrapper .login { 12 | min-height: unset; 13 | } 14 | 15 | .helperText { 16 | font-size: 0.55rem; 17 | line-height: 0.6rem; 18 | color: #9c9c9c; 19 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/login/login-routing.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { Routes, RouterModule } from '@angular/router'; 8 | import { BasicLoginComponent } from './basic-login/basic-login.component'; 9 | import { LoginComponent } from './login/login.component'; 10 | 11 | const routes: Routes = [ 12 | { 13 | path: 'basic', 14 | component: BasicLoginComponent, 15 | data: { 16 | title: 'loopBasicLogin', 17 | }, 18 | }, 19 | { 20 | path: 'authenticate', 21 | component: LoginComponent, 22 | data: { title: 'loopAuthenticateLogin' }, 23 | }, 24 | ]; 25 | 26 | @NgModule({ 27 | imports: [RouterModule.forChild(routes)], 28 | exports: [RouterModule], 29 | declarations: [], 30 | }) 31 | export class LoginRoutingModule {} 32 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/login/login.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { CommonModule } from '@angular/common'; 8 | import { LoginRoutingModule } from './login-routing.module'; 9 | import { BasicLoginComponent } from './basic-login/basic-login.component'; 10 | import { SharedModule } from 'src/app/shared/shared.module'; 11 | import { LoginComponent } from './login/login.component'; 12 | 13 | @NgModule({ 14 | declarations: [BasicLoginComponent, LoginComponent], 15 | imports: [CommonModule, LoginRoutingModule, SharedModule], 16 | }) 17 | export class LoginsModule {} 18 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/login/login/login.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 7 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/login/login/login.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .uploadLoading { 7 | background-color: rgba(0, 119, 184, 0.2); 8 | border-color: rgba(0, 119, 184, 0.2); 9 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/login/login/login.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, OnInit } from '@angular/core'; 7 | import { Router, ActivatedRoute } from '@angular/router'; 8 | import { Location } from '@angular/common'; 9 | import { UserAuthService } from 'src/app/services/user-auth.service'; 10 | import { map, mergeMap } from 'rxjs/operators'; 11 | import { throwError } from 'rxjs'; 12 | import { User } from '../../../model/user'; 13 | import { EnvironmentsService } from 'src/app/services/environments.service'; 14 | 15 | @Component({ 16 | selector: 'app-login', 17 | templateUrl: './login.component.html', 18 | styles: [ 19 | ` 20 | .filter-artifacts { 21 | text-align: right; 22 | } 23 | `, 24 | ], 25 | }) 26 | export class LoginComponent implements OnInit { 27 | user = { email: '', password: '' }; 28 | returnUrl!: string; 29 | errorMessage = ''; 30 | authSource?: number; 31 | loggedUser: User | null = null; 32 | loading?: boolean; 33 | 34 | constructor( 35 | private route: ActivatedRoute, 36 | private router: Router, 37 | private userAuthService: UserAuthService, 38 | private location: Location, 39 | private env: EnvironmentsService, 40 | ) {} 41 | 42 | ngOnInit() { 43 | this.returnUrl = this.route.snapshot.queryParams['returnUrl'] || '/'; 44 | this.loading = true; 45 | this.route.queryParams 46 | .pipe( 47 | map((params) => params), 48 | mergeMap((data: any) => { 49 | const code = data['code']; 50 | const state = data['state']; 51 | if (state !== this.env.config.STATE) { 52 | return throwError('Wrong state...'); 53 | } 54 | const redirectUrl = 55 | window.location.origin + 56 | this.location.prepareExternalUrl(this.env.config.redirectUrl ? this.env.config.redirectUrl : '/loop-home'); 57 | return this.userAuthService.loging(code, this.env.config.CLIENT_ID, redirectUrl); 58 | }), 59 | ) 60 | .subscribe( 61 | (data) => { 62 | this.loading = false; 63 | this.router.navigate([this.returnUrl]); 64 | }, 65 | (error) => { 66 | this.loading = false; 67 | this.router.navigate(['/loop-home']); 68 | }, 69 | () => { 70 | this.loading = false; 71 | }, 72 | ); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/page-not-found/page-not-found.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 |
6 |
7 |
8 |
9 |
404
10 |
11 |
12 |

Sorry,We couldn't find the page you requested.

13 |

14 | Click here 15 | to go back to the homepage. 16 |

17 |
18 |
19 |
20 |
21 |
22 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/page-not-found/page-not-found.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .circleBox { 7 | display: flex; 8 | align-items: center; 9 | justify-content: center; 10 | } 11 | 12 | .circle { 13 | background-color: #002538; 14 | width: 5rem; 15 | height: 5rem; 16 | line-height: 5rem; 17 | text-align: center; 18 | border-radius: 50%; 19 | color: #fafafa; 20 | font-size: 1.5rem; 21 | font-weight: bold; 22 | } 23 | 24 | .box { 25 | position: absolute; 26 | top: 35%; 27 | left: 50%; 28 | transform: translate(-50%, 50%); 29 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/page-not-found/page-not-found.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component } from '@angular/core'; 7 | 8 | @Component({ 9 | selector: 'app-page-not-found', 10 | templateUrl: './page-not-found.component.html', 11 | styleUrls: ['./page-not-found.component.scss'], 12 | }) 13 | export class PageNotFoundComponent {} 14 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/permission/permissions-routing.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { Routes, RouterModule } from '@angular/router'; 8 | import { PermissionsComponent } from './permissions/permissions.component'; 9 | 10 | const routes: Routes = [ 11 | { 12 | path: 'users', 13 | component: PermissionsComponent, 14 | data: { 15 | title: 'loopPermissions', 16 | }, 17 | }, 18 | ]; 19 | 20 | @NgModule({ 21 | imports: [RouterModule.forChild(routes)], 22 | exports: [RouterModule], 23 | declarations: [], 24 | }) 25 | export class PermissionsRoutingModule {} 26 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/permission/permissions.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { CommonModule } from '@angular/common'; 8 | import { PermissionsRoutingModule } from './permissions-routing.module'; 9 | import { PermissionsComponent } from './permissions/permissions.component'; 10 | import { SharedModule } from 'src/app/shared/shared.module'; 11 | 12 | @NgModule({ 13 | declarations: [PermissionsComponent], 14 | imports: [CommonModule, PermissionsRoutingModule, SharedModule], 15 | }) 16 | export class PermissionsModule {} 17 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/permission/permissions/permissions.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .h2Title { 7 | margin-top: unset; 8 | } 9 | 10 | .floatRight { 11 | float: right; 12 | margin-right: unset !important; 13 | } 14 | 15 | .actionClass { 16 | min-width: unset; 17 | } 18 | 19 | .editModal input[type='text'], 20 | .editModal input[type='file'] { 21 | width: 100% !important; 22 | margin-bottom: 0px; 23 | } 24 | 25 | ::ng-deep .editModal .modal-dialog { 26 | min-width: 50%; 27 | 28 | .modal-body-wrapper { 29 | min-height: 25vh; 30 | } 31 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/annotate-progress-board/annotate-progress-board.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .left-project-info { 7 | margin-top: 0.5rem; 8 | } 9 | 10 | h5 { 11 | font-weight: bold; 12 | } 13 | 14 | .historyBox { 15 | max-height: 11rem; 16 | overflow: auto; 17 | height: 11rem; 18 | padding-right: 1rem; 19 | } 20 | 21 | .fontSizeSmall { 22 | font-size: 0.9em; 23 | } 24 | 25 | .historyText { 26 | overflow: hidden; 27 | min-width: 0; 28 | text-align: left; 29 | white-space: nowrap; 30 | padding-right: 1rem; 31 | text-overflow: ellipsis; 32 | cursor: pointer; 33 | } 34 | 35 | .historyPosition:hover { 36 | background-color: #ececec; 37 | border-radius: 4px; 38 | 39 | .historyIcon { 40 | color: #0077b8; 41 | } 42 | } 43 | 44 | .historyIcon { 45 | position: absolute; 46 | top: 0.3rem; 47 | } 48 | 49 | .historyPosition { 50 | position: relative; 51 | padding-right: 1rem; 52 | padding-left: 1.5rem; 53 | } 54 | 55 | .ellipsis { 56 | text-overflow: ellipsis; 57 | overflow: hidden; 58 | } 59 | 60 | .selectedHistory { 61 | background-color: #e1f1f6 !important; 62 | border-radius: 4px !important; 63 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/annotate-progress-board/annotate-progress-board.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; 7 | import * as _ from 'lodash'; 8 | import { UserAuthService } from 'src/app/services/user-auth.service'; 9 | 10 | @Component({ 11 | selector: 'app-annotate-progress-board', 12 | templateUrl: './annotate-progress-board.component.html', 13 | styleUrls: ['./annotate-progress-board.component.scss'], 14 | }) 15 | export class AnnotateProgressBoardComponent implements OnInit { 16 | @Input() msg; 17 | @Output('outClickHistory') 18 | outClickHistoryEmitter = new EventEmitter(); 19 | @Output('outClickReviewOrder') 20 | outClickReviewOrder = new EventEmitter(); 21 | @Output('outSelectReviewee') 22 | outSelectReviewee = new EventEmitter(); 23 | 24 | reviewOrder: string; 25 | reviewee: string; 26 | isAllowedAnnotate: boolean; 27 | user: any; 28 | optionList: []; 29 | 30 | constructor(private userAuthService: UserAuthService) { 31 | this.user = this.userAuthService.loggedUser()?.user; 32 | } 33 | 34 | ngOnInit(): void { 35 | this.reviewOrder = this.msg.projectInfo.assignmentLogic; 36 | this.reviewee = this.msg.reviewee; 37 | this.isAllowedAnnotate = this.msg.projectInfo.annotator.indexOf(this.user.email) > -1 ? true : false; 38 | this.optionList = this.msg?.projectInfo?.userCompleteCase; 39 | } 40 | 41 | ngOnChanges() {} 42 | 43 | historyBack(index, id) { 44 | let value = { 45 | index, 46 | id, 47 | }; 48 | this.outClickHistoryEmitter.emit(value); 49 | } 50 | 51 | changeReviewOrder(e) { 52 | this.reviewOrder = e.target.value; 53 | this.outClickReviewOrder.emit(this.reviewOrder); 54 | } 55 | 56 | onSelectingReviewee(e) { 57 | this.outSelectReviewee.emit(this.reviewee); 58 | } 59 | 60 | clickUncertain(e) { 61 | if (e.target.innerText === 'Uncertain' && this.reviewOrder !== 'most_uncertain') { 62 | this.changeReviewOrder({ target: { value: 'most_uncertain' } }); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/download/download.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .modal .modal-dialog { 7 | width: inherit !important; 8 | } 9 | 10 | .modal .modal-dialog .modal-content { 11 | width: 800px; 12 | } 13 | 14 | clr-alert { 15 | margin-top: 1rem; 16 | } 17 | 18 | label span { 19 | color: red; 20 | } 21 | 22 | .form-group .select { 23 | width: 100%; 24 | } 25 | 26 | ::ng-deep .dcr-doc-form .select2-class { 27 | width: 100% !important; 28 | } 29 | 30 | .form-group input[type='text'], 31 | .form-group input[type='file'] { 32 | width: 100% !important; 33 | margin-bottom: 0px; 34 | } 35 | 36 | .form-group .select, 37 | .form-group select { 38 | width: 100% !important; 39 | } 40 | 41 | .form-group textarea { 42 | width: 100% !important; 43 | } 44 | 45 | ::ng-deep .btn.btn-sm { 46 | margin-top: 7px !important; 47 | } 48 | 49 | ::ng-deep .md-check { 50 | margin-left: 10px !important; 51 | } 52 | 53 | ::ng-deep .footerLeft .modal-footer { 54 | display: inherit !important; 55 | } 56 | 57 | .uploadLoading { 58 | background-color: rgba(0, 119, 184, 0.2); 59 | border-color: rgba(0, 119, 184, 0.2); 60 | padding-top: 0.2rem; 61 | margin-left: 0.2rem; 62 | } 63 | 64 | .removeEntry { 65 | margin-top: 0.5rem; 66 | margin-left: 0.05rem; 67 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/generate/generate.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .modal .modal-dialog { 7 | width: inherit !important; 8 | } 9 | 10 | .modal .modal-dialog .modal-content { 11 | width: 800px; 12 | } 13 | 14 | clr-alert { 15 | margin-top: 1rem; 16 | } 17 | 18 | label span { 19 | color: red; 20 | } 21 | 22 | .form-group .select { 23 | width: 100%; 24 | } 25 | 26 | ::ng-deep .dcr-doc-form .select2-class { 27 | width: 100% !important; 28 | } 29 | 30 | .form-group input[type='text'], 31 | .form-group input[type='file'] { 32 | width: 100% !important; 33 | margin-bottom: 0px; 34 | } 35 | 36 | .form-group .select, 37 | .form-group select { 38 | width: 100% !important; 39 | } 40 | 41 | .form-group textarea { 42 | width: 100% !important; 43 | } 44 | 45 | ::ng-deep .btn.btn-sm { 46 | margin-top: 7px !important; 47 | } 48 | 49 | ::ng-deep .md-check { 50 | margin-left: 10px !important; 51 | } 52 | 53 | ::ng-deep .footerLeft .modal-footer { 54 | display: inherit !important; 55 | } 56 | 57 | .uploadLoading { 58 | background-color: rgba(0, 119, 184, 0.2); 59 | border-color: rgba(0, 119, 184, 0.2); 60 | padding-top: 0.2rem; 61 | margin-left: 0.2rem; 62 | } 63 | 64 | .removeEntry { 65 | margin-top: 0.5rem; 66 | margin-left: 0.12rem; 67 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/generate/generate.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core'; 7 | import { UserAuthService } from '../../../services/user-auth.service'; 8 | import { ApiService } from '../../../services/api.service'; 9 | import { DownloadService } from 'src/app/services/common/download.service'; 10 | 11 | @Component({ 12 | selector: 'app-generate', 13 | templateUrl: './generate.component.html', 14 | styleUrls: ['./generate.component.scss'], 15 | }) 16 | export class GenerateComponent implements OnInit { 17 | @Input() msg: any; 18 | 19 | @Output('onCloseGenerateDialog') 20 | onCloseGenerateDialogEmitter = new EventEmitter(); 21 | 22 | @Output() private refreshProject = new EventEmitter(); 23 | 24 | user: any; 25 | loading = false; 26 | errorMessage = ''; 27 | infoMessage = ''; 28 | format = ''; 29 | loadingGenerate = false; 30 | onlyLabelled = true; 31 | 32 | constructor( 33 | private apiService: ApiService, 34 | private userAuthService: UserAuthService, 35 | private downloadService: DownloadService, 36 | ) { 37 | this.user = this.userAuthService.loggedUser()?.user.email; 38 | } 39 | 40 | ngOnInit() { 41 | this.format = 'standard'; 42 | } 43 | 44 | onCloseGenerateDialog() { 45 | this.onCloseGenerateDialogEmitter.emit(); 46 | } 47 | 48 | generateNewProject() { 49 | this.loadingGenerate = true; 50 | this.apiService.generate(this.msg.id, this.format, this.msg.src, this.onlyLabelled ? 'Yes' : 'No').subscribe( 51 | (res) => { 52 | res.Modal = 'generate'; 53 | this.refreshProject.emit(res); 54 | }, 55 | (error: any) => { 56 | this.loadingGenerate = false; 57 | this.loading = false; 58 | }, 59 | ); 60 | } 61 | 62 | removeUnlabel(e) { 63 | this.onlyLabelled = e.target.checked; 64 | } 65 | 66 | downloadOriginal(urls) { 67 | this.downloadService.downloadMultiple(urls); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/labeling-task-list/projects.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 |
6 |
7 |

Labeling Tasks Lists

8 |
9 |
10 |
11 |  REFRESH 12 |
13 | 14 |
15 |
16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/labeling-task-list/projects.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .pageTitle { 7 | margin-left: 0.2rem; 8 | align-items: baseline; 9 | justify-content: space-between; 10 | margin-bottom: 1rem; 11 | } 12 | 13 | .h2Title { 14 | margin-top: unset; 15 | } 16 | 17 | .floatRight { 18 | float: right; 19 | margin-right: unset !important; 20 | } 21 | 22 | .refreshBtn { 23 | display: inline-block; 24 | margin-right: 2rem; 25 | color: #0079b8; 26 | font-weight: 600; 27 | font-size: 0.7rem; 28 | font-style: normal; 29 | cursor: pointer; 30 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/labeling-task-list/projects.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, OnInit } from '@angular/core'; 7 | import { UserAuthService } from 'src/app/services/user-auth.service'; 8 | import { ActivatedRoute } from '@angular/router'; 9 | 10 | @Component({ 11 | selector: 'app-projects', 12 | templateUrl: './projects.component.html', 13 | styleUrls: ['./projects.component.scss'], 14 | }) 15 | export class ProjectsComponent implements OnInit { 16 | user; 17 | msg; 18 | 19 | constructor(private userAuthService: UserAuthService, private route: ActivatedRoute) {} 20 | 21 | ngOnInit(): void { 22 | this.user = this.userAuthService.loggedUser()?.user; 23 | this.msg = { tab: 'annotate' }; 24 | this.route.queryParams.subscribe((queryParams) => { 25 | this.msg.tab = queryParams['tabType'] || 'annotate'; 26 | }); 27 | } 28 | 29 | clickTaskTab(tab) { 30 | this.msg.tab = tab; 31 | } 32 | 33 | reload() { 34 | if (this.msg.tab == 'annotate') { 35 | this.msg.reload = 'annotate'; 36 | } else { 37 | this.msg.reload = 'admin'; 38 | } 39 | this.msg = JSON.parse(JSON.stringify(this.msg)); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/project-analyze/latest-annotation-data/latest-annotation-data.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .modelChartBox { 6 | border-radius: 8px; 7 | background-color: white; 8 | height: 20rem; 9 | margin-bottom: 3rem; 10 | } 11 | 12 | .studioCell { 13 | min-width: 70% !important 14 | } 15 | 16 | .annotationLoading { 17 | background-color: white; 18 | border-radius: 8px; 19 | min-height: 9rem; 20 | text-align: center; 21 | } 22 | 23 | .loadingSpan { 24 | position: relative; 25 | display: inline-block; 26 | text-align: center; 27 | padding: 0.75rem 1rem; 28 | color: grey; 29 | opacity: 0.65; 30 | font-size: 0.7rem; 31 | } 32 | 33 | .previewHeight { 34 | height: 3.4rem !important; 35 | } 36 | 37 | .btnOutline { 38 | margin-top: 0.3rem; 39 | padding-left: unset; 40 | outline-color: transparent; 41 | } 42 | :host ::ng-deep input.disabled-checkbox+label::before { 43 | background-color: #eee; 44 | cursor: not-allowed; 45 | opacity: 0.3; 46 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/project-analyze/user-category-d3/user-category-d3.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 |
6 |

Number of Assigned Labels

7 |
8 |
9 |
10 | 11 | 12 |
13 |
14 | 15 | 16 |
17 |
18 |
19 |
20 |
# Annotations Per User
21 |
22 |
23 | 24 |
25 |
26 |    NO ANNOTATION 27 |
28 |
29 |
35 |
36 |
37 |
38 |
39 |
# Annotations Per Category
40 |
# Annotations Per Value Range
41 |
42 |
43 | 44 |
45 |
46 |    NO ANNOTATION 47 |
48 |
49 |
55 |
56 |
57 |
58 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/project-analyze/user-category-d3/user-category-d3.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | .topBoardBox { 7 | height: 15rem; 8 | background-color: white; 9 | border-radius: 8px; 10 | margin-top: 0.5rem; 11 | position: relative; 12 | } 13 | 14 | .topBoard { 15 | padding: unset; 16 | } 17 | 18 | .floatRight { 19 | text-align: right; 20 | margin-top: 1.2rem; 21 | display: block; 22 | margin-right: unset; 23 | 24 | } 25 | 26 | .loadingD3 { 27 | text-align: center; 28 | padding-top: 5rem; 29 | position: absolute; 30 | transform: translate(-50%); 31 | left: 50%; 32 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/projects-routing.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { Routes, RouterModule } from '@angular/router'; 8 | import { CreateProjectComponent } from './create-project/create-project.component'; 9 | import { ProjectAnalyzeComponent } from './project-analyze/project-analyze.component'; 10 | import { ProjectsComponent } from './labeling-task-list/projects.component'; 11 | import { AuthGuard } from 'src/app/guards/auth.guard'; 12 | 13 | const routes: Routes = [ 14 | { 15 | path: 'list', 16 | component: ProjectsComponent, 17 | data: { 18 | title: 'loopProjectList', 19 | }, 20 | canActivate: [AuthGuard], 21 | }, 22 | { 23 | path: 'create', 24 | component: CreateProjectComponent, 25 | data: { 26 | title: 'loopProjectCreate', 27 | }, 28 | canActivate: [AuthGuard], 29 | }, 30 | { 31 | path: 'analyze', 32 | component: ProjectAnalyzeComponent, 33 | data: { 34 | title: 'loopProjectAnalyze', 35 | }, 36 | canActivate: [AuthGuard], 37 | }, 38 | ]; 39 | 40 | @NgModule({ 41 | imports: [RouterModule.forChild(routes)], 42 | exports: [RouterModule], 43 | declarations: [], 44 | }) 45 | export class ProjectsRoutingModule {} 46 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/projects.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { CommonModule } from '@angular/common'; 8 | import { SharedModule } from 'src/app/shared/shared.module'; 9 | import { ProjectsRoutingModule } from './projects-routing.module'; 10 | import { GenerateComponent } from './generate/generate.component'; 11 | import { DownloadComponent } from './download/download.component'; 12 | import { TreeviewModalComponent } from './treeview-modal/treeview-modal.component'; 13 | import { CdsIconsService } from 'src/app/services/cds-icon.service'; 14 | import { CreateProjectComponent } from './create-project/create-project.component'; 15 | import { ProjectsComponent } from './labeling-task-list/projects.component'; 16 | import { ProjectAnalyzeComponent } from './project-analyze/project-analyze.component'; 17 | import { NgxSliderModule } from '@angular-slider/ngx-slider'; 18 | import { AnnotateProgressBoardComponent } from './annotate-progress-board/annotate-progress-board.component'; 19 | import { LabelStudioService } from 'src/app/services/label-studio.service'; 20 | import { LatestAnnotationDataComponent } from './project-analyze/latest-annotation-data/latest-annotation-data.component'; 21 | import { UserCategoryD3Component } from './project-analyze/user-category-d3/user-category-d3.component'; 22 | import { AppendComponent } from './project-analyze/append/append.component'; 23 | import { EditProjectComponent } from './labeling-task-list/edit-project/edit-project.component'; 24 | import { NgSelectModule } from '@ng-select/ng-select'; 25 | import { TaskDatagridComponent } from './labeling-task-list/task-datagrid/task-datagrid.component'; 26 | 27 | @NgModule({ 28 | declarations: [ 29 | GenerateComponent, 30 | DownloadComponent, 31 | TreeviewModalComponent, 32 | CreateProjectComponent, 33 | ProjectsComponent, 34 | ProjectAnalyzeComponent, 35 | AnnotateProgressBoardComponent, 36 | LatestAnnotationDataComponent, 37 | UserCategoryD3Component, 38 | AppendComponent, 39 | EditProjectComponent, 40 | TaskDatagridComponent, 41 | ], 42 | imports: [CommonModule, ProjectsRoutingModule, SharedModule, NgxSliderModule, NgSelectModule], 43 | providers: [CdsIconsService, LabelStudioService], 44 | exports: [GenerateComponent, DownloadComponent, TreeviewModalComponent], 45 | }) 46 | export class ProjectsModule {} 47 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/treeview-modal/treeview-modal.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 | 27 |
28 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/treeview-modal/treeview-modal.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .disableTree { 6 | ::ng-deep .clr-checkbox-wrapper .clr-control-label { 7 | pointer-events: none; 8 | opacity: 0.5; 9 | } 10 | 11 | ::ng-deep .clr-checkbox-wrapper input[type='checkbox'] { 12 | pointer-events: none; 13 | } 14 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/projects/treeview-modal/treeview-modal.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; 6 | 7 | @Component({ 8 | selector: 'app-treeview-modal', 9 | templateUrl: './treeview-modal.component.html', 10 | styleUrls: ['./treeview-modal.component.scss'], 11 | }) 12 | export class TreeviewModalComponent implements OnInit { 13 | @Input() treeData: any; 14 | 15 | @Output('onCloseTreeDialog') 16 | onCloseTreeDialog = new EventEmitter(); 17 | getChildren: any; 18 | constructor() {} 19 | 20 | ngOnInit() { 21 | this.getChildren = (folder) => folder.children; 22 | } 23 | 24 | onCloseDialog() { 25 | this.onCloseTreeDialog.emit(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/sidenav/sidenav.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | Datasets 8 | 9 | Datasets List 10 | Upload New Dataset 11 | 12 | 13 | 14 | 15 | Labeling Tasks 16 | 17 | Labeling Tasks List 18 | Create New Labeling Task 19 | 20 | 21 |
22 | 23 | 24 | Permissions 25 | 26 |
27 |
28 | 29 | 30 | Feedback 31 | 32 | 33 | 34 | Help 35 | 36 | 40 | 44 |
45 | 46 | -------------------------------------------------------------------------------- /annotation-app/src/app/component/sidenav/sidenav.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | clr-vertical-nav { 7 | height: 100%; 8 | position: relative; 9 | } 10 | 11 | .bottomHelpLink { 12 | margin-top: 50vh; 13 | } -------------------------------------------------------------------------------- /annotation-app/src/app/component/sidenav/sidenav.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, OnInit } from '@angular/core'; 7 | import { EnvironmentsService } from 'src/app/services/environments.service'; 8 | import { UserAuthService } from 'src/app/services/user-auth.service'; 9 | 10 | @Component({ 11 | selector: 'app-sidenav', 12 | templateUrl: './sidenav.component.html', 13 | styleUrls: ['./sidenav.component.scss'], 14 | }) 15 | export class SidenavComponent implements OnInit { 16 | navCollapsible: any; 17 | index; 18 | userRole; 19 | 20 | constructor(public env: EnvironmentsService, private userAuthService: UserAuthService) {} 21 | 22 | ngOnInit(): void { 23 | this.navCollapsible = true; 24 | if (!this.env.config.embedded) { 25 | this.userAuthService.loggedUserListener().subscribe((res) => { 26 | if (res) { 27 | this.userRole = res.user.role; 28 | } 29 | }); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /annotation-app/src/app/core.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule, Optional, SkipSelf } from '@angular/core'; 7 | import { HttpClientModule } from '@angular/common/http'; 8 | import { JwtModule, JWT_OPTIONS } from '@auth0/angular-jwt'; 9 | import { UserAuthService } from '../app/services/user-auth.service'; 10 | import { EnvironmentsService } from '../app/services/environments.service'; 11 | 12 | export function jwtOptionsFactory(authService: UserAuthService, env: EnvironmentsService) { 13 | return { 14 | tokenGetter: () => { 15 | // console.log('tokenGetter: ()---', authService.loggedUser()); 16 | if (authService.loggedUser()) { 17 | return authService.loggedUser().token ? authService.loggedUser().token.access_token : null; 18 | } 19 | return null; 20 | }, 21 | whitelistedDomains: [ 22 | `${env.config.annotationService}`.replace(/(http|https):\/\//, '').split('/')[0], 23 | `${env.config.authUrl}`.replace(/(http|https):\/\//, ''), 24 | `${env.config.inUrl}`.substring(0, `${env.config.inUrl}`.length - 8).replace(/(http|https):\/\//, ''), 25 | `${env.config.hubService}`.substring(0, `${env.config.hubService}`.length - 8).replace(/(http|https):\/\//, ''), 26 | ], 27 | }; 28 | } 29 | 30 | @NgModule({ 31 | declarations: [], 32 | imports: [ 33 | HttpClientModule, 34 | JwtModule.forRoot({ 35 | jwtOptionsProvider: { 36 | provide: JWT_OPTIONS, 37 | useFactory: jwtOptionsFactory, 38 | deps: [UserAuthService, EnvironmentsService], 39 | }, 40 | }), 41 | ], 42 | exports: [], 43 | }) 44 | export class CoreModule { 45 | constructor(@Optional() @SkipSelf() core: CoreModule) { 46 | if (core) { 47 | throw new Error('You should import core module only in the root module'); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /annotation-app/src/app/guards/auth.guard.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Injectable } from '@angular/core'; 7 | import { Router, CanActivate, ActivatedRouteSnapshot, RouterStateSnapshot } from '@angular/router'; 8 | import { UserAuthService } from 'src/app/services/user-auth.service'; 9 | import { Observable } from 'rxjs'; 10 | import { EnvironmentsService } from 'src/app/services/environments.service'; 11 | 12 | @Injectable({ providedIn: 'root' }) 13 | export class AuthGuard implements CanActivate { 14 | constructor(private router: Router, private userAuthService: UserAuthService, private env: EnvironmentsService) {} 15 | 16 | canActivate( 17 | route: ActivatedRouteSnapshot, 18 | state: RouterStateSnapshot, 19 | ): Observable | Promise | boolean { 20 | const user = this.userAuthService.loggedUser()?.user; 21 | if (user) { 22 | return true; 23 | } 24 | if (!this.env.config.embedded) { 25 | if (this.userAuthService.isLoggedIn()) { 26 | return this.checkRole(state.url); 27 | } else { 28 | if (this.env.config.authUrl) { 29 | this.userAuthService.redirectToLogin(); 30 | return false; 31 | } else { 32 | this.router.navigateByUrl('/login/basic'); 33 | } 34 | } 35 | } 36 | return false; 37 | } 38 | 39 | checkRole(url) { 40 | const role = JSON.parse(localStorage.getItem(this.env.config.sessionKey)).user.role; 41 | if (role && role != '') { 42 | if (role != 'Power User' && url.includes('permissions')) { 43 | this.router.navigate(['loop-home']); 44 | return false; 45 | } 46 | return true; 47 | } else { 48 | return false; 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /annotation-app/src/app/model/authentication.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | export enum SessionStatus { 7 | NOT_AUTHENTICATED, 8 | AUTHENTICATED, 9 | EXPIRED, 10 | } 11 | 12 | export interface AuthRequestToken { 13 | grant_type?: string; 14 | client_id?: string; 15 | redirect_uri?: string; 16 | code?: string; 17 | client_secret?: string; 18 | provider?: string; 19 | username?: string; 20 | password?: string; 21 | refresh_token?: string; 22 | bypass_uri_check?: boolean; 23 | } 24 | 25 | export interface AuthResponseToken { 26 | access_token?: string; 27 | access_type?: string; 28 | expires_in?: number; 29 | refresh_token?: string; 30 | expires_time?: number; 31 | } 32 | 33 | export interface AuthUser { 34 | token?: AuthResponseToken; 35 | user?: { 36 | role?: any; 37 | email?: string; 38 | provider?: string; 39 | sub?: string; 40 | username?: string; 41 | exp?: number; 42 | iat?: number; 43 | iss?: string; 44 | name?: string; 45 | }; 46 | } 47 | 48 | export class AuthUtil { 49 | public static isValidUser(user: AuthUser): boolean { 50 | if (user && user.user.email && user.token && user.token.access_token && user.token.refresh_token) { 51 | return true; 52 | } 53 | return false; 54 | } 55 | public static isValidBasicUser(user): boolean { 56 | if (user && user.user.email && user.token && user.token.access_token && user.token.expires_time) { 57 | return true; 58 | } 59 | return false; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /annotation-app/src/app/model/constant.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | export interface Incentive { 7 | points: number; 8 | donation: number; 9 | } 10 | 11 | export const ColorsRainbow = [ 12 | '#00ffff', 13 | '#ff00ff', 14 | '#00ff7f', 15 | '#ff6347', 16 | '#9B0D54', 17 | '#00bfff', 18 | '#FF0000', 19 | '#ff69b4', 20 | '#7fffd4', 21 | '#ffd700', 22 | '#FBC1DA', 23 | '#4D007A', 24 | '#ffdab9', 25 | '#adff2f', 26 | '#FFA500', 27 | '#FFFF00', 28 | '#583fcf', 29 | '#A32100', 30 | '#0F1E82', 31 | '#F89997', 32 | '#003D79', 33 | '#00D4B8', 34 | '#6C5F59', 35 | '#AADB1E', 36 | '#36C9E1', 37 | '#D0ACE4', 38 | '#798893', 39 | '#ED186F', 40 | '#D87093', 41 | '#DAA520', 42 | '#20B2AA', 43 | '#cb6360', 44 | '#c98886', 45 | '#703d3b', 46 | '#4c0b09', 47 | '#1890ff', 48 | '#189044', 49 | '#36c9d9', 50 | '#40a9ff', 51 | '#ff40a8', 52 | '#673ab7', 53 | '#faad14', 54 | '#03a9f4', 55 | '#9b4f4a', 56 | '#FF1493', 57 | '#228B22', 58 | '#0077b8', 59 | '#ff7875', 60 | '#97d778', 61 | '#2F4F4F', 62 | ]; 63 | 64 | export const PopLabelColors = [ 65 | '#55b128', 66 | '#d70c3b', 67 | '#3377dd', 68 | '#973633', 69 | '#f7a604', 70 | '#864ac1', 71 | '#09cbe5', 72 | '#a0f709', 73 | '#edf709', 74 | '#e9098f', 75 | ]; 76 | 77 | export const ClrTimelineStepState = { 78 | NOT_STARTED: 'not-started', 79 | CURRENT: 'current', 80 | SUCCESS: 'success', 81 | ERROR: 'error', 82 | PROCESSING: 'processing', 83 | }; 84 | 85 | export const Classifier = [ 86 | { name: 'RandomForestClassifier', value: 'RFC' }, 87 | { name: 'KNeighborsClassifier', value: 'KNC' }, 88 | { name: 'GradientBoostingClassifier', value: 'GBC' }, 89 | ]; 90 | 91 | export const Encoder = [ 92 | { name: ' One-Hot Encoding', value: 'oneHot' }, 93 | { name: 'Categorical Embeddings', value: 'embeddings' }, 94 | ]; 95 | 96 | export const QueryStrategyBase = [ 97 | { name: 'uncertainty_sampling', value: 'PB_UNS' }, 98 | { name: 'margin_sampling', value: 'PB_MS' }, 99 | { name: 'entropy_sampling', value: 'PB_ES' }, 100 | { name: 'uncertainty_batch_sampling', value: 'RBM_UNBS' }, 101 | ]; 102 | 103 | export const PopLabels = [ 104 | { name: 'Positive', value: 'Positive', setLableErrMessage: '' }, 105 | { name: 'Negative', value: 'Negative', setLableErrMessage: '' }, 106 | { name: 'Neutral', value: 'Neutral', setLableErrMessage: '' }, 107 | ]; 108 | -------------------------------------------------------------------------------- /annotation-app/src/app/model/env.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | export interface Env { 7 | embedded?: boolean; 8 | production?: boolean; 9 | annotationService?: string; 10 | redirectUrl?: string; 11 | serviceTitle?: string; 12 | provider?: string; 13 | USER_KEY?: string; 14 | STATE?: string; 15 | enableSendEmail?: boolean; 16 | authUrl?: string; 17 | tokenUrl?: string; 18 | logoutUrl?: string; 19 | CLIENT_ID?: string; 20 | lumosUrl?: string; 21 | googleTrackId?: any; 22 | enableAWSS3?: boolean; 23 | contactEmail?: string; 24 | contactSlack?: string; 25 | fileSize?: number; 26 | enableSlack?: boolean; 27 | slackAppName?: string; 28 | sessionKey?: string; 29 | inUrl?: string; 30 | videoSrc?: string; 31 | hubService?: string; 32 | } 33 | -------------------------------------------------------------------------------- /annotation-app/src/app/model/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | export * from './env'; 7 | export * from './dataset'; 8 | export * from './user'; 9 | export * from './constant'; 10 | -------------------------------------------------------------------------------- /annotation-app/src/app/model/sr.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | export interface SR { 7 | _id?: number; 8 | originalData?: any; 9 | problemCategory?: any; 10 | productName?: string; 11 | issueType?: string; 12 | resoltionCode?: string; 13 | resolution?: string; 14 | caseNumber?: string; 15 | userInputs?: UserInput[]; 16 | MSG?: string; 17 | flag?: any; 18 | images?: any; 19 | userInputsLength?: number; 20 | fileInfo?: any; 21 | ticketQuestions?: any; 22 | questionForText?: any; 23 | userInput?: UserInput[]; 24 | pid?: number; 25 | } 26 | export interface UserInput { 27 | problemCategory?: any; 28 | timestamp?: string; 29 | tid?: number; 30 | user?: string; 31 | logFreeText?: string; 32 | questionForText?: QaChat[]; 33 | } 34 | 35 | export interface SrUserInput { 36 | pid?: number; 37 | user?: string; 38 | userInput?: any; 39 | } 40 | 41 | export interface QaChat { 42 | prompt: string; 43 | response: string; 44 | reference?: any; 45 | followUps?: QaChat[]; 46 | } 47 | 48 | export class DatasetUtil { 49 | static initQaChat(): SR { 50 | return { 51 | userInput: [{ questionForText: [{ prompt: '', response: '', reference: [], followUps: [] }] }], 52 | }; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /annotation-app/src/app/model/user.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | export interface User { 7 | id?: string; 8 | email?: string; 9 | btoa?: string; 10 | fullName?: string; 11 | srs?: number[]; 12 | srCount?: number; 13 | optOutProducts?: string[]; 14 | points?: number; 15 | percentage?: number; 16 | } 17 | 18 | export const LabelStudio = undefined; 19 | -------------------------------------------------------------------------------- /annotation-app/src/app/pipes/full-name.pipe.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Pipe, PipeTransform } from '@angular/core'; 7 | 8 | /** 9 | * Remove extra numbers after the full name (those numbers comes fromt the authentication service) 10 | */ 11 | @Pipe({ name: 'fullNamePipe' }) 12 | export class FullNamePipe implements PipeTransform { 13 | transform(fullName: string): string { 14 | const match = /^([^\d]+)/.exec(fullName); 15 | return match && match[1] ? match[1] : fullName; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /annotation-app/src/app/pipes/math-round.pipe.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Pipe, PipeTransform } from '@angular/core'; 7 | 8 | /** 9 | * Remove extra numbers after the full name (those numbers comes fromt the authentication service) 10 | */ 11 | @Pipe({ name: 'mathRoundPipe' }) 12 | export class MathRoundPipe implements PipeTransform { 13 | transform(mathRound: number): string { 14 | const a = Math.round(mathRound); 15 | return String(a); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /annotation-app/src/app/pipes/sliceText.pipe.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Pipe, PipeTransform } from '@angular/core'; 7 | 8 | @Pipe({ name: 'SliceTextPipe' }) 9 | export class SliceTextPipe implements PipeTransform { 10 | transform(fullText: string): string { 11 | const sliceText = fullText.slice(0, 20) + '...'; 12 | return sliceText; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /annotation-app/src/app/services/cds-icon.service.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Injectable } from '@angular/core'; 7 | import '@cds/core/icon/register.js'; 8 | import { 9 | ClarityIcons, 10 | userIcon, 11 | unknownIcon, 12 | talkBubblesIcon, 13 | happyFaceIcon, 14 | tagsIcon, 15 | dataClusterIcon, 16 | helpIcon, 17 | infoStandardIcon, 18 | envelopeIcon, 19 | flaskIcon, 20 | boltIcon, 21 | rackServerIcon, 22 | successStandardIcon, 23 | lightbulbIcon, 24 | textIcon, 25 | unknownStatusIcon, 26 | bundleIcon, 27 | imageIcon, 28 | errorStandardIcon, 29 | tableIcon, 30 | uploadCloudIcon, 31 | fileIcon, 32 | windowCloseIcon, 33 | viewListIcon, 34 | downloadIcon, 35 | barsIcon, 36 | flagIcon, 37 | childArrowIcon, 38 | ellipsisVerticalIcon, 39 | exclamationCircleIcon, 40 | angleIcon, 41 | circleIcon, 42 | banIcon, 43 | checkIcon, 44 | noteIcon, 45 | trashIcon, 46 | uploadIcon, 47 | usersIcon, 48 | pencilIcon, 49 | plusIcon, 50 | checkboxListIcon, 51 | refreshIcon, 52 | sadFaceIcon, 53 | circleArrowIcon, 54 | timesCircleIcon, 55 | linkIcon, 56 | } from '@cds/core/icon'; 57 | import { ClrSpinner } from '@clr/angular'; 58 | 59 | ClarityIcons.addIcons( 60 | unknownIcon, 61 | userIcon, 62 | talkBubblesIcon, 63 | happyFaceIcon, 64 | tagsIcon, 65 | dataClusterIcon, 66 | helpIcon, 67 | infoStandardIcon, 68 | envelopeIcon, 69 | flaskIcon, 70 | boltIcon, 71 | rackServerIcon, 72 | successStandardIcon, 73 | lightbulbIcon, 74 | textIcon, 75 | unknownStatusIcon, 76 | bundleIcon, 77 | imageIcon, 78 | errorStandardIcon, 79 | tableIcon, 80 | uploadCloudIcon, 81 | fileIcon, 82 | windowCloseIcon, 83 | viewListIcon, 84 | downloadIcon, 85 | barsIcon, 86 | flagIcon, 87 | childArrowIcon, 88 | ellipsisVerticalIcon, 89 | exclamationCircleIcon, 90 | angleIcon, 91 | circleIcon, 92 | successStandardIcon, 93 | banIcon, 94 | checkIcon, 95 | noteIcon, 96 | trashIcon, 97 | uploadIcon, 98 | usersIcon, 99 | pencilIcon, 100 | plusIcon, 101 | checkboxListIcon, 102 | refreshIcon, 103 | sadFaceIcon, 104 | circleArrowIcon, 105 | timesCircleIcon, 106 | linkIcon, 107 | ); 108 | 109 | @Injectable({ 110 | providedIn: 'root', 111 | }) 112 | export class CdsIconsService { 113 | constructor() {} 114 | } 115 | -------------------------------------------------------------------------------- /annotation-app/src/app/services/common/download.service.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Injectable } from '@angular/core'; 7 | import * as _ from 'lodash'; 8 | import { EnvironmentsService } from 'src/app/services/environments.service'; 9 | import { WebAnalyticsService } from '../web-analytics.service'; 10 | 11 | @Injectable() 12 | export class DownloadService { 13 | constructor(public env: EnvironmentsService, private wa: WebAnalyticsService) {} 14 | 15 | downloadMultiple(urls) { 16 | urls.forEach((url, index) => { 17 | let hiddenIFrameID = 'hiddenDownloader' + index; 18 | let iframe = document.createElement('iframe'); 19 | iframe.id = hiddenIFrameID; 20 | iframe.style.display = 'none'; 21 | document.body.appendChild(iframe); 22 | iframe.src = this.env.config.enableAWSS3 23 | ? url 24 | : `${this.env.config.annotationService}/api/v1.0/datasets/download-from-local-system?file=${url}&token=${ 25 | JSON.parse(localStorage.getItem(this.env.config.sessionKey)).token.access_token 26 | }`; 27 | if (this.env.config.embedded && this.env.config.lumosUrl) { 28 | this.wa.toRecordDownloadWebAnalytics(url); 29 | } 30 | }); 31 | } 32 | 33 | downloadFile(url) { 34 | if (this.env.config.enableAWSS3) { 35 | window.location.href = url; 36 | if (this.env.config.embedded && this.env.config.lumosUrl) { 37 | this.wa.toRecordDownloadWebAnalytics(url); 38 | } 39 | } else { 40 | window.location.href = `${ 41 | this.env.config.annotationService 42 | }/api/v1.0/datasets/download-from-local-system?file=${url}&token=${ 43 | JSON.parse(localStorage.getItem(this.env.config.sessionKey)).token.access_token 44 | }`; 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /annotation-app/src/app/services/common/email.service.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Injectable } from '@angular/core'; 7 | import * as _ from 'lodash'; 8 | import { ToolService } from 'src/app/services/common/tool.service'; 9 | import { ApiService } from '../api.service'; 10 | import { InternalApiService } from '../internal-api.service'; 11 | 12 | @Injectable() 13 | export class EmailService { 14 | constructor( 15 | private toolService: ToolService, 16 | private apiService: ApiService, 17 | private internalApiService: InternalApiService, 18 | ) {} 19 | 20 | public sendEmailToAnnotator(param) { 21 | this.internalApiService.sendEmailToAnnotator(param).subscribe( 22 | (res) => {}, 23 | (error: any) => {}, 24 | ); 25 | } 26 | 27 | public sendEmailToOwner(param) { 28 | this.internalApiService.sendEmailToOwner(param).subscribe( 29 | (res) => {}, 30 | (error: any) => {}, 31 | ); 32 | } 33 | 34 | public sendEmail(inputProjectName, msg, ownerList, assigneeList) { 35 | const param: object = { 36 | pname: inputProjectName, 37 | fileName: msg.dataSource, 38 | }; 39 | const ownerDiff = _.difference(ownerList, msg.creator); 40 | let aa = []; 41 | assigneeList.forEach((element) => { 42 | aa.push(element.email); 43 | }); 44 | const annotatorDiff = _.difference(aa, msg.annotator); 45 | 46 | if (annotatorDiff.length > 0) { 47 | param['annotator'] = annotatorDiff; 48 | this.sendEmailToAnnotator(param); 49 | } 50 | if (ownerDiff.length > 0) { 51 | param['projectOwner'] = ownerDiff; 52 | this.sendEmailToOwner(param); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /annotation-app/src/app/services/common/markdown-parser.service.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Injectable } from '@angular/core'; 7 | import * as marked from 'marked'; 8 | 9 | @Injectable() 10 | export class MarkdownParserService { 11 | private md: any; 12 | constructor() { 13 | this.md = marked; 14 | this.md.setOptions({ 15 | renderer: new marked.Renderer(), 16 | pedantic: false, 17 | gfm: true, 18 | breaks: true, 19 | sanitize: false, 20 | smartLists: true, 21 | smartypants: false, 22 | xhtml: false, 23 | }); 24 | } 25 | 26 | convert(markdown) { 27 | return this.md.parse(markdown); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /annotation-app/src/app/services/common/tool.utils.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | function dateTransfer(time) { 7 | let date = new Date(time); 8 | let Y = date.getFullYear() + '-'; 9 | let M = (date.getMonth() + 1 < 10 ? '0' + (date.getMonth() + 1) : date.getMonth() + 1) + '-'; 10 | let D = (date.getDate() < 10 ? '0' + date.getDate() : date.getDate()) + ' '; 11 | let h = (date.getHours() < 10 ? '0' + date.getHours() : date.getHours()) + ':'; 12 | let m = (date.getMinutes() < 10 ? '0' + date.getMinutes() : date.getMinutes()) + ':'; 13 | let s = date.getSeconds() < 10 ? '0' + date.getSeconds() : date.getSeconds(); 14 | return Y + M + D + h + m + s; 15 | } 16 | 17 | -------------------------------------------------------------------------------- /annotation-app/src/app/services/environments.service.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Injectable } from '@angular/core'; 7 | import { NavigationEnd, Router } from '@angular/router'; 8 | 9 | declare let gtag: (parameterA: any, parameterB: any, parameterC: any) => void; 10 | 11 | @Injectable({ 12 | providedIn: 'root', 13 | }) 14 | export class EnvironmentsService { 15 | private configuration = '${APP_CONFIG}'; 16 | private env: any; 17 | private nodeEnvironment: string; 18 | constructor(public router: Router) { 19 | console.log('LOOP_APP_CONFIG:', this.configuration); 20 | this.nodeEnvironment = 21 | this.configuration === '' || this.configuration.startsWith('$') ? '' : `.${this.configuration}`; 22 | this.env = require('../../environments/environment' + this.nodeEnvironment); 23 | // Global site tag (gtag.js) - Google Analytics Start 24 | if (this.configuration == 'prod' && this.env.environment.googleTrackId) { 25 | this.router.events.subscribe((event) => { 26 | if (event instanceof NavigationEnd) { 27 | gtag('config', this.env.environment.googleTrackId, { 28 | page_path: event.urlAfterRedirects, 29 | }); 30 | } 31 | }); 32 | } 33 | } 34 | 35 | get config() { 36 | return this.env.environment; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /annotation-app/src/app/services/web-analytics.service.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Injectable } from '@angular/core'; 6 | declare var common: any; 7 | 8 | @Injectable() 9 | export class WebAnalyticsService { 10 | toRecordDownloadWebAnalytics(url: string) { 11 | var _paq = (common.lumos._paq = common.lumos._paq || []); 12 | _paq.push(['trackLink', url, 'download']); 13 | } 14 | 15 | toTrackEventWebAnalytics(category: string, action: string, name?: string, value?: number) { 16 | var _paq = (common.lumos._paq = common.lumos._paq || []); 17 | _paq.push(['trackEvent', category, action, name]); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /annotation-app/src/app/shared/clr-filter/datagridFilter.component.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | .clr-form { 6 | width: 20vw; 7 | 8 | .clr-control-container { 9 | width: 100%; 10 | 11 | .clr-input { 12 | width: 100%; 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /annotation-app/src/app/shared/clr-filter/datagridFilter.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { Component, EventEmitter, Input, Output, ViewChild, ElementRef } from '@angular/core'; 6 | import { ClrDatagridFilterInterface, ClrDatagridFilter } from '@clr/angular'; 7 | import { Subject } from 'rxjs'; 8 | import { debounceTime, distinctUntilChanged } from 'rxjs/operators'; 9 | import { ClrLoadingState } from '@clr/angular'; 10 | 11 | @Component({ 12 | selector: 'my-filter', 13 | templateUrl: './datagridFilter.component.html', 14 | styleUrls: ['./datagridFilter.component.scss'], 15 | }) 16 | export class MyFilter implements ClrDatagridFilterInterface { 17 | @Input() filterMsg?: any; 18 | @Input() filteredTotal; 19 | @Input() replaceStatus; 20 | @Output() filter = new EventEmitter(); 21 | @Output() replace = new EventEmitter(); 22 | 23 | inputString: string = ''; 24 | inputStringFilter = new Subject(); 25 | changes = new Subject(); 26 | inputString1: string = ''; 27 | loadingReplace: ClrLoadingState = ClrLoadingState.DEFAULT; 28 | 29 | constructor(private filterContainer: ClrDatagridFilter) { 30 | filterContainer.setFilter(this); 31 | this.inputStringFilter.pipe(debounceTime(400), distinctUntilChanged()).subscribe((value) => { 32 | this.loadingReplace = ClrLoadingState.DEFAULT; 33 | this.filter.emit(value.trim()); 34 | }); 35 | } 36 | 37 | ngOnChanges() { 38 | if (this.replaceStatus == 'succeed') { 39 | this.loadingReplace = ClrLoadingState.SUCCESS; 40 | } 41 | } 42 | 43 | isActive(): boolean { 44 | return !(this.inputString.trim().length == 0); 45 | } 46 | 47 | accepts() { 48 | return true; 49 | } 50 | 51 | change() { 52 | this.loadingReplace = ClrLoadingState.DEFAULT; 53 | } 54 | 55 | replaceAll() { 56 | if ( 57 | this.inputString.trim() && 58 | this.filteredTotal && 59 | this.filteredTotal > 0 && 60 | this.inputString1.trim() && 61 | this.inputString !== this.inputString1 62 | ) { 63 | this.loadingReplace = ClrLoadingState.LOADING; 64 | let data = { 65 | filter: this.inputString.trim(), 66 | replace: this.inputString1.trim(), 67 | }; 68 | this.replace.emit(data); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /annotation-app/src/app/shared/form-validators/form-validator-util.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { FormControl, FormGroup, FormArray, AbstractControl } from '@angular/forms'; 7 | 8 | export class FormValidatorUtil { 9 | static markControlsAsTouched(formElement: AbstractControl): void { 10 | if (formElement instanceof FormControl) { 11 | formElement.markAsTouched(); 12 | } else if (formElement instanceof FormGroup) { 13 | Object.keys(formElement.controls).forEach((key) => { 14 | this.markControlsAsTouched(formElement.get(key)!); 15 | }); 16 | } else if (formElement instanceof FormArray) { 17 | formElement.controls.forEach((control) => { 18 | this.markControlsAsTouched(control); 19 | }); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /annotation-app/src/app/shared/modal-confirm/modal-confirm.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 |
5 | 27 |
28 | 29 | -------------------------------------------------------------------------------- /annotation-app/src/app/shared/modal-confirm/modal-confirm.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, Input, OnInit, Output, EventEmitter } from '@angular/core'; 7 | 8 | @Component({ 9 | selector: 'app-modal-confirm', 10 | templateUrl: './modal-confirm.component.html', 11 | }) 12 | export class ModalConfirmComponent implements OnInit { 13 | @Input() msg; 14 | @Output('cancelBtn') 15 | onCloseConfirmDialogEmitter = new EventEmitter(); 16 | @Output('okBtn') 17 | okBtnEmitter = new EventEmitter(); 18 | 19 | showModal = true; 20 | clickOkBtn = false; 21 | constructor() {} 22 | 23 | ngOnInit(): void {} 24 | 25 | cancelBtn() { 26 | this.onCloseConfirmDialogEmitter.emit(true); 27 | } 28 | okBtn() { 29 | this.clickOkBtn = true; 30 | this.okBtnEmitter.emit(true); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /annotation-app/src/app/shared/shared.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { NgModule } from '@angular/core'; 7 | import { CommonModule } from '@angular/common'; 8 | import { FormsModule, ReactiveFormsModule } from '@angular/forms'; 9 | import { ClarityModule } from '@clr/angular'; 10 | import { RouterModule } from '@angular/router'; 11 | import { ModalConfirmComponent } from './modal-confirm/modal-confirm.component'; 12 | import { MyFilter } from './clr-filter/datagridFilter.component'; 13 | import { FullNamePipe } from '../pipes/full-name.pipe'; 14 | import { MathRoundPipe } from '../pipes/math-round.pipe'; 15 | import { SliceTextPipe } from '../pipes/sliceText.pipe'; 16 | import { UploadFileComponent } from '../component/datasets/upload-file/upload-file.component'; 17 | import { DndDirective } from '../component/datasets/dnd.directive'; 18 | import { CreateNewDatasetModalComponent } from '../component/datasets/create-new-dataset-modal/create-new-dataset-modal.component'; 19 | import { CreateNewDatasetComponent } from '../component/datasets/create-new-dataset/create-new-dataset.component'; 20 | 21 | @NgModule({ 22 | declarations: [ 23 | ModalConfirmComponent, 24 | MyFilter, 25 | FullNamePipe, 26 | MathRoundPipe, 27 | SliceTextPipe, 28 | UploadFileComponent, 29 | DndDirective, 30 | CreateNewDatasetComponent, 31 | CreateNewDatasetModalComponent, 32 | ], 33 | imports: [CommonModule, ClarityModule, FormsModule, ReactiveFormsModule, RouterModule], 34 | exports: [ 35 | FormsModule, 36 | CommonModule, 37 | ReactiveFormsModule, 38 | ClarityModule, 39 | ModalConfirmComponent, 40 | MyFilter, 41 | FullNamePipe, 42 | MathRoundPipe, 43 | SliceTextPipe, 44 | UploadFileComponent, 45 | DndDirective, 46 | CreateNewDatasetModalComponent, 47 | CreateNewDatasetComponent, 48 | ], 49 | }) 50 | export class SharedModule {} 51 | -------------------------------------------------------------------------------- /annotation-app/src/app/shared/utils/index.es5.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | export default function insertAfter(node: any, _ref: any) { 6 | var nextSibling = _ref.nextSibling, 7 | parentNode = _ref.parentNode; 8 | 9 | return parentNode.insertBefore(node, nextSibling); 10 | } 11 | -------------------------------------------------------------------------------- /annotation-app/src/app/shared/utils/treeView.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | export function filterTreeLabel(treeArr: any) { 6 | const findItem = (arr: any) => { 7 | let res = []; 8 | if (arr && arr.length > 0) { 9 | res = arr.filter((item: any) => { 10 | if (item.children && item.children.length > 0) { 11 | item.children = childFilter(item.children); 12 | } 13 | return item && item.children && item.children.length ? (item.enable = 1) : item.enable; 14 | }); 15 | } 16 | return res; 17 | }; 18 | const childFilter = (childArr: any) => { 19 | return childArr.filter((item: any) => { 20 | if (item.children && item.children.length > 0) { 21 | item.children = childFilter(item.children); 22 | } 23 | return item && item.children && item.children.length ? (item.enable = 1) : item.enable; 24 | }); 25 | }; 26 | return findItem(treeArr); 27 | } 28 | -------------------------------------------------------------------------------- /annotation-app/src/assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/annotation-app/src/assets/.gitkeep -------------------------------------------------------------------------------- /annotation-app/src/assets/files/taxonomy_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | { 4 | "name": "Products", 5 | "children": [ 6 | { 7 | "name": "vSphere", 8 | "children": [ 9 | { 10 | "name": "vSphere1" 11 | }, 12 | { 13 | "name": "vSphere2" 14 | }, 15 | { 16 | "name": "vSphere3" 17 | }, 18 | { 19 | "name": "vSphere4" 20 | } 21 | ] 22 | }, 23 | { 24 | "name": "vMware", 25 | "children": [ 26 | { 27 | "name": "vMware1" 28 | }, 29 | { 30 | "name": "vMware2" 31 | } 32 | ] 33 | }, 34 | { 35 | "name": "FS" 36 | }, 37 | { 38 | "name": "Stack" 39 | } 40 | ] 41 | }, 42 | { 43 | "name": "Customer1" 44 | }, 45 | { 46 | "name": "Customer2" 47 | } 48 | ] 49 | } 50 | -------------------------------------------------------------------------------- /annotation-app/src/assets/files/taxonomy_sample.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2023 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | data: 4 | - name: Products 5 | children: 6 | - name: vSphere 7 | children: 8 | - name: vSphere1 9 | - name: vSphere2 10 | - name: vSphere3 11 | - name: vSphere4 12 | - name: vMware 13 | children: 14 | - name: vMware1 15 | - name: vMware2 16 | - name: FS 17 | - name: Stack 18 | - name: Customer1 19 | - name: Customer2 20 | -------------------------------------------------------------------------------- /annotation-app/src/assets/images/ava-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/annotation-app/src/assets/images/ava-logo.png -------------------------------------------------------------------------------- /annotation-app/src/assets/images/ava-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/annotation-app/src/assets/images/ava-small.png -------------------------------------------------------------------------------- /annotation-app/src/assets/images/demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/annotation-app/src/assets/images/demo.png -------------------------------------------------------------------------------- /annotation-app/src/assets/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/annotation-app/src/assets/images/favicon.ico -------------------------------------------------------------------------------- /annotation-app/src/assets/images/polygon.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 6 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /annotation-app/src/assets/images/rect.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 6 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /annotation-app/src/assets/images/refresh.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 6 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /annotation-app/src/assets/images/trash.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 6 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /annotation-app/src/assets/images/vm-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/annotation-app/src/assets/images/vm-logo.png -------------------------------------------------------------------------------- /annotation-app/src/dev-platform/dev-platform.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { CommonModule } from '@angular/common'; 7 | import { NgModule } from '@angular/core'; 8 | import { FormsModule } from '@angular/forms'; 9 | import { RouterModule } from '@angular/router'; 10 | import { AppModule } from '../app/app.module'; 11 | import { DevPlatformComponent } from './dev-platform/dev-platform.component'; 12 | 13 | @NgModule({ 14 | declarations: [DevPlatformComponent], 15 | imports: [ 16 | CommonModule, 17 | FormsModule, 18 | RouterModule.forRoot([ 19 | { 20 | path: '', 21 | pathMatch: 'full', 22 | component: DevPlatformComponent, 23 | data: { title: 'loop' }, 24 | }, 25 | ]), 26 | AppModule, 27 | ], 28 | bootstrap: [DevPlatformComponent], 29 | }) 30 | export class DevPlatformModule {} 31 | -------------------------------------------------------------------------------- /annotation-app/src/dev-platform/dev-platform/dev-platform.component.css: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2024 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ -------------------------------------------------------------------------------- /annotation-app/src/dev-platform/dev-platform/dev-platform.component.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /annotation-app/src/dev-platform/dev-platform/dev-platform.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, OnInit } from '@angular/core'; 7 | import { Title } from '@angular/platform-browser'; 8 | import { ActivatedRoute, NavigationEnd, Router } from '@angular/router'; 9 | import { filter, map } from 'rxjs/operators'; 10 | 11 | @Component({ 12 | selector: 'mf-dev-platform', 13 | templateUrl: './dev-platform.component.html', 14 | styleUrls: ['./dev-platform.component.css'], 15 | }) 16 | export class DevPlatformComponent implements OnInit { 17 | constructor(private router: Router, private titleService: Title) {} 18 | 19 | ngOnInit(): void { 20 | this.updatePageTitleOnRouteChange(); 21 | } 22 | 23 | private updatePageTitleOnRouteChange() { 24 | this.router.events 25 | .pipe( 26 | filter((event) => event instanceof NavigationEnd), 27 | map(() => { 28 | let route: ActivatedRoute = this.router.routerState.root; 29 | let routeTitle = ''; 30 | while (route!.firstChild) { 31 | route = route.firstChild; 32 | } 33 | if (route.snapshot.data['title']) { 34 | routeTitle = route!.snapshot.data['title']; 35 | } 36 | return routeTitle; 37 | }), 38 | ) 39 | .subscribe((title) => { 40 | if (title) { 41 | this.titleService.setTitle(`${title} - Microtrains`); 42 | } 43 | }); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /annotation-app/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Env } from 'src/app/model/index'; 7 | 8 | export const environment: Env = { 9 | production: true, 10 | }; 11 | -------------------------------------------------------------------------------- /annotation-app/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Env } from 'src/app/model/index'; 7 | 8 | export const environment: Env = { 9 | // This section is required 10 | production: false, 11 | annotationService: 'http://localhost:3000', // Annotation service url 12 | // This section is optional 13 | serviceTitle: 'Data Annotator for Machine Learning', // UI name of annotation-app. 14 | googleTrackId: null, // google track ID 15 | redirectUrl: '/', // redirect URL after logout or token is expired 16 | enableSendEmail: false, // Set to true to enable email notification for project creation, annotator assignment or edit project creator 17 | enableAWSS3: false, // Set to true to upload and download files with AWS S3 that requires some related AWS CONFIG IAM to be configured in annotation-service 18 | enableSlack: false, // Set to true to allow annotator use slack to post message and do annotate. And please follow the path annotation-service/config/app-os.js to set buildSlackApp=true at the same time. While first of all, in order to support this feature you need to create an app from an app manifest which containing basic info, scopes, settings, and features. You can find this yaml file in the path vmware/data-annotator-for-machine-learning/master/docs/manifest.yml. 19 | slackAppName: '', // If enableSlack=true, here should be the slack app name 20 | embedded: false, // Set to true to wrap the service into angular custom element 'mf-loop-entry', and run the script bundle_ to build one complete static js which is required for an micro-front-end app 21 | sessionKey: 'app-session', // All token and user info is saved after this key name in local storage 22 | }; 23 | -------------------------------------------------------------------------------- /annotation-app/src/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/annotation-app/src/favicon.ico -------------------------------------------------------------------------------- /annotation-app/src/index.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /annotation-app/src/main.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 7 | import { DevPlatformModule } from './dev-platform/dev-platform.module'; 8 | import { environment } from './environments/environment'; 9 | import { MicroFrontendModule } from './micro-frontend/micro-frontend.module'; 10 | import { enableProdMode } from '@angular/core'; 11 | 12 | if (environment.production && environment.googleTrackId) { 13 | enableProdMode(); 14 | // Global site tag (gtag.js) - Google Analytics 15 | const tp = `document.getElementById('track').src ='https://www.googletagmanager.com/gtag/js?id=${environment.googleTrackId}';window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}gtag('js',new Date());`; 16 | const script1 = document.createElement('script'); 17 | script1.async = true; 18 | script1.id = 'track'; 19 | document.head.appendChild(script1); 20 | const script2 = document.createElement('script'); 21 | script2.innerText = tp; 22 | document.head.appendChild(script2); 23 | } 24 | 25 | const bootstrapModule = environment.embedded ? MicroFrontendModule : DevPlatformModule; 26 | 27 | platformBrowserDynamic() 28 | .bootstrapModule(bootstrapModule) 29 | .catch((err) => console.error('platformBrowserDynamic:::', err)); 30 | -------------------------------------------------------------------------------- /annotation-app/src/micro-frontend/entry.component.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | import { Component, EventEmitter, Input, OnChanges, OnDestroy, Output, SimpleChanges } from '@angular/core'; 7 | import { Router, RoutesRecognized } from '@angular/router'; 8 | import { Subscription } from 'rxjs'; 9 | 10 | export interface RouterEvent { 11 | url: string; 12 | replaceUrl: boolean; 13 | } 14 | 15 | @Component({ 16 | template: '', 17 | }) 18 | export class EntryComponent implements OnChanges, OnDestroy { 19 | @Input() route?: string; 20 | @Output() routeChange = new EventEmitter(); 21 | 22 | private subscription: Subscription; 23 | 24 | constructor(private router: Router) { 25 | const routingSubscription = this.registerOutgoingRouting(); 26 | this.subscription = routingSubscription; 27 | } 28 | 29 | ngOnChanges(changes: SimpleChanges) { 30 | if (changes['route'] && this.route) { 31 | this.router.navigateByUrl(this.route, { state: { fromPlatform: true } }); 32 | } 33 | } 34 | 35 | ngOnDestroy(): void { 36 | this.subscription.unsubscribe(); 37 | } 38 | 39 | private registerOutgoingRouting(): Subscription { 40 | return this.router.events.subscribe((event) => { 41 | if (event instanceof RoutesRecognized && (!this.isRouteChangeFromPlatform() || this.isRedirect(event))) { 42 | this.routeChange.next({ 43 | url: event.urlAfterRedirects, 44 | replaceUrl: this.isRedirect(event), 45 | }); 46 | } 47 | }); 48 | } 49 | 50 | private isRouteChangeFromPlatform(): boolean { 51 | return this.router.getCurrentNavigation()?.extras?.state?.['fromPlatform']; 52 | } 53 | 54 | private isRedirect(event: RoutesRecognized): boolean { 55 | return event.url !== event.urlAfterRedirects; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /annotation-app/src/micro-frontend/micro-frontend.module.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | import { DoBootstrap, Injector, NgModule } from '@angular/core'; 6 | import { createCustomElement } from '@angular/elements'; 7 | import { AppModule } from '../app/app.module'; 8 | import { EntryComponent } from './entry.component'; 9 | @NgModule({ 10 | declarations: [EntryComponent], 11 | imports: [AppModule], 12 | }) 13 | export class MicroFrontendModule implements DoBootstrap { 14 | constructor(private injector: Injector) {} 15 | 16 | ngDoBootstrap(): void { 17 | const customElement = createCustomElement(EntryComponent, { 18 | injector: this.injector, 19 | }); 20 | window.customElements.define('mf-loop-entry', customElement); 21 | console.log('Registered custom element mf-loop-entry'); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /annotation-app/src/polyfills.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | /** 6 | * This file includes polyfills needed by Angular and is loaded before the app. 7 | * You can add your own extra polyfills to this file. 8 | * 9 | * This file is divided into 2 sections: 10 | * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. 11 | * 2. Application imports. Files imported after ZoneJS that should be loaded before your main 12 | * file. 13 | * 14 | * The current setup is for so-called "evergreen" browsers; the last versions of browsers that 15 | * automatically update themselves. This includes recent versions of Safari, Chrome (including 16 | * Opera), Edge on the desktop, and iOS and Chrome on mobile. 17 | * 18 | * Learn more in https://angular.io/guide/browser-support 19 | */ 20 | 21 | /*************************************************************************************************** 22 | * BROWSER POLYFILLS 23 | */ 24 | 25 | /** 26 | * By default, zone.js will patch all possible macroTask and DomEvents 27 | * user can disable parts of macroTask/DomEvents patch by setting following flags 28 | * because those flags need to be set before `zone.js` being loaded, and webpack 29 | * will put import in the top of bundle, so user need to create a separate file 30 | * in this directory (for example: zone-flags.ts), and put the following flags 31 | * into that file, and then add the following code before importing zone.js. 32 | * import './zone-flags'; 33 | * 34 | * The flags allowed in zone-flags.ts are listed here. 35 | * 36 | * The following flags will work for all browsers. 37 | * 38 | * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame 39 | * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick 40 | * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames 41 | * 42 | * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js 43 | * with the following flag, it will bypass `zone.js` patch for IE/Edge 44 | * 45 | * (window as any).__Zone_enable_cross_context_check = true; 46 | * 47 | */ 48 | 49 | /*************************************************************************************************** 50 | * Zone JS is required by default for Angular itself. 51 | */ 52 | import 'zone.js'; // Included with Angular CLI. 53 | (window as any).global = window; 54 | 55 | /*************************************************************************************************** 56 | * APPLICATION IMPORTS 57 | */ 58 | -------------------------------------------------------------------------------- /annotation-app/src/styles.scss: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019-2023 VMware, Inc. 3 | SPDX-License-Identifier: Apache-2.0 4 | */ 5 | /* You can add global styles to this file, and also import other style files */ 6 | @import '~@ng-select/ng-select/themes/material.theme.css'; 7 | 8 | a { 9 | color: currentColor; 10 | } -------------------------------------------------------------------------------- /annotation-app/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/app", 6 | "types": [] 7 | }, 8 | "files": [ 9 | "src/main.ts", 10 | "src/polyfills.ts" 11 | ], 12 | "include": [ 13 | "src/environments/*.*.ts", 14 | "src/**/*.d.ts" 15 | ] 16 | } -------------------------------------------------------------------------------- /annotation-app/tsconfig.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "compileOnSave": false, 4 | "compilerOptions": { 5 | "baseUrl": "./", 6 | "outDir": "./dist/out-tsc", 7 | "forceConsistentCasingInFileNames": true, 8 | // "strict": true, 9 | "noImplicitOverride": true, 10 | "noPropertyAccessFromIndexSignature": true, 11 | "noImplicitReturns": false, 12 | "noFallthroughCasesInSwitch": true, 13 | "sourceMap": true, 14 | "declaration": false, 15 | "downlevelIteration": true, 16 | "experimentalDecorators": true, 17 | "moduleResolution": "node", 18 | "importHelpers": true, 19 | "target": "es2020", 20 | "module": "es2020", 21 | "lib": [ 22 | "es2020", 23 | "dom" 24 | ] 25 | }, 26 | "angularCompilerOptions": { 27 | "enableI18nLegacyMessageIdFormat": false, 28 | // "strictInjectionParameters": true, 29 | // "strictInputAccessModifiers": true, 30 | // "strictTemplates": true 31 | }, 32 | } -------------------------------------------------------------------------------- /annotation-app/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint:recommended", "tslint-angular", "tslint-config-prettier"], 3 | "rules": { 4 | "array-type": false, 5 | "arrow-parens": false, 6 | "deprecation": { 7 | "severity": "warn" 8 | }, 9 | "component-class-suffix": true, 10 | "contextual-lifecycle": true, 11 | "directive-class-suffix": true, 12 | "directive-selector": [true, "attribute", "app", "camelCase"], 13 | "component-selector": [true, "element", "app", "kebab-case"], 14 | "import-blacklist": [true, "rxjs/Rx"], 15 | "interface-name": false, 16 | "max-classes-per-file": false, 17 | "max-line-length": [true, 140], 18 | "member-access": false, 19 | "member-ordering": [ 20 | true, 21 | { 22 | "order": ["static-field", "instance-field", "static-method", "instance-method"] 23 | } 24 | ], 25 | "no-bitwise": false, 26 | "no-consecutive-blank-lines": false, 27 | "no-console": [true, "debug", "info", "time", "timeEnd", "trace"], 28 | "no-empty": false, 29 | "no-inferrable-types": [true, "ignore-params"], 30 | "no-non-null-assertion": false, 31 | "no-redundant-jsdoc": true, 32 | "no-switch-case-fall-through": true, 33 | "no-var-requires": false, 34 | "object-literal-key-quotes": [true, "as-needed"], 35 | "object-literal-sort-keys": false, 36 | "ordered-imports": false, 37 | "quotemark": [true, "single"], 38 | "trailing-comma": false, 39 | "no-conflicting-lifecycle": true, 40 | "no-host-metadata-property": true, 41 | "no-input-rename": true, 42 | "no-inputs-metadata-property": true, 43 | "no-output-native": true, 44 | "no-output-on-prefix": false, 45 | "no-output-rename": false, 46 | "no-outputs-metadata-property": true, 47 | "template-banana-in-box": true, 48 | "template-no-negated-async": true, 49 | "triple-equals": [false, "allow-null-check"], 50 | "use-lifecycle-interface": true, 51 | "use-pipe-transform-interface": true, 52 | "prefer-for-of": false, 53 | "only-arrow-functions": [false], 54 | "no-shadowed-variable": false, 55 | "prefer-const": false, 56 | "no-eval": false 57 | }, 58 | "rulesDirectory": ["codelyzer"] 59 | } 60 | -------------------------------------------------------------------------------- /annotation-service/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | # Create image based on the official Node 6 image from the dockerhub 5 | FROM node:16-alpine 6 | 7 | COPY package*.json . 8 | 9 | RUN npm i && mkdir /app && cp -R ./node_modules ./app 10 | 11 | WORKDIR /app 12 | 13 | COPY . . 14 | 15 | # Expose the port the app runs in 16 | EXPOSE 3000 17 | 18 | # Serve the app 19 | CMD ["npm", "run", "serve"] 20 | -------------------------------------------------------------------------------- /annotation-service/config/config.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | const sysEnv = process.env.SYS_ENV || "os"; 9 | const app = require(`./app-${sysEnv}`); 10 | 11 | module.exports = app 12 | 13 | console.log('----------------------------------------------------------------------------------------'); 14 | console.log(`[ CONFIG ] [ SYS_ENV ]=${sysEnv} [app]=`, module.exports); 15 | console.log('----------------------------------------------------------------------------------------'); 16 | -------------------------------------------------------------------------------- /annotation-service/middlewares/jwt.middleware.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | const jwt = require('express-jwt'); 9 | const config = require('../config/config'); 10 | const JWTS = require('jsonwebtoken'); 11 | const APIs = require('../resources/APIs'); 12 | 13 | jwtTokenAuthrization = (data) => { 14 | if (config.ESP) { 15 | return jwt({ 16 | secret: data.key.replace(/RSA /g, ''), 17 | algorithms: [data.alg], 18 | issuer: config.tokenIssuer, 19 | getToken: fromHeaderOrQuerystring, 20 | requestProperty: 'auth', 21 | }).unless({ 22 | path: [ 23 | `${config.API_BASE_PATH}/api/${config.API_VERSION}${APIs.EMAIL_REGULAR_NOTIFICATION}`, 24 | ], 25 | }); 26 | } else { 27 | return jwt({ 28 | secret: config.TOKEN_SECRET_OR_PRIVATE_KEY, 29 | algorithms: [config.TOKEN_ALGORITHM], 30 | getToken: fromHeaderOrQuerystring, 31 | requestProperty: 'auth', 32 | }).unless({ 33 | path: [ 34 | `${config.API_BASE_PATH}/api/${config.API_VERSION}${APIs.REGISTER}`, 35 | `${config.API_BASE_PATH}/api/${config.API_VERSION}${APIs.LOGIN}`, 36 | `${config.API_BASE_PATH}/api/${config.API_VERSION}${APIs.EMAIL_REGULAR_NOTIFICATION}`, 37 | ], 38 | }); 39 | } 40 | 41 | }; 42 | 43 | function fromHeaderOrQuerystring(req) { 44 | 45 | if (req.headers.authorization && req.headers.authorization.split(' ')[0] === 'Bearer') { 46 | return req.headers.authorization.split(' ')[1]; 47 | } else if (req.query && req.query.token) { 48 | req.headers.authorization = "Bearer " + req.query.token; 49 | return req.query.token; 50 | } 51 | return null; 52 | } 53 | 54 | async function generateBasicToken(user) { 55 | 56 | const expires_time = Math.floor(Date.now() / 1000) + config.TOKEN_EXPIRE_TIME; 57 | const access_token = await JWTS.sign({ 58 | exp: expires_time, 59 | email: user 60 | }, 61 | config.TOKEN_SECRET_OR_PRIVATE_KEY, 62 | { 63 | algorithm: config.TOKEN_ALGORITHM 64 | } 65 | ); 66 | return { 67 | access_token: access_token, 68 | access_type: "Bearer", 69 | expires_in: config.TOKEN_EXPIRE_TIME, 70 | expires_time: expires_time 71 | } 72 | } 73 | 74 | module.exports = { 75 | jwtTokenAuthrization, 76 | generateBasicToken, 77 | } -------------------------------------------------------------------------------- /annotation-service/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "annotation-service", 3 | "version": "1.0.0", 4 | "private": true, 5 | "scripts": { 6 | "serve": "node server.js", 7 | "start": "nodemon server.js" 8 | }, 9 | "dependencies": { 10 | "@slack/bolt": "^3.11.1", 11 | "@slack/web-api": "^6.7.1", 12 | "aws-sdk": "^2.849.0", 13 | "axios": "^0.24.0", 14 | "body-parser": "^1.19.0", 15 | "compressing": "^1.5.1", 16 | "cors": "2.8.5", 17 | "cron": "^2.0.0", 18 | "csv-writer": "^1.6.0", 19 | "csvtojson": "^2.0.10", 20 | "express": "^4.17.1", 21 | "express-jwt": "^6.0.0", 22 | "hosted-git-info": ">=2.8.9", 23 | "jsonwebtoken": "^8.5.1", 24 | "lodash": "^4.17.21", 25 | "moment": "^2.29.1", 26 | "mongodb": "^3.6.4", 27 | "mongoose": "^5.11.17", 28 | "mongoose-paginate-v2": "^1.8.0", 29 | "multer": "^1.4.2", 30 | "nodemailer": "^6.4.18", 31 | "path": "0.12.7", 32 | "request": "^2.88.2", 33 | "sqs-consumer": "^5.5.0", 34 | "swagger-ui-express": "^4.1.6", 35 | "universal-analytics": "^0.4.23", 36 | "unzip-stream": "^0.3.1" 37 | }, 38 | "devDependencies": { 39 | "nodemon": "^2.0.7" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /annotation-service/resources/template-annotator.html: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 |

Hi,

10 |

A new ${team} annotation task has been assigned to you by 11 | ${projectOwner} for the 12 | ${projectName} project. 13 |

14 |

Click on the link above to view your annotation task. You may need to login first.

15 |

Kind regards,

16 |

${team} Team

17 |
18 |
19 |
-------------------------------------------------------------------------------- /annotation-service/resources/template-generater.html: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 |

Hi,

10 |

You have received a message from ${team} regarding:

11 |

Your dataset ${fileName} is ready for download.

12 |

You may need click the link to login first.

13 |

Kind regards,

14 |

${team} Team

15 |
16 |
17 |
-------------------------------------------------------------------------------- /annotation-service/resources/template-notFinish.html: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 |

Hi ${user},

10 |

Your annotation task ${projectName} assigned to you on ${assignedDate} by ${projectOwner} is nearly complete.

11 |

Please visit the annotations page to finish your task.

12 |

If you don't want to receive such emails in the future, please unsubscribe from here: STOP CURRENT PROJECT REGULAR NOTIFICATION for this project.

13 |

You can also unsubscribe such notification from all projects from here: STOP ALL PROJECTS REGULAR NOTIFICATION.

14 |

Regards,

15 |

${team} Team

16 |
17 |
18 |
-------------------------------------------------------------------------------- /annotation-service/resources/template-notStart.html: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 |

Hi ${user},

10 |

A new annotation task has been assigned to you by ${projectOwner} for the ${projectName} project at ${assignedDate}.

11 |

Please click on the link above and start the annotation task.

12 |

If you don't want to receive such emails in the future, please unsubscribe from here: STOP CURRENT PROJECT REGULAR NOTIFICATION for this project.

13 |

You can also unsubscribe such notification from all projects from here: STOP ALL PROJECTS REGULAR NOTIFICATION.

14 |

Regards,

15 |

${team} Team

16 |
17 |
18 |
-------------------------------------------------------------------------------- /annotation-service/resources/template-owner.html: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 |
9 |

Hi,

10 |

You have received a message from ${team} regarding:

11 |

Your project, ${projectName} has been created successfully which generates all annotation cases from ${fileName}

12 |

Click on the link above to view your annotation project. You may need to login first.

13 |

Kind regards,

14 |

${team} Team

15 |
16 |
17 |
-------------------------------------------------------------------------------- /annotation-service/routers/auth-controller.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const express = require("express"); 10 | const router = express.Router(); 11 | const userService = require("../services/user-service"); 12 | const APIs = require('../resources/APIs'); 13 | const authService = require("../services/auth.service"); 14 | 15 | router.put(APIs.REGISTER, (req, res) => { 16 | console.log(`[ PROJECT ] [ ACCESS ] Router ${req.originalUrl}`); 17 | userService.saveUser(req).then(response => { 18 | console.log(`[ PROJECT ] [ SUCCESS ] Router ${req.originalUrl}`); 19 | res.status(200).json(response); 20 | }).catch(error => { 21 | console.error(`[ PROJECT ] [ ERROR ] Router ${req.originalUrl}`, error); 22 | res.status(500).send(error); 23 | }); 24 | }); 25 | 26 | router.post(APIs.LOGIN, (req, res) => { 27 | console.log(`[ PROJECT ] [ ACCESS ] Router ${req.originalUrl}`); 28 | authService.login(req).then(response => { 29 | console.log(`[ PROJECT ] [ SUCCESS ] Router ${req.originalUrl}`); 30 | res.status(200).json(response); 31 | }).catch(error => { 32 | console.error(`[ PROJECT ] [ ERROR ] Router ${req.originalUrl}`, error); 33 | res.status(500).send(error); 34 | }); 35 | }); 36 | 37 | router.get(APIs.TOKEN_REFRESH, (req, res) => { 38 | console.log(`[ PROJECT ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 39 | authService.refreshToken(req).then(response => { 40 | console.log(`[ PROJECT ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 41 | res.status(200).json(response); 42 | }).catch(error => { 43 | console.error(`[ PROJECT ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 44 | res.status(500).send(error); 45 | }); 46 | }); 47 | 48 | module.exports = router; -------------------------------------------------------------------------------- /annotation-service/routers/community-controller.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const express = require("express"); 10 | const router = express.Router(); 11 | const APIs = require('../resources/APIs'); 12 | const communityService = require('../services/community.service'); 13 | 14 | 15 | router.post(APIs.COUNT_COMMUNITY_DOWNLOAD, (req, res) => { 16 | console.log(`[ FILE ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 17 | communityService.countCommunityDownload(req).then((response) => { 18 | console.log(`[ FILE ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 19 | res.status(200).json(response); 20 | }).catch(error => { 21 | console.error(`[ FILE ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 22 | res.status(500).send(error); 23 | }); 24 | }); 25 | 26 | module.exports = router; -------------------------------------------------------------------------------- /annotation-service/routers/db-controller.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const express = require("express"); 10 | const router = express.Router(); 11 | const APIs = require('../resources/APIs'); 12 | const DBOP = require('../utils/DB.OPERATIONS'); 13 | 14 | 15 | router.post(APIs.DB_UPDATE_COLUMN_TYPE, (req, res) => { 16 | console.log(`[ DB ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 17 | DBOP.updateDBColumnType(req).then((response) => { 18 | console.log(`[ DB ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 19 | res.status(200).json(response); 20 | }).catch(error => { 21 | console.error(`[ DB ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 22 | res.status(500).send(error); 23 | }); 24 | }); 25 | 26 | 27 | module.exports = router; -------------------------------------------------------------------------------- /annotation-service/routers/email-controller.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const express = require("express"); 10 | const router = express.Router(); 11 | const emailService = require('../services/email-service'); 12 | const APIs = require('../resources/APIs'); 13 | const taskSchedule = require('../utils/taskSchedule'); 14 | const config = require('../config/config'); 15 | 16 | router.post(APIs.EMAIL_TO_OWNER, (req, res) => { 17 | console.log(`[ EMAIL ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 18 | emailService.sendEmailToOwner(req).then(() => { 19 | console.log(`[ EMAIL ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 20 | res.status(200).json("email ok"); 21 | }).catch(error => { 22 | console.error(`[ EMAIL ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 23 | res.status(500).json(error); 24 | }); 25 | }); 26 | 27 | 28 | router.post(APIs.EMAIL_TO_ANNOTATOR, (req, res) => { 29 | console.log(`[ EMAIL ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 30 | emailService.sendEmailToAnnotator(req).then(() => { 31 | console.log(`[ EMAIL ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 32 | res.status(200).json("email ok"); 33 | }).catch(error => { 34 | console.error(`[ EMAIL ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 35 | res.status(500).json(error); 36 | }); 37 | }); 38 | 39 | router.get(APIs.EMAIL_REGULAR_NOTIFICATION, (req, res) => { 40 | console.log(`[ EMAIL ] [ ACCESS ] Router ${req.originalUrl} ${req.query.u}`); 41 | taskSchedule.regularNotificationSubscription(req).then(() => { 42 | console.log(`[ EMAIL ] [ SUCCESS ] Router ${req.originalUrl} ${req.query.u}`); 43 | res.status(200).redirect(`${config.WebClientUrl}/home?o=email&s=1`); 44 | }).catch(error => { 45 | console.error(`[ EMAIL ] [ ERROR ] Router ${req.originalUrl} ${req.query.u}`, error); 46 | res.status(500).redirect(`${config.WebClientUrl}/home?o=email&s=0`); 47 | }); 48 | }); 49 | 50 | module.exports = router; -------------------------------------------------------------------------------- /annotation-service/routers/integration-controller.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const express = require("express"); 10 | const router = express.Router(); 11 | const APIs = require('../resources/APIs'); 12 | const IntegrationService = require("../services/integration.service"); 13 | 14 | router.post(APIs.INTEGRATION_CSV, (req, res) => { 15 | console.log(`[ INTEGRATION ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 16 | IntegrationService.SyncLabelledCaseToInstaML(req).then(response =>{ 17 | console.log(`[ INTEGRATION ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 18 | res.status(200).json(response); 19 | }).catch(error =>{ 20 | console.error(`[ INTEGRATION ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 21 | res.status(500).send(error); 22 | }); 23 | }); 24 | 25 | module.exports = router; -------------------------------------------------------------------------------- /annotation-service/routers/scripts-controller.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2022 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const express = require("express"); 10 | const router = express.Router(); 11 | const APIs = require('../resources/APIs'); 12 | const IPDS = require('../utils/ImportDataset.util'); 13 | const RIM = require('../scripts/reveiwInfoMigration'); 14 | 15 | router.post(APIs.DATASET_IMPORT, (req, res) => { 16 | console.log(`[ SCRIPTE ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 17 | IPDS.importDataset(req).then((response) => { 18 | console.log(`[ SCRIPTE ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 19 | res.status(200).json(response); 20 | }).catch(error => { 21 | console.error(`[ SCRIPTE ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 22 | res.status(500).send(error); 23 | }); 24 | }); 25 | 26 | router.get(APIs.MIGRATION_REVIEW_INFO, (req, res) => { 27 | console.log(`[ SCRIPTE ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 28 | RIM.migrationAllTicketsReviewInfo(req).then((response) => { 29 | console.log(`[ SCRIPTE ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 30 | res.status(200).json(response); 31 | }).catch(error => { 32 | console.error(`[ SCRIPTE ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 33 | res.status(500).send(error); 34 | }); 35 | }); 36 | 37 | module.exports = router; -------------------------------------------------------------------------------- /annotation-service/routers/slack-controller.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const express = require("express"); 10 | const router = express.Router(); 11 | const APIs = require('../resources/APIs'); 12 | const slackConversations = require("../services/slack/slackConversations.service") 13 | 14 | router.post(APIs.CONVERSATION_LIST, (req, res) => { 15 | console.log(`[ SLACK ] [ ACCESS ] Router ${req.originalUrl} ${req.auth.email}`); 16 | slackConversations.findConversation(req).then(response => { 17 | console.log(`[ SLACK ] [ SUCCESS ] Router ${req.originalUrl} ${req.auth.email}`); 18 | res.status(200).json(response); 19 | }).catch(error => { 20 | console.error(`[ SLACK ] [ ERROR ] Router ${req.originalUrl} ${req.auth.email}`, error); 21 | res.status(500).send(error); 22 | }); 23 | }); 24 | 25 | 26 | 27 | module.exports = router; -------------------------------------------------------------------------------- /annotation-service/services/authForNoe.service.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const axios = require('axios'); 10 | const config = require('../config/config'); 11 | const MESSAGE = require('../config/code_msg'); 12 | 13 | let token; 14 | let expirationDate; 15 | 16 | const generateEsp2NoeToken = async() => { 17 | 18 | const response = await axios 19 | .create({ 20 | validateStatus: () => { 21 | return true; 22 | }, 23 | }) 24 | .post(`${config.authServiceUrl}/api/auth/v1/tokens`, { 25 | grant_type: 'client_credentials', 26 | client_id: `${config.esp2NoeClientId}`, 27 | client_secret: `${config.esp2NoeClientSecret}`, 28 | }); 29 | 30 | if (response.status === 200) { 31 | console.log('[ ESP-TOKEN ] Utils generateEsp2NoeToken success'); 32 | return response.data; 33 | } 34 | 35 | console.error('[ ESP-TOKEN ] [ ERROR ] Utils generateEsp2NoeToken: ', response.data); 36 | MESSAGE.VALIDATION_UNAUTH_TOKEN.DATA = [{ statusText: response.statusText, data: response.data }] 37 | throw MESSAGE.VALIDATION_UNAUTH_TOKEN; 38 | }; 39 | 40 | module.exports.getEsp2NoeToken = async() => { 41 | console.log('[ ESP-TOKEN ] Utils getEsp2NoeToken'); 42 | if (token === undefined || expirationDate === undefined || expirationDate < new Date()) { 43 | const data = await generateEsp2NoeToken(); 44 | token = data.access_token; 45 | expirationDate = new Date(Date.now() + data.expires_in * 1000 - 1000); 46 | } 47 | return token; 48 | }; -------------------------------------------------------------------------------- /annotation-service/services/community.service.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const { DOWNLOADSRC } = require('../config/constant'); 10 | const ObjectId = require("mongodb").ObjectID; 11 | const mongoDb = require('../db/mongo.db'); 12 | const { ProjectModel } = require('../db/db-connect'); 13 | 14 | async function countCommunityDownload(req){ 15 | 16 | const src = req.body.src? req.body.src: req.query.src; 17 | 18 | if(src == DOWNLOADSRC.COMMUNITY){ 19 | 20 | const pid = req.body.pid? req.body.pid: req.query.pid; 21 | 22 | console.log(`[ COMMUNITY ] service countCommunityDownload proId: ${pid} user: ${req.auth.email}`); 23 | 24 | const conditions = { _id: ObjectId(pid) }; 25 | const update = { $inc: {"downloadCount.community": 1 } }; 26 | await mongoDb.findOneAndUpdate(ProjectModel, conditions, update); 27 | 28 | } 29 | 30 | } 31 | 32 | module.exports = { 33 | countCommunityDownload, 34 | } -------------------------------------------------------------------------------- /annotation-service/services/s3.service.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const STS = require('../utils/sts'); 10 | const S3 = require('../utils/s3'); 11 | const config = require('../config/config'); 12 | const localFileSysService = require('./localFileSys.service'); 13 | const { ACCESS_TIME_60, AWSRESOURCE } = require('../config/constant'); 14 | const mongoDb = require('../db/mongo.db'); 15 | const { UserModel } = require('../db/db-connect'); 16 | const MESSAGE = require('../config/code_msg'); 17 | 18 | 19 | async function prepareS3Configs(req) { 20 | console.log(`[ S3 ] Service prepareS3Configs user: `, req.auth.email); 21 | const user = await mongoDb.findById(UserModel, req.auth.email); 22 | if (!user) { 23 | throw MESSAGE.VALIDATION_PERMITION; 24 | } 25 | console.log(`[ S3 ] Service check if is valid user`); 26 | 27 | let reponse = { 28 | bucket: Buffer.from(config.bucketName).toString('base64'), 29 | region: Buffer.from(config.region).toString('base64'), 30 | key: Buffer.from(`upload/${user._id}`).toString('base64'), 31 | apiVersion: Buffer.from('2006-03-01').toString('base64'), 32 | credentials: "" 33 | } 34 | const data = await STS.prepareCredentials(AWSRESOURCE.S3, ACCESS_TIME_60); 35 | reponse.credentials = { 36 | accessKeyId: Buffer.from(data.Credentials.AccessKeyId).toString('base64'), 37 | secretAccessKey: Buffer.from(data.Credentials.SecretAccessKey).toString('base64'), 38 | sessionToken: Buffer.from(data.Credentials.SessionToken).toString('base64') 39 | } 40 | console.log(`[ S3 ] Service prepare sts access credentials `); 41 | return reponse; 42 | } 43 | 44 | 45 | async function deleteOriginalFile(key) { 46 | console.log(`[ S3 ] Service deleteOriginalFile.S3.deleteAnObject`, key); 47 | return S3.deleteAnObject(key); 48 | } 49 | 50 | async function uploadFileToS3(file, key) { 51 | console.log(`[ S3 ] Service uploadFileToS3 read temp file stream`, file); 52 | let fileStream = await localFileSysService.readFileFromLocalSys(file); 53 | 54 | console.log(`[ S3 ] Service uploadFileToS3.S3.uploadObject`, key); 55 | return S3.uploadObject(key, fileStream); 56 | 57 | } 58 | 59 | module.exports = { 60 | prepareS3Configs, 61 | deleteOriginalFile, 62 | uploadFileToS3, 63 | 64 | } -------------------------------------------------------------------------------- /annotation-service/services/slack/slack.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const config = require('../../config/config'); 10 | const { App: Bolt, LogLevel } = require('@slack/bolt'); 11 | const slackBuildAppHome = require('./slackBuildAppHome.service'); 12 | const slackAnnotateModalService = require('./slackAnnotateModal.service') 13 | 14 | 15 | 16 | async function newBolt() { 17 | return new Bolt({ 18 | token: config.slackBotUserOAuthToken, 19 | signingSecret: config.slackSigningSecret, 20 | socketMode: true, 21 | appToken: config.slackAppToken, 22 | // logLevel: LogLevel.DEBUG 23 | logLevel: LogLevel.ERROR 24 | 25 | }); 26 | } 27 | 28 | 29 | async function slackStart() { 30 | // start your app 31 | const bolt = await newBolt(); 32 | await bolt.start(); 33 | console.log(`⚡️⚡️⚡️ Bolt app is running on localhost:${config.serverPort}`) 34 | await slackBuildAppHome.buildAppHome(bolt); 35 | 36 | // to listening all the start-annotate-btn 37 | bolt.action({ action_id: "click_btn_start_annotate" }, 38 | async ({ body, client, ack, logger }) => { 39 | await ack(); 40 | try { 41 | if (body.user && body.actions) { 42 | //open modal within 3 seconds 43 | await slackAnnotateModalService.openModal(bolt, client, body) 44 | } 45 | } 46 | catch (error) { 47 | logger.error(error); 48 | } 49 | }); 50 | 51 | // to listening all the radio 52 | await slackAnnotateModalService.actionListening(bolt, "annotate_label_radio") 53 | await slackAnnotateModalService.actionListening(bolt, "flag_btn") 54 | await slackAnnotateModalService.actionListening(bolt, "skip_btn") 55 | 56 | } 57 | 58 | 59 | 60 | 61 | module.exports = { slackStart } -------------------------------------------------------------------------------- /annotation-service/services/slack/slackChat.service.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | const config = require('../../config/config'); 9 | const { WebClient } = require('@slack/web-api'); 10 | const web = new WebClient(config.slackBotUserOAuthToken); 11 | 12 | 13 | 14 | async function publishMessage(data) { 15 | try { 16 | let blocks = await generateMessageBlocks(data); 17 | // Call the chat.postMessage method using the built-in WebClient 18 | for (let i = 0; i < data.channels.length; i++) { 19 | await web.chat.postMessage({ 20 | channel: data.channels[i].slackName, 21 | text: `:loud_sound:${data.pname} is ready to annotate.`, 22 | blocks: blocks 23 | }); 24 | } 25 | } 26 | catch (error) { 27 | console.error(error); 28 | return error; 29 | } 30 | } 31 | 32 | 33 | async function generateMessageBlocks(data) { 34 | return [ 35 | { 36 | "type": "header", 37 | "text": { 38 | "type": "plain_text", 39 | "text": `${data.pname} is ready to annotate` 40 | } 41 | }, 42 | { 43 | "type": "divider" 44 | }, 45 | { 46 | "type": "section", 47 | "text": { 48 | "type": "mrkdwn", 49 | "text": ` >*Project Name:* ${data.pname} \n>*Create Date:* ${data.createdDate}\n>*Creator:* ${data.creator}\n>*Total Tickets:* ${data.totalCase}\n` 50 | } 51 | }, 52 | { 53 | "type": "actions", 54 | "elements": [ 55 | { 56 | "type": "button", 57 | "text": { 58 | "type": "plain_text", 59 | "text": "Start Annotate" 60 | }, 61 | "style": "primary", 62 | "value": `${data.pid}`, 63 | "action_id": "click_btn_start_annotate" 64 | } 65 | ] 66 | }, 67 | { 68 | "type": "context", 69 | "elements": [ 70 | { 71 | "type": "mrkdwn", 72 | "text": `:information_desk_person: For more detail, please visit <${config.WebClientUrl}|${config.teamTitle}>` 73 | } 74 | ] 75 | } 76 | ] 77 | 78 | } 79 | module.exports = { 80 | publishMessage, 81 | } 82 | 83 | -------------------------------------------------------------------------------- /annotation-service/services/slack/slackConversations.service.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | const config = require('../../config/config'); 9 | const { WebClient } = require('@slack/web-api'); 10 | const web = new WebClient(config.slackBotUserOAuthToken); 11 | 12 | 13 | 14 | async function findConversation(req) { 15 | try { 16 | let id = req.body.slackId 17 | const result = await web.conversations.info({ channel: id }); 18 | if (result.ok) { 19 | return { id: result.channel.id, name: result.channel.name, is_member: result.channel.is_member } 20 | } else { 21 | return 22 | } 23 | } 24 | catch (error) { 25 | return 26 | } 27 | } 28 | 29 | 30 | 31 | 32 | module.exports = { 33 | findConversation 34 | } 35 | 36 | -------------------------------------------------------------------------------- /annotation-service/services/slack/slackUsers.service.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | const config = require('../../config/config'); 9 | const _ = require("lodash"); 10 | const { WebClient } = require('@slack/web-api'); 11 | const web = new WebClient(config.slackBotUserOAuthToken); 12 | 13 | 14 | 15 | async function findUserInChannels(event, slackIds) { 16 | try { 17 | let params = { 18 | limit: 1000, 19 | user: event.user, 20 | team_id: event.view.team_id, 21 | types: "public_channel,private_channel", 22 | cursor: '' 23 | } 24 | let found = 0; 25 | while (found === 0) { 26 | const result = await web.users.conversations(params); 27 | params.cursor = result.response_metadata.next_cursor; 28 | let arr = _.intersection(slackIds, _.map(result.channels, "id")); 29 | if (arr.length > 0) { 30 | found = 1; 31 | return found; 32 | } 33 | if (result.response_metadata.next_cursor === "") { 34 | found = 2; 35 | return found; 36 | } 37 | } 38 | } 39 | catch (error) { 40 | return error 41 | } 42 | } 43 | 44 | 45 | module.exports = { 46 | findUserInChannels 47 | } 48 | 49 | -------------------------------------------------------------------------------- /annotation-service/services/slack/utils/accessToken.service.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const axios = require('axios'); 10 | const config = require('../../../config/config'); 11 | const qs = require('qs'); 12 | const { generateBasicToken } = require("../../../middlewares/jwt.middleware") 13 | const MESSAGE = require('../../../config/code_msg'); 14 | 15 | let token; 16 | let expirationDate; 17 | 18 | 19 | async function generateEspToken() { 20 | const url = config.espTokenAuthorizeUrl; 21 | const data = qs.stringify({ token: config.espToken }); 22 | const configs = { headers: { "Content-Type": "application/x-www-form-urlencoded" } } 23 | try { 24 | const res = await axios.post(url, data, configs); 25 | if (res.status === 200) { 26 | console.log('[ ESP-TOKEN ] Utils generateEspToken success'); 27 | return res.data; 28 | } 29 | } 30 | catch (error) { 31 | console.error('[ ESP-TOKEN ] [ ERROR ] Utils generateEspToken: ', error); 32 | MESSAGE.VALIDATION_UNAUTH_TOKEN.DATA = [error.data]; 33 | MESSAGE.VALIDATION_UNAUTH_TOKEN.MSG = error.msg; 34 | throw MESSAGE.VALIDATION_UNAUTH_TOKEN; 35 | } 36 | } 37 | 38 | 39 | 40 | async function getEspToken(email) { 41 | console.log('[ ESP-TOKEN ] Utils getEspToken'); 42 | if (token === undefined || expirationDate === undefined || expirationDate < new Date()) { 43 | if (config.ESP) { 44 | var dataObj = await generateEspToken(); 45 | } else { 46 | var dataObj = await generateBasicToken(email); 47 | } 48 | token = dataObj.access_token; 49 | expirationDate = new Date(Date.now() + dataObj.expires_in * 1000 - 1000); 50 | } 51 | return token; 52 | } 53 | 54 | module.exports = { getEspToken } -------------------------------------------------------------------------------- /annotation-service/services/slack/utils/slack.utils.js: -------------------------------------------------------------------------------- 1 | 2 | /*** 3 | * 4 | * Copyright 2019-2021 VMware, Inc. 5 | * SPDX-License-Identifier: Apache-2.0 6 | * 7 | ***/ 8 | const _ = require("lodash"); 9 | 10 | 11 | 12 | async function generateObj(type, text, style, value, action_id, block_id) { 13 | let obj = { 14 | "type": type, 15 | "text": text, 16 | "style": style, 17 | "value": value, 18 | "action_id": action_id, 19 | "block_id": block_id 20 | 21 | } 22 | return obj 23 | } 24 | 25 | 26 | async function returnAllPageFunc(pageSize, arr) { 27 | let pageNum = 1 28 | let pageObj = { 29 | pageSize, 30 | total: arr.length, 31 | pageNum: 1, 32 | data: [] 33 | } 34 | let pageResult = [] 35 | 36 | let newArr = _.cloneDeep(arr); 37 | let totalPage = newArr.length ? Math.ceil(arr.length / pageSize) : 0 38 | 39 | for (let i = 1; i <= totalPage; i++) { 40 | if (totalPage == 1) { 41 | pageNum += 1 42 | pageObj.data = newArr.splice(0, arr.length) 43 | } else if (i <= totalPage) { 44 | pageNum += 1 45 | pageObj.data = newArr.splice(0, pageSize) 46 | } else { 47 | pageObj.data = newArr.splice(0, arr.length % pageSize) 48 | } 49 | pageResult.push(pageObj) 50 | pageObj = { 51 | pageSize, 52 | total: arr.length, 53 | pageNum: pageNum, 54 | data: [] 55 | } 56 | } 57 | return pageResult 58 | } 59 | 60 | module.exports = { 61 | generateObj, 62 | returnAllPageFunc 63 | } 64 | -------------------------------------------------------------------------------- /annotation-service/utils/DB.OPERATIONS.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const { ProjectModel, UserModel, DataSetModel, ImgModel, SrModel } = require("../db/db-connect"); 10 | const mongoDb = require("../db/mongo.db"); 11 | 12 | 13 | async function updateDBColumnType(req) { 14 | const MODEL = await getProjectModel(req.body.collection); 15 | const column = req.body.column; 16 | const datas = await mongoDb.findByConditions(MODEL, {}, "_id " + column); 17 | console.log(`[ DB.OPERATIONS ] Utils total case ${datas.length}`); 18 | let updateIndex = 0, start = Date.now(); 19 | for (const data of datas) { 20 | let update = null; 21 | if (req.body.str2array) { 22 | update = { 23 | $set: { 24 | [column]: data[column] 25 | } 26 | } 27 | } 28 | updateIndex += 1; 29 | await mongoDb.findOneAndUpdate(MODEL, {_id: data._id}, update); 30 | } 31 | console.log(`[ DB.OPERATIONS ] Utils update total case ${updateIndex} time: ${ (Date.now() - start)/1000 }`); 32 | return {TCASE: datas.length, UPCASE: updateIndex}; 33 | 34 | } 35 | 36 | async function getProjectModel(collection) { 37 | switch (collection) { 38 | case "projects": 39 | return ProjectModel; 40 | case "srs": 41 | return SrModel; 42 | case "images": 43 | return ImgModel; 44 | case "users": 45 | return UserModel; 46 | case "datasets": 47 | return DataSetModel; 48 | default: 49 | return; 50 | } 51 | } 52 | 53 | module.exports = { 54 | updateDBColumnType, 55 | getProjectModel, 56 | } -------------------------------------------------------------------------------- /annotation-service/utils/common.utils.js: -------------------------------------------------------------------------------- 1 | 2 | /*** 3 | * 4 | * Copyright 2019-2021 VMware, Inc. 5 | * SPDX-License-Identifier: Apache-2.0 6 | * 7 | ***/ 8 | 9 | 10 | const _ = require("lodash"); 11 | 12 | //Find the most frequent element in the array 13 | async function findFrequentlyElementInArray(arr) { 14 | 15 | let maxEle; 16 | let maxNum = 1; 17 | const obj = arr.reduce(function (p, k) { 18 | p[k] ? p[k]++ : p[k] = 1; 19 | if (p[k] >= maxNum) { 20 | maxEle = k; 21 | maxNum++; 22 | } 23 | return p; 24 | }, {}) 25 | 26 | const times = Object.values(obj); 27 | const max = _.max(times); 28 | 29 | if (_.indexOf(times, max) == _.lastIndexOf(times, max)) { 30 | return maxEle; 31 | } 32 | return null; 33 | 34 | } 35 | 36 | async function findFrequentlyElementInObject(obj) { 37 | 38 | const values = Object.values(obj); 39 | const keys = Object.keys(obj); 40 | const maxV = _.max(values); 41 | 42 | if (!_.sum(values)) { 43 | return null; 44 | } 45 | // more than one max value take first 46 | const key = keys[_.indexOf(values, maxV)]; 47 | return { [key]: obj[key] }; 48 | 49 | //max value is only one 50 | // if(_.indexOf(values, maxV) == _.lastIndexOf(values, maxV)){ 51 | // let resp = {}; 52 | // resp[keys[_.indexOf(values, maxV)]] = obj[keys[_.indexOf(values, maxV)]]; 53 | // return resp; 54 | // } 55 | 56 | } 57 | 58 | 59 | async function probabilisticInObject(obj) { 60 | 61 | const values = Object.values(obj); 62 | const total = (_.sum(values) == 0 ? 1 : _.sum(values)); 63 | 64 | let probabilistic = {}; 65 | for (const [key, value] of Object.entries(obj)) { 66 | probabilistic[key] = _.round(_.divide(value, total), 2); 67 | } 68 | return probabilistic; 69 | } 70 | 71 | async function formatDate(timestamp) { 72 | let date = new Date(timestamp); 73 | let year = date.getFullYear(); 74 | let month = date.getMonth() + 1; 75 | month = month < 10 ? ('0' + month) : month; 76 | let day = date.getDate(); 77 | day = day < 10 ? ('0' + day) : day; 78 | let h = date.getHours(); 79 | h = h < 10 ? ('0' + h) : h; 80 | let m = date.getMinutes(); 81 | m = m < 10 ? ('0' + m) : m; 82 | let s = date.getSeconds(); 83 | s = s < 10 ? ('0' + s) : s; 84 | return year + '-' + month + '-' + day + ' ' + h + ':' + m + ':' + s; 85 | } 86 | 87 | module.exports = { 88 | findFrequentlyElementInArray, 89 | findFrequentlyElementInObject, 90 | probabilisticInObject, 91 | formatDate 92 | } 93 | -------------------------------------------------------------------------------- /annotation-service/utils/fileSystem.utils.js: -------------------------------------------------------------------------------- 1 | 2 | /*** 3 | * 4 | * Copyright 2019-2021 VMware, Inc. 5 | * SPDX-License-Identifier: Apache-2.0 6 | * 7 | ***/ 8 | 9 | const config = require("../config/config"); 10 | const localFileSysService = require('../services/localFileSys.service'); 11 | const request = require('request'); 12 | const S3Utils = require('./s3'); 13 | const { S3OPERATIONS } = require('../config/constant'); 14 | const MESSAGE = require('../config/code_msg') 15 | 16 | async function handleFileStream(fileLocation) { 17 | 18 | if (config.ESP || !config.useLocalFileSys && config.useAWS && config.bucketName && config.s3RoleArn) { 19 | 20 | console.log(`[ FILE_SYSTEM ] Utils S3Utils.signedUrlByS3`); 21 | const signedUrl = await S3Utils.signedUrlByS3(S3OPERATIONS.GETOBJECT, fileLocation); 22 | return request.get(signedUrl); 23 | 24 | }else if (config.useLocalFileSys && !config.useAWS) { 25 | 26 | console.log(`[ FILE_SYSTEM ] Utils localFileSysService.readFileFromLocalSys`); 27 | return localFileSysService.readFileFromLocalSys(fileLocation); 28 | 29 | }else{ 30 | 31 | throw MESSAGE.VALIDATION_FILE_SYS; 32 | 33 | } 34 | 35 | } 36 | 37 | 38 | module.exports={ 39 | handleFileStream, 40 | } -------------------------------------------------------------------------------- /annotation-service/utils/mongoModel.utils.js: -------------------------------------------------------------------------------- 1 | /*** 2 | * 3 | * Copyright 2019-2021 VMware, Inc. 4 | * SPDX-License-Identifier: Apache-2.0 5 | * 6 | ***/ 7 | 8 | 9 | const { PROJECTTYPE } = require("../config/constant"); 10 | const { ImgModel, SrModel, LogModel } = require("../db/db-connect"); 11 | const validator = require("./validator"); 12 | 13 | 14 | async function getModelProject(conditions) { 15 | const project = await validator.checkProjectByconditions(conditions, true); 16 | let model = SrModel; 17 | if (project[0].projectType == PROJECTTYPE.IMGAGE) { 18 | model = ImgModel; 19 | }else if (project[0].projectType == PROJECTTYPE.LOG) { 20 | model = LogModel; 21 | } 22 | return {model: model, project: project[0]}; 23 | } 24 | 25 | 26 | module.exports = { 27 | getModelProject, 28 | } -------------------------------------------------------------------------------- /docker-compose-host-db.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | version: '2' 5 | services: 6 | annotation-app: 7 | container_name: app 8 | image: 1107899648/vmware-daml-annotation-app:latest 9 | depends_on: 10 | - annotation-service 11 | ports: 12 | - "4200:4200" 13 | networks: 14 | - bridge 15 | restart: always 16 | 17 | annotation-service: 18 | container_name: as 19 | image: 1107899648/vmware-daml-annotation-service:latest 20 | environment: 21 | - MONGODB_URL=mongodb://host.docker.internal:27017/daml 22 | - LOOP_AL_URL=http://als:8000/api 23 | depends_on: 24 | - active-learning-service 25 | ports: 26 | - "3000:3000" 27 | networks: 28 | - bridge 29 | volumes: 30 | - "./data/as:/app/FILE_SYS" 31 | restart: always 32 | 33 | active-learning-service: 34 | container_name: als 35 | image: 1107899648/vmware-daml-active-learning-service:latest 36 | environment: 37 | - MONGODB_URL=mongodb://host.docker.internal:27017/daml 38 | - SPACY_MODEL=en_core_web_md 39 | ports: 40 | - "8000:8000" 41 | networks: 42 | - bridge 43 | volumes: 44 | - "./data/als:/app/models" 45 | restart: always 46 | 47 | networks: 48 | bridge: 49 | driver: bridge -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2019-2021 VMware, Inc. 2 | # SPDX-License-Identifier: Apache-2.0 3 | 4 | version: '2' 5 | services: 6 | annotation-app: 7 | container_name: app 8 | image: 1107899648/vmware-daml-annotation-app:latest 9 | depends_on: 10 | - annotation-service 11 | ports: 12 | - "4200:4200" 13 | networks: 14 | - bridge 15 | restart: always 16 | 17 | annotation-service: 18 | container_name: as 19 | image: 1107899648/vmware-daml-annotation-service:latest 20 | environment: 21 | - MONGODB_URL=mongodb://db:27017/daml 22 | - LOOP_AL_URL=http://als:8000/api 23 | depends_on: 24 | - mongo 25 | - active-learning-service 26 | ports: 27 | - "3000:3000" 28 | networks: 29 | - bridge 30 | volumes: 31 | - "./data/as:/app/FILE_SYS" 32 | restart: always 33 | 34 | active-learning-service: 35 | container_name: als 36 | image: 1107899648/vmware-daml-active-learning-service:latest 37 | environment: 38 | - MONGODB_URL=mongodb://db:27017/daml 39 | - SPACY_MODEL=en_core_web_md 40 | depends_on: 41 | - mongo 42 | ports: 43 | - "8000:8000" 44 | networks: 45 | - bridge 46 | volumes: 47 | - "./data/als:/app/models" 48 | restart: always 49 | 50 | mongo: 51 | image: mongo:latest 52 | container_name: db 53 | ports: 54 | - "27017:27017" 55 | networks: 56 | - bridge 57 | volumes: 58 | - "./data/db:/data/db" 59 | restart: always 60 | 61 | networks: 62 | bridge: 63 | driver: bridge 64 | -------------------------------------------------------------------------------- /docs/AWS-step-by-step-config-with-chart.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/docs/AWS-step-by-step-config-with-chart.pdf -------------------------------------------------------------------------------- /docs/manifest.yml: -------------------------------------------------------------------------------- 1 | display_information: 2 | name: please enter your slack app name here 3 | features: 4 | app_home: 5 | home_tab_enabled: true 6 | messages_tab_enabled: true 7 | messages_tab_read_only_enabled: false 8 | bot_user: 9 | display_name: please enter your slack app name here 10 | always_online: true 11 | oauth_config: 12 | scopes: 13 | user: 14 | - users:read.email 15 | - users:read 16 | bot: 17 | - channels:read 18 | - chat:write 19 | - chat:write.customize 20 | - groups:read 21 | - im:history 22 | - im:read 23 | - incoming-webhook 24 | - mpim:read 25 | - users:read.email 26 | - users:read 27 | settings: 28 | event_subscriptions: 29 | bot_events: 30 | - app_home_opened 31 | - message.im 32 | interactivity: 33 | is_enabled: true 34 | org_deploy_enabled: false 35 | socket_mode_enabled: true 36 | token_rotation_enabled: false 37 | -------------------------------------------------------------------------------- /docs/tutorial/annotator-screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/docs/tutorial/annotator-screen.png -------------------------------------------------------------------------------- /docs/tutorial/project-screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware/data-annotator-for-machine-learning/ccc6dbfcc0bd680557f4068df89be5f20c93fcd9/docs/tutorial/project-screen.png --------------------------------------------------------------------------------