├── .github ├── .vsts-ci.yml ├── codeql │ └── codeql-config.yml └── jobs │ ├── cd-backend.yml │ ├── cd-frontend.yml │ └── cd-helm.yml ├── .gitignore ├── .vscode ├── extensions.json └── settings.json ├── .yarn ├── plugins │ └── @yarnpkg │ │ ├── plugin-interactive-tools.cjs │ │ └── plugin-typescript.cjs ├── releases │ └── yarn-3.4.1.cjs └── sdks │ ├── eslint │ ├── bin │ │ └── eslint.js │ └── package.json │ ├── integrations.yml │ └── typescript │ ├── bin │ ├── tsc │ └── tsserver │ └── package.json ├── .yarnrc.yml ├── Backend.dockerfile ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── Frontend.dockerfile ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── docker-compose.yml ├── docs └── deployment │ ├── AKS_DEPLOY.md │ └── LOCAL_DEVELOPMENT.md ├── helm ├── .helmignore ├── Chart.yaml ├── aks │ └── nginx-ingress.yaml ├── templates │ ├── ingresses.yaml │ ├── namespaces.yaml │ └── services.yaml ├── values.local.yaml ├── values.prod.yaml └── values.yaml ├── javascript └── webapp │ ├── .eslintignore │ ├── .eslintrc │ ├── .gitignore │ ├── .prettierignore │ ├── .prettierrc │ ├── Dockerfile │ ├── config │ └── index.hbs │ ├── package.deployed.json │ ├── package.json │ ├── public │ ├── data │ │ ├── entity-graphs │ │ │ └── sample-220.json │ │ ├── entity-reports │ │ │ └── sample-220.json │ │ └── related-entity-graphs │ │ │ ├── sample-220-212.json │ │ │ └── sample-220-270.json │ └── web.config │ ├── src │ ├── App │ │ ├── App.tsx │ │ ├── ErrorBoundary.hooks.ts │ │ ├── ErrorBoundary.tsx │ │ ├── Header.styles.ts │ │ ├── Header.tsx │ │ ├── Layout.styles.ts │ │ ├── Layout.tsx │ │ ├── Pages.ts │ │ ├── Router.tsx │ │ ├── StyleContext.styles.ts │ │ ├── StyleContext.tsx │ │ └── StyleContext.types.ts │ ├── api │ │ └── index.ts │ ├── bootstrap.tsx │ ├── components │ │ ├── ComplexTableComponent │ │ │ ├── ComplexTableComponent.tsx │ │ │ └── index.ts │ │ ├── GraphComponent │ │ │ ├── GraphComponent.tsx │ │ │ └── index.ts │ │ ├── GraphLegend │ │ │ └── GraphLegend.tsx │ │ ├── ReportTemplateComponent │ │ │ ├── ReportTemplate.tsx │ │ │ ├── ReportTemplateSection.tsx │ │ │ └── index.ts │ │ ├── SingleChartComponent │ │ │ ├── SingleChartComponent.tsx │ │ │ └── index.ts │ │ ├── TableComponent │ │ │ ├── TableComponent.tsx │ │ │ └── index.ts │ │ ├── VegaHost │ │ │ ├── VegaHost.constants.ts │ │ │ ├── VegaHost.tsx │ │ │ ├── VegaHost.types.ts │ │ │ ├── VegaHost.util.ts │ │ │ ├── hooks │ │ │ │ ├── composition.ts │ │ │ │ ├── data.ts │ │ │ │ ├── events.ts │ │ │ │ ├── index.ts │ │ │ │ ├── log.ts │ │ │ │ ├── signals.ts │ │ │ │ └── view.ts │ │ │ └── index.ts │ │ └── tables │ │ │ ├── AttributeCountsCountPercentRankRow.tsx │ │ │ ├── AttributeCountsCountPercentRow.tsx │ │ │ ├── AttributeCountsCountRow.tsx │ │ │ ├── AttributeValuesListRow.tsx │ │ │ ├── AttributeValuesOwnFlagsRow.tsx │ │ │ ├── AttributeValuesRelatedFlagsMeasurementsRow.tsx │ │ │ ├── AttributeValuesRelatedFlagsRow.tsx │ │ │ ├── EvidenceCellContent.tsx │ │ │ ├── TableHead.tsx │ │ │ └── styles.ts │ ├── declarations.d.ts │ ├── hooks │ │ ├── useBarChartData.ts │ │ ├── useDonutChartData.ts │ │ └── useGraphData.ts │ ├── index.tsx │ ├── pages │ │ ├── EntityReportPage.hooks.ts │ │ ├── EntityReportPage.styles.ts │ │ ├── EntityReportPage.tsx │ │ └── HomePage.tsx │ ├── styles │ │ └── reports.ts │ ├── types.ts │ ├── utils │ │ └── graphs.ts │ └── vega │ │ ├── VegaChart.tsx │ │ ├── VegaGraph.tsx │ │ ├── bar-chart.json │ │ ├── donut-chart.json │ │ └── force-graph.json │ ├── tsconfig.json │ └── webpack.config.js ├── package.json ├── powerbi └── TransparencyEngine.pbit ├── python ├── .gitattributes ├── .gitignore ├── api-backend │ ├── .dockerignore │ ├── .flake8 │ ├── .gitattributes │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── api_backend │ │ ├── __init__.py │ │ ├── api_main.py │ │ ├── entities │ │ │ ├── api │ │ │ │ ├── __init__.py │ │ │ │ ├── get_graph.py │ │ │ │ ├── get_graph_from_db.py │ │ │ │ └── router.py │ │ │ ├── constants │ │ │ │ ├── __init__.py │ │ │ │ └── schema.json │ │ │ └── util │ │ │ │ ├── __init__.py │ │ │ │ └── style_graph.py │ │ ├── model │ │ │ ├── __init__.py │ │ │ ├── graph_model.py │ │ │ └── report_model.py │ │ ├── report │ │ │ ├── api │ │ │ │ ├── __init__.py │ │ │ │ ├── get_report.py │ │ │ │ ├── get_report_from_db.py │ │ │ │ ├── get_report_score.py │ │ │ │ ├── get_report_score_from_db.py │ │ │ │ ├── get_report_url.py │ │ │ │ ├── get_report_url_from_db.py │ │ │ │ └── router.py │ │ │ ├── builders │ │ │ │ ├── __init__.py │ │ │ │ ├── activity_analysis.py │ │ │ │ ├── activity_summary.py │ │ │ │ ├── entity_overview.py │ │ │ │ ├── own_review_flag_analysis.py │ │ │ │ ├── related_entities.py │ │ │ │ ├── report.py │ │ │ │ └── review_flag_summary.py │ │ │ ├── constants │ │ │ │ ├── __init__.py │ │ │ │ ├── attributes.py │ │ │ │ ├── config.json │ │ │ │ └── report_data_mapping.py │ │ │ ├── util │ │ │ │ ├── __init__.py │ │ │ │ ├── parsers.py │ │ │ │ └── util.py │ │ │ └── yarn.lock │ │ ├── search │ │ │ └── api │ │ │ │ ├── __init__.py │ │ │ │ └── router.py │ │ └── util │ │ │ ├── __init__.py │ │ │ └── db_engine.py │ ├── poetry.lock │ ├── pyproject.toml │ ├── scripts │ │ ├── install_python_deps.sh │ │ └── start.sh │ ├── setup.cfg │ └── tests │ │ └── __init__.py └── transparency-engine │ ├── .codespellignore │ ├── .devcontainer │ └── devcontainer.json │ ├── .dockerignore │ ├── .flake8 │ ├── .gitattributes │ ├── .gitignore │ ├── Dockerfile │ ├── docs │ ├── Makefile │ ├── conf.py │ ├── contributing │ │ └── index.rst │ ├── getting_started │ │ ├── index.rst │ │ └── install.rst │ ├── index.rst │ ├── make.bat │ ├── modules.rst │ ├── transparency_engine.io.rst │ ├── transparency_engine.modules.graph.link_inference.rst │ ├── transparency_engine.modules.graph.preprocessing.rst │ ├── transparency_engine.modules.graph.rst │ ├── transparency_engine.modules.graphs.rst │ ├── transparency_engine.modules.graphs.uase.rst │ ├── transparency_engine.modules.math.rst │ ├── transparency_engine.modules.rst │ ├── transparency_engine.modules.stats.rst │ ├── transparency_engine.modules.text.rst │ ├── transparency_engine.pipeline.config.rst │ ├── transparency_engine.pipeline.rst │ ├── transparency_engine.pipeline.step.data_load.rst │ ├── transparency_engine.pipeline.step.rst │ ├── transparency_engine.preprocessing.graph.rst │ ├── transparency_engine.preprocessing.rst │ ├── transparency_engine.preprocessing.text.rst │ ├── transparency_engine.rst │ └── transparency_engine.spark.rst │ ├── poetry.lock │ ├── pyproject.toml │ ├── samples │ └── config │ │ ├── pipeline.json │ │ └── steps.json │ ├── scripts │ └── install_python_deps.sh │ ├── setup.cfg │ ├── tests │ └── __init__.py │ └── transparency_engine │ ├── __init__.py │ ├── analysis │ ├── __init__.py │ ├── link_filtering │ │ ├── __init__.py │ │ ├── base_link_filter.py │ │ └── graph │ │ │ ├── dynamic_individual_link_filtering.py │ │ │ └── macro_link_filtering.py │ ├── link_inference │ │ ├── __init__.py │ │ ├── base_link_estimator.py │ │ ├── dynamic_link_estimator.py │ │ ├── macro_link_estimator.py │ │ └── static_link_estimator.py │ └── scoring │ │ ├── __init__.py │ │ ├── entity_scoring.py │ │ ├── measures.py │ │ └── network_scoring.py │ ├── base_transformer.py │ ├── containers.py │ ├── io │ ├── __init__.py │ └── data_handler.py │ ├── main.py │ ├── modules │ ├── __init__.py │ ├── data_generator │ │ ├── __init__.py │ │ ├── data_replacement.py │ │ └── synthetic_data_generator.py │ ├── data_shaper │ │ ├── __init__.py │ │ └── spark_transform.py │ ├── graph │ │ ├── __init__.py │ │ ├── embed │ │ │ ├── __init__.py │ │ │ ├── base_embed.py │ │ │ ├── use_embedding.py │ │ │ └── use_graph.py │ │ ├── link_filtering │ │ │ ├── __init__.py │ │ │ ├── activity_scoring.py │ │ │ ├── dynamic_link_scoring.py │ │ │ ├── macro_links.py │ │ │ ├── normalizer.py │ │ │ ├── period_scoring.py │ │ │ └── period_similarity.py │ │ ├── link_inference │ │ │ ├── __init__.py │ │ │ ├── base_link_prediction.py │ │ │ └── use_link_prediction.py │ │ └── preprocessing │ │ │ ├── __init__.py │ │ │ ├── data_formatting.py │ │ │ └── graph_edges.py │ ├── math │ │ ├── __init__.py │ │ ├── embedding_processing.py │ │ └── matrix_operations.py │ ├── similarity │ │ ├── __init__.py │ │ ├── ann_search.py │ │ └── similarity_score.py │ ├── stats │ │ ├── __init__.py │ │ ├── aggregation.py │ │ ├── anomaly_detection.py │ │ └── arithmetic_operators.py │ └── text │ │ ├── __init__.py │ │ └── fuzzy_matching.py │ ├── pipeline │ ├── __init__.py │ ├── schemas.py │ └── transparency_pipeline.py │ ├── preprocessing │ ├── __init__.py │ ├── flag │ │ ├── __init__.py │ │ └── flag_filtering.py │ ├── graph │ │ ├── __init__.py │ │ ├── multipartite_graph.py │ │ └── multiplex_graph.py │ └── text │ │ ├── __init__.py │ │ └── lsh_fuzzy_matching.py │ ├── reporting │ ├── __init__.py │ ├── entity_activities.py │ ├── entity_attributes.py │ ├── entity_flags.py │ ├── entity_relationship_network.py │ ├── entity_report.py │ └── report_schemas.py │ ├── spark │ ├── __init__.py │ └── utils.py │ ├── synthetic_data │ ├── __init__.py │ └── public_procurement.py │ └── typing.py ├── rome.json ├── scripts ├── build-backend-images.sh ├── build-frontend-images.sh ├── install-charts.sh ├── list-resources.sh ├── push-images.sh ├── start-backend.sh ├── start-frontend.sh └── uninstall-charts.sh ├── turbo.json └── yarn.lock /.github/.vsts-ci.yml: -------------------------------------------------------------------------------- 1 | name: transparency-engine compliance CI 2 | pool: 3 | vmImage: ubuntu-latest 4 | 5 | trigger: 6 | batch: true 7 | branches: 8 | include: 9 | - main 10 | 11 | variables: 12 | Codeql.Enabled: true 13 | NODE_VERSION: "16.x" 14 | YARN_CACHE_FOLDER: $(Pipeline.Workspace)/.yarn/cache 15 | 16 | stages: 17 | - stage: Compliance 18 | dependsOn: [] 19 | jobs: 20 | - job: ComplianceJob 21 | pool: 22 | vmImage: windows-latest 23 | steps: 24 | - task: CredScan@3 25 | inputs: 26 | outputFormat: sarif 27 | debugMode: false 28 | 29 | - task: ComponentGovernanceComponentDetection@0 30 | inputs: 31 | scanType: "Register" 32 | verbosity: "Verbose" 33 | alertWarningLevel: "High" 34 | 35 | - task: PublishSecurityAnalysisLogs@3 36 | inputs: 37 | ArtifactName: "CodeAnalysisLogs" 38 | ArtifactType: "Container" 39 | 40 | - stage: DeployBuildFrontendImages 41 | displayName: Deploy - Frontend Images 42 | dependsOn: [] 43 | #if main branch var 44 | condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') 45 | jobs: 46 | - template: jobs/cd-frontend.yml 47 | parameters: 48 | name: "DeployBuildFrontendImagesJob" 49 | subscription: "$(subscription)" 50 | containerRegistry: "$(container-registry)" 51 | 52 | - stage: DeployBuildBackendImages 53 | displayName: Deploy - Backend Images 54 | dependsOn: [] 55 | condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') 56 | jobs: 57 | - template: jobs/cd-backend.yml 58 | parameters: 59 | name: "DeployBuildBackendImagesJob" 60 | subscription: "$(subscription)" 61 | containerRegistry: "$(container-registry)" 62 | 63 | - stage: DeployHelmCharts 64 | displayName: Deploy Helm Charts 65 | dependsOn: 66 | - DeployBuildFrontendImages 67 | - DeployBuildBackendImages 68 | condition: succeeded() 69 | jobs: 70 | - template: jobs/cd-helm.yml 71 | parameters: 72 | name: "DeployHelmCharts" 73 | subscription: "$(subscription)" 74 | aksClusterName: "$(aks-cluster-name)" 75 | aksResourceGroup: "$(aks-resource-group)" 76 | valueFile: "helm/values.prod.yaml" 77 | DB_NAME: $(db-name) 78 | SQL_ENDPOINT: $(sql-endpoint) 79 | SQL_USERNAME: $(sql-username) 80 | SQL_PASSWORD: $(sql-password) 81 | 82 | -------------------------------------------------------------------------------- /.github/codeql/codeql-config.yml: -------------------------------------------------------------------------------- 1 | name: 'CodeQL Configuration' 2 | paths: 3 | - javascript/ 4 | - python/ 5 | paths-ignore: 6 | - '**/*.spec.ts' 7 | - '.pnp.*' 8 | -------------------------------------------------------------------------------- /.github/jobs/cd-backend.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | name: "" 3 | subscription: "" 4 | containerRegistry: "" 5 | 6 | jobs: 7 | - job: ${{ parameters.name }} 8 | steps: 9 | # 10 | # Build Service Images 11 | # 12 | - task: Docker@1 13 | displayName: "Build backend service image" 14 | inputs: 15 | azureSubscriptionEndpoint: "${{ parameters.subscription }}" 16 | azureContainerRegistry: "${{ parameters.containerRegistry }}" 17 | dockerFile: python/api-backend/Dockerfile 18 | imageName: "backend:$(Build.BuildId)" 19 | includeLatestTag: true 20 | 21 | # 22 | # Push Service Images 23 | # 24 | - task: Docker@1 25 | displayName: "Push backend service image" 26 | inputs: 27 | azureSubscriptionEndpoint: "${{ parameters.subscription }}" 28 | azureContainerRegistry: "${{ parameters.containerRegistry }}" 29 | command: "Push an image" 30 | imageName: backend:latest 31 | -------------------------------------------------------------------------------- /.github/jobs/cd-frontend.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | name: "" 3 | subscription: "" 4 | containerRegistry: "" 5 | 6 | jobs: 7 | - job: ${{ parameters.name }} 8 | steps: 9 | # 10 | # Install Tooling 11 | # 12 | - task: NodeTool@0 13 | displayName: Install Node 14 | inputs: 15 | versionSpec: $(NODE_VERSION) 16 | 17 | # 18 | # Restore Caches 19 | # 20 | - task: Cache@2 21 | displayName: Cache Yarn cache 22 | inputs: 23 | key: '"yarn" | "$(Agent.OS)" | yarn.lock' 24 | restoreKeys: | 25 | yarn | "$(Agent.OS)" 26 | yarn 27 | path: $(YARN_CACHE_FOLDER) 28 | 29 | # 30 | # Install Dependencies 31 | # 32 | - task: Bash@3 33 | displayName: Install Node Dependencies 34 | env: 35 | CI: true 36 | inputs: 37 | targetType: "inline" 38 | script: yarn install --immutable 39 | 40 | # 41 | # Bundle Applications 42 | # 43 | - task: Bash@3 44 | displayName: Bundle Webapp 45 | env: 46 | CI: true 47 | inputs: 48 | targetType: "inline" 49 | script: yarn bundle 50 | 51 | # 52 | # Build Frontend Images 53 | # 54 | - task: Docker@1 55 | displayName: "Build frontend image" 56 | inputs: 57 | azureSubscriptionEndpoint: "${{ parameters.subscription }}" 58 | azureContainerRegistry: "${{ parameters.containerRegistry }}" 59 | dockerFile: javascript/webapp/Dockerfile 60 | imageName: "frontend:$(Build.BuildId)" 61 | includeLatestTag: true 62 | # 63 | # Push Frontend Images 64 | # 65 | - task: Docker@1 66 | displayName: "Push shell frontend image" 67 | inputs: 68 | azureSubscriptionEndpoint: "${{ parameters.subscription }}" 69 | azureContainerRegistry: "${{ parameters.containerRegistry }}" 70 | command: "Push an image" 71 | imageName: frontend:latest 72 | -------------------------------------------------------------------------------- /.github/jobs/cd-helm.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | name: "" 3 | subscription: "" 4 | aksClusterName: "" 5 | aksResourceGroup: "" 6 | valueFile: "" 7 | DB_NAME: "" 8 | SQL_ENDPOINT: "" 9 | SQL_USERNAME: "" 10 | SQL_PASSWORD: "" 11 | 12 | jobs: 13 | - job: ${{ parameters.name }} 14 | steps: 15 | # 16 | # Deploy Helm Chart 17 | # 18 | - task: HelmInstaller@0 19 | displayName: "Install Helm 3.9.0" 20 | inputs: 21 | helmVersion: 3.9.0 22 | 23 | - task: HelmDeploy@0 24 | displayName: "Validate helm package" 25 | inputs: 26 | command: package 27 | chartPath: "helm" 28 | destination: "helm/dist" 29 | save: false 30 | 31 | - task: HelmDeploy@0 32 | displayName: "Deploy chart to cluster" 33 | inputs: 34 | azureSubscription: "${{ parameters.subscription }}" 35 | azureResourceGroup: "${{ parameters.aksResourceGroup }}" 36 | kubernetesCluster: "${{ parameters.aksClusterName }}" 37 | command: upgrade 38 | chartType: FilePath 39 | chartPath: "helm" 40 | releaseName: "transparency-engine" 41 | valueFile: "${{ parameters.valueFile }}" 42 | arguments: --set DB_NAME=${{ parameters.DB_NAME }},SQL_ENDPOINT=${{ parameters.SQL_ENDPOINT }},SQL_USERNAME=${{ parameters.SQL_USERNAME }},SQL_PASSWORD=${{ parameters.SQL_PASSWORD }} 43 | waitForExecution: false 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # https://yarnpkg.com/getting-started/qa/#which-files-should-be-gitignored 2 | .pnp.* 3 | .yarn/* 4 | !.yarn/patches 5 | !.yarn/plugins 6 | !.yarn/releases 7 | !.yarn/sdks 8 | !.yarn/versions 9 | .poetry/ 10 | 11 | dist/ 12 | lib/ 13 | build/ 14 | coverage/ 15 | node_modules/ 16 | .turbo/ 17 | *.zip 18 | *-*.log 19 | 20 | .env.local 21 | .env.development.local 22 | .env.test.local 23 | .env.production.local 24 | 25 | .deploy-config/ -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "arcanis.vscode-zipfs", 4 | "dbaeumer.vscode-eslint", 5 | "esbenp.prettier-vscode" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": ".yarn/sdks/typescript/lib", 3 | "search.exclude": { 4 | "**/.yarn": true, 5 | "**/.pnp.*": true 6 | }, 7 | "eslint.nodePath": ".yarn/sdks", 8 | "prettier.prettierPath": ".yarn/sdks/prettier/index.js", 9 | "typescript.enablePromptUseWorkspaceTsdk": true 10 | } 11 | -------------------------------------------------------------------------------- /.yarn/sdks/eslint/bin/eslint.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const {existsSync} = require(`fs`); 4 | const {createRequire} = require(`module`); 5 | const {resolve} = require(`path`); 6 | 7 | const relPnpApiPath = "../../../../.pnp.cjs"; 8 | 9 | const absPnpApiPath = resolve(__dirname, relPnpApiPath); 10 | const absRequire = createRequire(absPnpApiPath); 11 | 12 | if (existsSync(absPnpApiPath)) { 13 | if (!process.versions.pnp) { 14 | // Setup the environment to be able to require eslint/bin/eslint.js 15 | require(absPnpApiPath).setup(); 16 | } 17 | } 18 | 19 | // Defer to the real eslint/bin/eslint.js your application uses 20 | module.exports = absRequire(`eslint/bin/eslint.js`); 21 | -------------------------------------------------------------------------------- /.yarn/sdks/eslint/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "eslint", 3 | "version": "8.37.0-sdk", 4 | "main": "./lib/api.js", 5 | "type": "commonjs" 6 | } 7 | -------------------------------------------------------------------------------- /.yarn/sdks/integrations.yml: -------------------------------------------------------------------------------- 1 | # This file is automatically generated by @yarnpkg/sdks. 2 | # Manual changes might be lost! 3 | 4 | integrations: 5 | - vscode 6 | -------------------------------------------------------------------------------- /.yarn/sdks/typescript/bin/tsc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const {existsSync} = require(`fs`); 4 | const {createRequire} = require(`module`); 5 | const {resolve} = require(`path`); 6 | 7 | const relPnpApiPath = "../../../../.pnp.cjs"; 8 | 9 | const absPnpApiPath = resolve(__dirname, relPnpApiPath); 10 | const absRequire = createRequire(absPnpApiPath); 11 | 12 | if (existsSync(absPnpApiPath)) { 13 | if (!process.versions.pnp) { 14 | // Setup the environment to be able to require typescript/bin/tsc 15 | require(absPnpApiPath).setup(); 16 | } 17 | } 18 | 19 | // Defer to the real typescript/bin/tsc your application uses 20 | module.exports = absRequire(`typescript/bin/tsc`); 21 | -------------------------------------------------------------------------------- /.yarn/sdks/typescript/bin/tsserver: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const {existsSync} = require(`fs`); 4 | const {createRequire} = require(`module`); 5 | const {resolve} = require(`path`); 6 | 7 | const relPnpApiPath = "../../../../.pnp.cjs"; 8 | 9 | const absPnpApiPath = resolve(__dirname, relPnpApiPath); 10 | const absRequire = createRequire(absPnpApiPath); 11 | 12 | if (existsSync(absPnpApiPath)) { 13 | if (!process.versions.pnp) { 14 | // Setup the environment to be able to require typescript/bin/tsserver 15 | require(absPnpApiPath).setup(); 16 | } 17 | } 18 | 19 | // Defer to the real typescript/bin/tsserver your application uses 20 | module.exports = absRequire(`typescript/bin/tsserver`); 21 | -------------------------------------------------------------------------------- /.yarn/sdks/typescript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "typescript", 3 | "version": "4.9.5-sdk", 4 | "main": "./lib/typescript.js", 5 | "type": "commonjs" 6 | } 7 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | 2 | packageExtensions: 3 | "@fluentui/font-icons-mdl2@*": 4 | peerDependencies: 5 | react: "*" 6 | "@fluentui/react-hooks@*": 7 | peerDependenciesMeta: 8 | "@types/react": 9 | optional: true 10 | "@fluentui/react@*": 11 | peerDependenciesMeta: 12 | "@types/react": 13 | optional: true 14 | "@types/react-dom": 15 | optional: true 16 | react-dom: 17 | optional: true 18 | "@fluentui/style-utilities@*": 19 | peerDependencies: 20 | "@types/react": "*" 21 | react: "*" 22 | "@fluentui/utilities@*": 23 | peerDependenciesMeta: 24 | "@types/react": 25 | optional: true 26 | "@thematic/fluent@*": 27 | peerDependencies: 28 | react-dom: "*" 29 | styled-components: "*" 30 | react-router-dom@*: 31 | peerDependenciesMeta: 32 | react-dom: 33 | optional: true 34 | styled-components@*: 35 | dependencies: 36 | react-is: ^16 37 | peerDependenciesMeta: 38 | react-dom: 39 | optional: true 40 | 41 | plugins: 42 | - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs 43 | spec: "@yarnpkg/plugin-interactive-tools" 44 | - path: .yarn/plugins/@yarnpkg/plugin-typescript.cjs 45 | spec: "@yarnpkg/plugin-typescript" 46 | 47 | yarnPath: .yarn/releases/yarn-3.4.1.cjs 48 | -------------------------------------------------------------------------------- /Backend.dockerfile: -------------------------------------------------------------------------------- 1 | # https://eng.ms/docs/more/containers-secure-supply-chain/approved-images 2 | FROM mcr.microsoft.com/oryx/python:3.9 3 | 4 | # Keeps Python from generating .pyc files in the container 5 | ENV PYTHONDONTWRITEBYTECODE 1 6 | # Turns off buffering for easier container logging 7 | ENV PYTHONUNBUFFERED 1 8 | 9 | # Install necessary dependencies to compile 10 | RUN apt-get update -y \ 11 | && apt-get install -y gcc \ 12 | && apt-get install -y --no-install-recommends curl \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | # Install Poetry 16 | RUN curl -sSL https://install.python-poetry.org | python - --version 1.3.2 17 | ENV PATH="${PATH}:/root/.local/bin" 18 | 19 | WORKDIR /api-backend 20 | 21 | COPY . . 22 | 23 | # Start backend either in API or worker mode, depending on the WORKER env var 24 | 25 | CMD [ "./scripts/start-backend.sh" ] -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in 2 | 3 | # the repo. Unless a later match takes precedence, 4 | 5 | # @global-owner1 and @global-owner2 will be requested for 6 | 7 | # review when someone opens a pull request. 8 | 9 | - @microsoft/societal-resilience 10 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Frontend.dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/cbl-mariner/base/nodejs:16 2 | 3 | RUN npm i -g yarn 4 | WORKDIR /app 5 | 6 | CMD ["./scripts/start-frontend.sh"] 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. 7 | - **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | frontend: 5 | build: 6 | context: . 7 | dockerfile: Frontend.dockerfile 8 | ports: 9 | - 3000:3000 10 | volumes: 11 | - ./:/app 12 | 13 | backendapi: 14 | build: ./python/api-backend 15 | environment: 16 | - ENABLE_CORS=http://localhost:3000 17 | - REDIS_URL=redis://redis:6379/0 18 | - STORAGE=/data 19 | - DB_NAME= 20 | - SQL_ENDPOINT= 21 | - SQL_USERNAME= 22 | - SQL_PASSWORD= 23 | - ACTIVITY_TABLE=entity_activity_report 24 | - ENTITY_TABLE=entity_attributes_report 25 | - GRAPH_TABLE=entity_graph_report 26 | - NETWORK_SCORING_TABLE=network_scoring 27 | - REPORT_TABLE=html_report 28 | - REPORT_URL_TABLE=report_url 29 | ports: 30 | - 8081:8081 31 | depends_on: 32 | - redis 33 | 34 | redis: 35 | image: redis:6-alpine 36 | -------------------------------------------------------------------------------- /docs/deployment/LOCAL_DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Local Development 2 | This tutorial describes how to set up the development environment for the [inference](./inference) component using Docker. 3 | 4 | ## Pre-requisites 5 | 1. [Docker](https://docs.docker.com/engine/install/) 6 | 2. [Visual Studio Code](https://code.visualstudio.com/) 7 | 3. [Visual Studio Code Remote Development Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) 8 | 9 | You can follow the instructions [here](https://code.visualstudio.com/docs/devcontainers/containers) for developing inside a Docker container. 10 | 11 | # Build and Test 12 | TODO: Describe and show how to build your code and run the tests. 13 | -------------------------------------------------------------------------------- /helm/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: transparency-engine 3 | description: Chart to compose microservices 4 | type: application 5 | 6 | # This is the chart version. This version number should be incremented each time you make changes 7 | # to the chart and its templates, including the app version. 8 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 9 | version: 0.1.0 10 | 11 | # This is the version number of the application being deployed. This version number should be 12 | # incremented each time you make changes to the application. Versions are not expected to 13 | # follow Semantic Versioning. They should reflect the version the application is using. 14 | # It is recommended to use it with quotes. 15 | appVersion: "0.1.0" 16 | -------------------------------------------------------------------------------- /helm/aks/nginx-ingress.yaml: -------------------------------------------------------------------------------- 1 | controller: 2 | service: 3 | loadBalancerIP: 20.25.93.29 4 | externalTrafficPolicy: Local 5 | -------------------------------------------------------------------------------- /helm/templates/ingresses.yaml: -------------------------------------------------------------------------------- 1 | {{- $context := . }} 2 | {{- range .Values.applicationGroups }} 3 | {{- $appGroup := . }} 4 | {{- if eq (tpl ($appGroup.disable | default "false") $context) "false" }} 5 | {{- $ingress := $appGroup.ingress | default dict }} 6 | apiVersion: networking.k8s.io/v1 7 | kind: Ingress 8 | metadata: 9 | name: {{ $appGroup.namespace }} 10 | namespace: {{ $appGroup.namespace }} 11 | labels: 12 | app: {{ $appGroup.namespace }} 13 | annotations: 14 | {{- tpl (toYaml $ingress.annotations) $context | nindent 4 }} 15 | spec: 16 | ingressClassName: nginx 17 | tls: 18 | {{- tpl (toYaml $ingress.tls) $context | nindent 4 }} 19 | rules: 20 | - host: {{ tpl $ingress.host $context }} 21 | http: 22 | paths: 23 | {{- range $appGroup.services }} 24 | {{- if .path }} 25 | - pathType: Prefix 26 | path: {{ .path }} 27 | backend: 28 | service: 29 | name: {{ .name }} 30 | port: 31 | name: {{ .name }} 32 | {{- end }} 33 | {{- end }} 34 | --- 35 | {{- end }} 36 | {{- end }} 37 | -------------------------------------------------------------------------------- /helm/templates/namespaces.yaml: -------------------------------------------------------------------------------- 1 | {{- $context := . }} 2 | {{- range .Values.applicationGroups }} 3 | {{- if eq (tpl (.disable | default "false") $context) "false" }} 4 | {{- if not .externalNamespace }} 5 | apiVersion: v1 6 | kind: Namespace 7 | metadata: 8 | name: {{ .namespace }} 9 | labels: 10 | name: {{ .namespace }} 11 | --- 12 | {{- end }} 13 | {{- end }} 14 | {{- end }} 15 | -------------------------------------------------------------------------------- /helm/templates/services.yaml: -------------------------------------------------------------------------------- 1 | {{- $context := . }} 2 | {{- range .Values.applicationGroups }} 3 | {{- $appGroup := . }} 4 | {{- if eq (tpl ($appGroup.disable | default "false") $context) "false" }} 5 | {{- range $appGroup.services }} 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | labels: 10 | app: {{ .name }} 11 | name: {{ .name }} 12 | namespace: {{ $appGroup.namespace }} 13 | spec: 14 | replicas: {{ .replicas }} 15 | revisionHistoryLimit: {{ $context.Values.revisionHistoryLimit }} 16 | selector: 17 | matchLabels: 18 | app: {{ .name }} 19 | template: 20 | metadata: 21 | labels: 22 | app: {{ .name }} 23 | {{- if $context.Values.recreatePodsOnUpgrade }} 24 | annotations: 25 | rollme: {{ randAlphaNum 5 | quote }} 26 | {{- end }} 27 | spec: 28 | containers: 29 | - name: {{ .name }} 30 | image: {{ tpl .image $context }} 31 | imagePullPolicy: {{ tpl .imagePullPolicy $context }} 32 | args: 33 | {{- tpl (toYaml .args) $context | nindent 12 }} 34 | env: 35 | {{- tpl (toYaml .env) $context | nindent 12 }} 36 | resources: 37 | {{- toYaml .resources | nindent 12 }} 38 | ports: 39 | - containerPort: {{ .containerPort }} 40 | {{- if .imagePullSecret }} 41 | imagePullSecrets: 42 | - name: {{ .imagePullSecret }} 43 | {{- end }} 44 | --- 45 | apiVersion: v1 46 | kind: Service 47 | metadata: 48 | labels: 49 | app: {{ .name }} 50 | name: {{ .name }} 51 | namespace: {{ $appGroup.namespace }} 52 | spec: 53 | selector: 54 | app: {{ .name }} 55 | ports: 56 | - port: {{ .servicePort }} 57 | name: {{ .name }} 58 | targetPort: {{ .containerPort }} 59 | --- 60 | {{- end }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /helm/values.local.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Settings for Local Development 3 | # 4 | domain: localhost 5 | enableAuthentication: false 6 | -------------------------------------------------------------------------------- /helm/values.prod.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # production parameters 3 | # (TODO: maybe turn this into pipeline variables and call helm with --set) 4 | # 5 | domain: transparency-engine.eastus.cloudapp.azure.com 6 | -------------------------------------------------------------------------------- /javascript/webapp/.eslintignore: -------------------------------------------------------------------------------- 1 | .yarn/ 2 | .pnp* 3 | build/ 4 | dist/ 5 | lib/ 6 | storybook-static/ 7 | coverage/ 8 | docs/ 9 | docsTemp/ -------------------------------------------------------------------------------- /javascript/webapp/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@essex/eslint-config", 3 | "root": true, 4 | "rules": { 5 | "@typescript-eslint/no-unsafe-argument": "off", 6 | "@typescript-eslint/no-floating-promises": "off", 7 | "@typescript-eslint/require-await": "off" 8 | } 9 | } -------------------------------------------------------------------------------- /javascript/webapp/.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | lib/ 3 | build/ 4 | coverage/ 5 | node_modules/ 6 | .turbo/ 7 | *.zip 8 | *-*.log 9 | 10 | .env.local 11 | .env.development.local 12 | .env.test.local 13 | .env.production.local 14 | 15 | .deploy-config/ 16 | 17 | .env* -------------------------------------------------------------------------------- /javascript/webapp/.prettierignore: -------------------------------------------------------------------------------- 1 | .yarn* 2 | yarn.lock 3 | .pnp.* 4 | 5 | node_modules/ 6 | storybook-static/ 7 | build/ 8 | dist/ 9 | lib/ 10 | coverage/ 11 | public/ 12 | docs/ 13 | 14 | .* 15 | 16 | config/helm/**/templates/*.yaml -------------------------------------------------------------------------------- /javascript/webapp/.prettierrc: -------------------------------------------------------------------------------- 1 | "@essex/prettier-config" -------------------------------------------------------------------------------- /javascript/webapp/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/cbl-mariner/base/nodejs:16 2 | 3 | WORKDIR /app 4 | COPY ./build ./build 5 | COPY package.deployed.json package.json 6 | RUN npm install --production 7 | CMD ["npm", "run", "start"] 8 | -------------------------------------------------------------------------------- /javascript/webapp/package.deployed.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@hrt/webapp", 3 | "private": true, 4 | "version": "0.0.1", 5 | "main": "src/index.ts", 6 | "license": "MIT", 7 | "scripts": { 8 | "start": "http-server build/" 9 | }, 10 | "dependencies": { 11 | "http-server": "^14.1.1" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /javascript/webapp/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "transparency-engine", 3 | "private": true, 4 | "version": "0.0.1", 5 | "main": "src/index.ts", 6 | "license": "MIT", 7 | "contributors": [ 8 | "Alonso Guevara ", 9 | "Chris Trevino ", 10 | "Dayenne Souza ", 11 | "Gaudy Blanco ", 12 | "Ha Trinh ", 14 | "Nathan Evans " 15 | ], 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/microsoft/transparency-engine" 19 | }, 20 | "bugs": { 21 | "url": "https://github.com/microsoft/transparency-engine/issues" 22 | }, 23 | "scripts": { 24 | "clean": "essex clean build dist lib node_modules", 25 | "build": "tsc --noEmit", 26 | "bundle": "essex bundle && shx cp -r public/* build/ && shx cp build/index.html build/404.html", 27 | "start": "essex serve", 28 | "fix": "essex lint --fix" 29 | }, 30 | "dependencies": { 31 | "@essex/hooks": "^4.0.18", 32 | "@fluentui/font-icons-mdl2": "^8.5.10", 33 | "@fluentui/react": "^8.106.2", 34 | "@fluentui/react-hooks": "^8.6.18", 35 | "@fluentui/utilities": "^8.13.8", 36 | "@thematic/core": "^4.0.4", 37 | "@thematic/d3": "^2.0.19", 38 | "@thematic/fluent": "^5.0.5", 39 | "@thematic/react": "^2.1.6", 40 | "@thematic/vega": "^3.0.4", 41 | "dom-to-pdf": "^0.3.2", 42 | "http-server": "^14.1.1", 43 | "jsonpath-plus": "^7.2.0", 44 | "lodash-es": "^4.17.21", 45 | "normalize.css": "^8.0.1", 46 | "react": "^18.2.0", 47 | "react-dom": "^18.2.0", 48 | "react-error-boundary": "^3.1.4", 49 | "react-if": "^4.1.4", 50 | "react-is": "^18.2.0", 51 | "react-router-dom": "^6.8.2", 52 | "recoil": "^0.7.7", 53 | "redbox-react": "^1.6.0", 54 | "styled-components": "^5.3.8", 55 | "vega": "^5.23.0" 56 | }, 57 | "devDependencies": { 58 | "@essex/eslint-config": "^20.3.5", 59 | "@essex/eslint-plugin": "^20.3.12", 60 | "@essex/jest-config": "^21.0.17", 61 | "@essex/prettier-config": "^18.0.4", 62 | "@essex/scripts": "^22.2.0", 63 | "@essex/webpack-config": "^21.0.9", 64 | "@types/lodash": "^4.14.191", 65 | "@types/lodash-es": "^4.17.6", 66 | "@types/node": "^18.14.4", 67 | "@types/react": "^18.0.28", 68 | "@types/react-dom": "^18.0.11", 69 | "@types/react-is": "^17.0.3", 70 | "@types/react-router-dom": "^5.3.3", 71 | "@types/styled-components": "^5.1.26", 72 | "eslint": "^8.27.0", 73 | "html-webpack-plugin": "^5.5.0", 74 | "prettier": "^2.8.4", 75 | "shx": "^0.3.4", 76 | "typescript": "^4.9.5", 77 | "webpack": "^5.75.0", 78 | "webpack-cli": "^4.10.0" 79 | }, 80 | "packageManager": "yarn@3.4.1" 81 | } 82 | -------------------------------------------------------------------------------- /javascript/webapp/public/data/entity-graphs/sample-220.json: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": [ 3 | { 4 | "id": 0, 5 | "name": "107-609-5431", 6 | "type": "phone number", 7 | "relationship": "attribute", 8 | "flag": 0 9 | }, 10 | { 11 | "id": 8, 12 | "name": "220", 13 | "type": "EntityID", 14 | "relationship": "target", 15 | "flag": 0 16 | }, 17 | { 18 | "id": 2, 19 | "name": "212--220", 20 | "type": "sync_activity", 21 | "relationship": "attribute", 22 | "flag": 0 23 | }, 24 | { 25 | "id": 6, 26 | "name": "461==CHRISTOPHER LEE", 27 | "type": "company director", 28 | "relationship": "attribute", 29 | "flag": 0 30 | }, 31 | { 32 | "id": 1, 33 | "name": "212", 34 | "type": "EntityID", 35 | "relationship": "related", 36 | "flag": 1 37 | }, 38 | { 39 | "id": 5, 40 | "name": "443==MELODY WHEELER", 41 | "type": "company director", 42 | "relationship": "attribute", 43 | "flag": 0 44 | }, 45 | { 46 | "id": 7, 47 | "name": "8729 MICHELLE SUMMIT JOHNSONCHESTER HAWAII 99155 UNITED STATES", 48 | "type": "address", 49 | "relationship": "attribute", 50 | "flag": 0 51 | }, 52 | { 53 | "id": 3, 54 | "name": "218", 55 | "type": "EntityID", 56 | "relationship": "related", 57 | "flag": 0 58 | }, 59 | { 60 | "id": 4, 61 | "name": "242", 62 | "type": "EntityID", 63 | "relationship": "related", 64 | "flag": 0 65 | } 66 | ], 67 | "edges": [ 68 | { 69 | "source": 0, 70 | "target": 8 71 | }, 72 | { 73 | "source": 2, 74 | "target": 8 75 | }, 76 | { 77 | "source": 6, 78 | "target": 8 79 | }, 80 | { 81 | "source": 1, 82 | "target": 2 83 | }, 84 | { 85 | "source": 1, 86 | "target": 5 87 | }, 88 | { 89 | "source": 1, 90 | "target": 7 91 | }, 92 | { 93 | "source": 3, 94 | "target": 0 95 | }, 96 | { 97 | "source": 4, 98 | "target": 6 99 | }, 100 | { 101 | "source": 5, 102 | "target": 4 103 | }, 104 | { 105 | "source": 7, 106 | "target": 3 107 | } 108 | ] 109 | } -------------------------------------------------------------------------------- /javascript/webapp/public/data/related-entity-graphs/sample-220-270.json: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": [ 3 | { 4 | "id": 8, 5 | "name": "270", 6 | "type": "EntityID", 7 | "relationship": "related", 8 | "flag": 1 9 | }, 10 | { 11 | "id": 0, 12 | "name": "+1-855-474-3553X795", 13 | "type": "phone number", 14 | "relationship": "attribute", 15 | "flag": 0 16 | }, 17 | { 18 | "id": 6, 19 | "name": "555==JOHN AGUIRRE", 20 | "type": "company director", 21 | "relationship": "attribute", 22 | "flag": 0 23 | }, 24 | { 25 | "id": 7, 26 | "name": "LISAGRAY@EXAMPLE.ORG", 27 | "type": "email", 28 | "relationship": "attribute", 29 | "flag": 0 30 | }, 31 | { 32 | "id": 1, 33 | "name": "220", 34 | "type": "EntityID", 35 | "relationship": "target", 36 | "flag": 0 37 | }, 38 | { 39 | "id": 4, 40 | "name": "3428113904", 41 | "type": "phone number", 42 | "relationship": "attribute", 43 | "flag": 0 44 | }, 45 | { 46 | "id": 5, 47 | "name": "491==STACEY PETERSON", 48 | "type": "beneficial owner", 49 | "relationship": "attribute", 50 | "flag": 0 51 | }, 52 | { 53 | "id": 2, 54 | "name": "240", 55 | "type": "EntityID", 56 | "relationship": "related", 57 | "flag": 0 58 | }, 59 | { 60 | "id": 3, 61 | "name": "294", 62 | "type": "EntityID", 63 | "relationship": "related", 64 | "flag": 0 65 | } 66 | ], 67 | "edges": [ 68 | { 69 | "source": 0, 70 | "target": 8 71 | }, 72 | { 73 | "source": 6, 74 | "target": 8 75 | }, 76 | { 77 | "source": 7, 78 | "target": 8 79 | }, 80 | { 81 | "source": 1, 82 | "target": 0 83 | }, 84 | { 85 | "source": 1, 86 | "target": 4 87 | }, 88 | { 89 | "source": 1, 90 | "target": 5 91 | }, 92 | { 93 | "source": 2, 94 | "target": 6 95 | }, 96 | { 97 | "source": 3, 98 | "target": 7 99 | }, 100 | { 101 | "source": 4, 102 | "target": 3 103 | }, 104 | { 105 | "source": 5, 106 | "target": 2 107 | } 108 | ] 109 | } -------------------------------------------------------------------------------- /javascript/webapp/public/web.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/App.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | /* eslint-disable @typescript-eslint/ban-ts-comment, @typescript-eslint/no-unsafe-return */ 6 | import { BrowserRouter } from 'react-router-dom' 7 | import { RecoilRoot } from 'recoil' 8 | 9 | import { Layout } from './Layout.js' 10 | import { Router } from './Router.js' 11 | import { StyleContext } from './StyleContext.js' 12 | 13 | export const App: React.FC = function App() { 14 | return ( 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/ErrorBoundary.hooks.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useCallback } from 'react' 6 | 7 | export function useOnReset() { 8 | return useCallback(() => { 9 | // reset the state of your app so the error doesn't happen again 10 | }, []) 11 | } 12 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/ErrorBoundary.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { ErrorBoundary as ErrorBoundaryLib } from 'react-error-boundary' 6 | import Redbox from 'redbox-react' 7 | 8 | import { useOnReset } from './ErrorBoundary.hooks.js' 9 | 10 | export const ErrorBoundary: React.FC< 11 | React.PropsWithChildren<{ 12 | /* nothing */ 13 | }> 14 | > = ({ children }) => { 15 | const onReset = useOnReset() 16 | return ( 17 | 18 | {children} 19 | 20 | ) 21 | } 22 | 23 | function ErrorFallback({ 24 | error, 25 | }: { 26 | error: Error 27 | resetErrorBoundary: () => void 28 | }) { 29 | return 30 | } 31 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/Header.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import type { ITheme } from '@fluentui/react' 7 | import styled from 'styled-components' 8 | 9 | // always ensure the header is dark, regardless of mode 10 | export const Container = styled.div` 11 | padding: 0 16px 0 16px; 12 | background: ${({ theme }: { theme: ITheme }) => 13 | theme.isInverted 14 | ? theme.palette.neutralQuaternary 15 | : theme.palette.neutralPrimary}; 16 | border-bottom: 1px solid 17 | ${({ theme }: { theme: ITheme }) => 18 | theme.isInverted 19 | ? theme.palette.neutralTertiary 20 | : theme.palette.neutralSecondary}; 21 | display: flex; 22 | flex-direction: row; 23 | justify-content: space-between; 24 | align-items: center; 25 | height: 42px; 26 | ` 27 | 28 | export const Title = styled.h1` 29 | cursor: pointer; 30 | font-size: 25px; 31 | align-self: center; 32 | margin: 0; 33 | padding: 0; 34 | color: ${({ theme }: { theme: ITheme }) => 35 | theme.isInverted 36 | ? theme.palette.neutralSecondary 37 | : theme.palette.neutralQuaternary}; 38 | ` 39 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/Header.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { memo } from 'react' 6 | import { useNavigate } from 'react-router-dom' 7 | 8 | import { Container, Title } from './Header.styles.js' 9 | 10 | export const Header: React.FC = memo(function Header() { 11 | const navigate = useNavigate() 12 | return ( 13 | 14 | navigate('/')}>Transparency Engine 15 | 16 | ) 17 | }) 18 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/Layout.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import styled from 'styled-components' 7 | 8 | export const Container = styled.div` 9 | display: flex; 10 | flex-direction: column; 11 | width: 100%; 12 | height: 100%; 13 | ` 14 | export const Main = styled.main` 15 | flex: 1; 16 | display: flex; 17 | flex-direction: row; 18 | max-height: calc(100vh - 42px); 19 | max-width: 100%; 20 | min-height: calc(100vh - 42px); 21 | ` 22 | export const Content = styled.article` 23 | flex: 1; 24 | display: flex; 25 | overflow: hidden; 26 | max-height: 100%; 27 | max-width: 100%; 28 | ` 29 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/Layout.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { Spinner } from '@fluentui/react' 6 | import type { PropsWithChildren } from 'react' 7 | import React, { memo, Suspense } from 'react' 8 | 9 | import { Header } from './Header.js' 10 | import { Container, Content, Main } from './Layout.styles.js' 11 | 12 | export const Layout: React.FC = memo(function Layout({ 13 | children, 14 | }) { 15 | return ( 16 | 17 |
18 |
19 | }> 20 | {children} 21 | 22 |
23 | 24 | ) 25 | }) 26 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/Pages.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export const Pages = { 6 | Report: { 7 | name: 'Transparency Engine Report', 8 | path: '/transparency-engine-report', 9 | }, 10 | } 11 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/Router.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | /* eslint-disable @typescript-eslint/ban-ts-comment, @typescript-eslint/no-unsafe-return */ 6 | import { Route, Routes } from 'react-router-dom' 7 | 8 | import { EntityReportPage } from '../pages/EntityReportPage.js' 9 | import { HomePage } from '../pages/HomePage.js' 10 | 11 | export const Router: React.FC = function Router() { 12 | return ( 13 | 14 | } /> 15 | } /> 16 | 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/StyleContext.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { Theme } from '@fluentui/react' 6 | import { createGlobalStyle } from 'styled-components' 7 | 8 | export const GlobalStyle = createGlobalStyle` 9 | * { 10 | box-sizing: border-box; 11 | } 12 | body { 13 | height: 100vh; 14 | margin: 0; 15 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 16 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 17 | sans-serif; 18 | -webkit-font-smoothing: antialiased; 19 | -moz-osx-font-smoothing: grayscale; 20 | 21 | * { 22 | scrollbar-width: thin; 23 | scrollbar-color: #9d9d9d #f1f1f1; 24 | } 25 | 26 | *::-webkit-scrollbar { 27 | width: 12px; 28 | height: 12px 29 | } 30 | 31 | *::-webkit-scrollbar-track { 32 | background: #f1f1f1; 33 | } 34 | 35 | *::-webkit-scrollbar-thumb { 36 | background-color: #9d9d9d; 37 | border-radius: 10px; 38 | border: 2px solid #f1f1f1; 39 | } 40 | } 41 | 42 | :root { 43 | --faint: ${({ theme }: { theme: Theme }) => theme.palette.neutralLighter}; 44 | --focus-border: ${({ theme }: { theme: Theme }) => 45 | theme.palette.neutralTertiaryAlt}; 46 | --separator-border: ${({ theme }: { theme: Theme }) => 47 | theme.palette.neutralTertiaryAlt}; 48 | 49 | } 50 | 51 | .header-command-bar { 52 | border: none; 53 | } 54 | 55 | ` 56 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/StyleContext.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { loadById } from '@thematic/core' 6 | import { loadFluentTheme, ThematicFluentProvider } from '@thematic/fluent' 7 | import { ApplicationStyles } from '@thematic/react' 8 | import { memo, useMemo } from 'react' 9 | import { ThemeProvider } from 'styled-components' 10 | 11 | import { GlobalStyle } from './StyleContext.styles.js' 12 | import type { StyleContextProps } from './StyleContext.types.js' 13 | 14 | export const StyleContext: React.FC = memo( 15 | function StyleContext({ children }) { 16 | const theme = loadById('default', { dark: false }) 17 | const fluentTheme = useMemo(() => loadFluentTheme(theme), [theme]) 18 | return ( 19 | 20 | 21 | 22 | 23 | {children} 24 | 25 | 26 | ) 27 | }, 28 | ) 29 | 30 | const fluentProviderStyle = { height: '100%', width: '100%' } 31 | -------------------------------------------------------------------------------- /javascript/webapp/src/App/StyleContext.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export interface StyleContextProps { 6 | children: React.ReactNode 7 | } 8 | -------------------------------------------------------------------------------- /javascript/webapp/src/api/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import type { GraphData, Report } from '../types' 7 | 8 | // if any entity id has a "sample" prefix, that means we just want to load a test file from the local public folder 9 | 10 | export async function getEntityGraph(entityId: string): Promise { 11 | const url = !entityId.toLowerCase().includes('sample') 12 | ? `api/entities/graph/${entityId}` 13 | : `data/entity-graphs/${entityId}.json` 14 | return fetch(url).then((res) => res.json() as Promise) 15 | } 16 | 17 | export async function getRelatedEntityGraph( 18 | entityId: string, 19 | relatedId: string, 20 | ): Promise { 21 | const url = !entityId.toLowerCase().includes('sample') 22 | ? `api/entities/graph/${entityId}?target=${relatedId}` 23 | : `data/related-entity-graphs/${entityId}-${relatedId}.json` 24 | return fetch(url).then((res) => res.json() as Promise) 25 | } 26 | 27 | export async function getEntityReport(entityId: string): Promise { 28 | const url = !entityId.toLowerCase().includes('sample') 29 | ? `api/report/${entityId}` 30 | : `data/entity-reports/${entityId}.json` 31 | return fetch(url) 32 | .then( 33 | (res) => res.json() as Promise<{ html_report: Omit }>, 34 | ) 35 | .then((json) => json.html_report) // unwrap from the html_report property (TODO: remove this server-side) 36 | .then( 37 | (report) => 38 | ({ 39 | entityId, 40 | ...report, 41 | }) as Report, 42 | ) 43 | } 44 | -------------------------------------------------------------------------------- /javascript/webapp/src/bootstrap.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { initializeIcons } from '@fluentui/react' 6 | import { createRoot } from 'react-dom/client' 7 | 8 | export async function bootstrap( 9 | app: JSX.Element, 10 | rootNodeId = 'root', 11 | ): Promise { 12 | try { 13 | /* eslint-disable-next-line @typescript-eslint/ban-ts-comment */ 14 | // @ts-ignore 15 | await import('normalize.css') 16 | initializeIcons(undefined, { disableWarnings: true }) 17 | 18 | const root = createRoot(getRootElement(rootNodeId)) 19 | root.render(app) 20 | } catch (err) { 21 | console.error('error rendering application', err) 22 | } 23 | } 24 | 25 | function getRootElement(id: string) { 26 | const element = document.getElementById(id) as HTMLElement 27 | if (element == null) { 28 | throw new Error(`could not find root with id "${id}"`) 29 | } 30 | return element 31 | } 32 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/ComplexTableComponent/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './ComplexTableComponent.js' 6 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/GraphComponent/GraphComponent.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import { memo } from 'react' 7 | import styled from 'styled-components' 8 | import type { Spec } from 'vega' 9 | 10 | import { EntityId } from '../../styles/reports.js' 11 | import type { GraphData } from '../../types.js' 12 | import template from '../../vega/force-graph.json' 13 | import { VegaGraph } from '../../vega/VegaGraph.js' 14 | 15 | export const GraphComponent: React.FC<{ 16 | data: GraphData 17 | width: number 18 | height: number 19 | title?: string 20 | }> = memo(function GraphComponent({ data, width, height, title }) { 21 | return ( 22 | 23 | {title && {title}} 24 | 30 | 31 | ) 32 | }) 33 | 34 | const Container = styled.div`` 35 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/GraphComponent/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './GraphComponent.js' 6 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/ReportTemplateComponent/ReportTemplate.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import { memo } from 'react' 7 | import styled from 'styled-components' 8 | 9 | import type { GraphData, Report } from '../../types.js' 10 | import { ReportTemplateSection } from './ReportTemplateSection.js' 11 | 12 | export interface ReportTemplateProps { 13 | report: Report 14 | relatedGraphs: Map | undefined 15 | } 16 | 17 | export const ReportTemplate: React.FC = memo( 18 | function ReportTemplate({ report, relatedGraphs }) { 19 | return ( 20 | 21 | {report?.reportSections.map((section) => ( 22 | 27 | ))} 28 | 29 | ) 30 | }, 31 | ) 32 | 33 | const Container = styled.div` 34 | display: flex; 35 | flex-direction: column; 36 | gap: 20px; 37 | ` 38 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/ReportTemplateComponent/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './ReportTemplate.js' 6 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/SingleChartComponent/SingleChartComponent.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | import { memo } from 'react' 7 | import styled from 'styled-components' 8 | import type { Spec } from 'vega' 9 | 10 | import { useBarChartData } from '../../hooks/useBarChartData.js' 11 | import { useDonutChartData } from '../../hooks/useDonutChartData.js' 12 | import type { 13 | RelatedEntityActivity, 14 | TargetEntityActivity, 15 | } from '../../types.js' 16 | import barChartTemplate from '../../vega/bar-chart.json' 17 | import donutChartTemplate from '../../vega/donut-chart.json' 18 | import { VegaChart } from '../../vega/VegaChart.js' 19 | 20 | export const SingleChartComponent: React.FC<{ 21 | target: TargetEntityActivity 22 | related: RelatedEntityActivity 23 | }> = memo(function SingleChartComponent({ target, related }) { 24 | //data manipulation 25 | const donutChartData = useDonutChartData(target, related) 26 | const barChartData = useBarChartData(target, related) 27 | 28 | return ( 29 | 30 | 37 | 38 | 45 | 46 | ) 47 | }) 48 | 49 | const Container = styled.div` 50 | display: flex; 51 | align-items: flex-start; 52 | justify-content: space-between; 53 | ` 54 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/SingleChartComponent/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './SingleChartComponent.js' 6 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/TableComponent/TableComponent.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { memo } from 'react' 6 | import { Case, Switch } from 'react-if' 7 | 8 | import { 9 | SubsectionContainer, 10 | SubsectionDescription, 11 | SubsectionTitle, 12 | } from '../../styles/reports.js' 13 | import type { 14 | AttributeBlock, 15 | AttributeCount, 16 | AttributeCountPercentRank, 17 | EntityValues, 18 | Flag, 19 | PrimitiveArray, 20 | TableAttribute, 21 | } from '../../types.js' 22 | import { ReportType } from '../../types.js' 23 | import { AttributeCountsCountPercentRankRow } from '../tables/AttributeCountsCountPercentRankRow.js' 24 | import { AttributeCountsCountPercentRow } from '../tables/AttributeCountsCountPercentRow.js' 25 | import { AttributeCountsCountRow } from '../tables/AttributeCountsCountRow.js' 26 | import { AttributeValuesListRow } from '../tables/AttributeValuesListRow.js' 27 | import { AttributeValuesOwnFlagsRow } from '../tables/AttributeValuesOwnFlagsRow.js' 28 | import { Table, Tbody } from '../tables/styles.js' 29 | import { TableHead } from '../tables/TableHead.js' 30 | 31 | export const TableComponent: React.FC<{ 32 | dataObject: AttributeBlock 33 | }> = memo(function TableComponent({ dataObject }) { 34 | const { type } = dataObject 35 | return ( 36 | 37 | {dataObject.title} 38 | {dataObject.intro} 39 | 40 | {dataObject.columns && } 41 | 42 | {dataObject.data?.map((row, ridx) => ( 43 | 44 | ))} 45 | 46 |
47 | 48 | ) 49 | }) 50 | 51 | const Row = ({ row, type }) => ( 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | ) 72 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/TableComponent/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './TableComponent.js' 6 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/VegaHost.constants.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | export const MIN_SPEC_ADDITIONAL_PADDING = 2 7 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/VegaHost.types.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import * as vega from 'vega' 6 | 7 | export enum LogLevel { 8 | None = vega.None, 9 | Error = vega.Error, 10 | Warn = vega.Warn, 11 | Info = vega.Info, 12 | Debug = vega.Debug, 13 | } 14 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/hooks/composition.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { isArray, merge, mergeWith } from 'lodash-es' 6 | import { Children, useMemo } from 'react' 7 | import type { Spec } from 'vega' 8 | /* eslint-disable */ 9 | 10 | /** 11 | * Lodash merge customizer to ensure that arrays are note overwritten, but rather concatenated. 12 | * In Vega, most root properties are arrays (data, signals, axes, etc.), so we want to concat 13 | * if we're allowing composition from multiple specs. 14 | * @param objValue 15 | * @param srcValue 16 | * @returns 17 | */ 18 | function customizer(objValue: unknown, srcValue: unknown) { 19 | if (isArray(objValue)) { 20 | return objValue.concat(srcValue) 21 | } 22 | } 23 | 24 | /** 25 | * Take a base spec and any child components with type-specific specs, and merge them together. 26 | * @param spec 27 | * @param children 28 | */ 29 | // ReactChildren doesn't type correctly with props 30 | // eslint-disable-next-line 31 | export function useMergeChildSpecs(spec: Spec, children: any): Spec { 32 | return useMemo(() => { 33 | let merged = merge({}, spec) 34 | Children.forEach(children, (child) => { 35 | merged = mergeWith(merged, child.props.spec, customizer) 36 | }) 37 | return merged 38 | }, [spec, children]) 39 | } 40 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/hooks/data.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useEffect } from 'react' 6 | import type { DataListenerHandler, View } from 'vega' 7 | 8 | // eslint-disable-next-line 9 | export function useData(view: View, data: { [key: string]: any }): void { 10 | useEffect(() => { 11 | Object.entries(data).forEach(([name, value]) => { 12 | view.data(name, value) 13 | }) 14 | }, [view, data]) 15 | } 16 | 17 | export function useDataListeners( 18 | view: View, 19 | listeners: { [key: string]: DataListenerHandler }, 20 | ): void { 21 | useEffect(() => { 22 | Object.entries(listeners).forEach(([name, value]) => { 23 | view.addDataListener(name, value) 24 | }) 25 | return () => { 26 | if (view) { 27 | Object.entries(listeners).forEach(([name, value]) => { 28 | view.removeDataListener(name, value) 29 | }) 30 | } 31 | } 32 | }, [view, listeners]) 33 | } 34 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/hooks/events.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useCallback, useEffect } from 'react' 6 | import type { EventListenerHandler, View } from 'vega' 7 | /* eslint-disable */ 8 | 9 | export function useAddClickHandler( 10 | view: View, 11 | onDatumClick?: (datum: any, coordinates: { x: number; y: number }) => void, 12 | onAxisClick?: (datum: any, axis: string) => void, 13 | ): void { 14 | const handleClick = useCallback( 15 | (_e: any, item: any) => { 16 | const { datum, mark } = item 17 | if (mark.role.includes('axis')) { 18 | const axis = item.align === 'left' || item.align === 'right' ? 'y' : 'x' 19 | onAxisClick?.(datum, axis) 20 | } else { 21 | onDatumClick?.(datum, { x: item.x, y: item.y }) 22 | } 23 | }, 24 | [onDatumClick, onAxisClick], 25 | ) 26 | useEffect(() => { 27 | view.addEventListener('click', handleClick) 28 | return () => { 29 | view.removeEventListener('click', handleClick) 30 | } 31 | }, [view, handleClick]) 32 | } 33 | 34 | export function useAddMouseOverHandler( 35 | view: View, 36 | onDatumMouseOver?: (datum: any) => void, 37 | ): void { 38 | const handleMouseOver = useCallback( 39 | (_e: any, item: any) => { 40 | const { datum } = item 41 | onDatumMouseOver?.(datum) 42 | }, 43 | [onDatumMouseOver], 44 | ) 45 | useEffect(() => { 46 | view.addEventListener('mouseover', handleMouseOver) 47 | return () => { 48 | view.removeEventListener('mouseover', handleMouseOver) 49 | } 50 | }, [view, handleMouseOver]) 51 | } 52 | 53 | export function useEventListeners( 54 | view: View, 55 | listeners: { [key: string]: EventListenerHandler }, 56 | ): void { 57 | useEffect(() => { 58 | Object.entries(listeners).forEach(([name, value]) => { 59 | view.addEventListener(name, value) 60 | }) 61 | return () => { 62 | if (view) { 63 | Object.entries(listeners).forEach(([name, value]) => { 64 | view.removeEventListener(name, value) 65 | }) 66 | } 67 | } 68 | }, [view, listeners]) 69 | } 70 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/hooks/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './composition.js' 6 | export * from './data.js' 7 | export * from './events.js' 8 | export * from './log.js' 9 | export * from './signals.js' 10 | export * from './view.js' 11 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/hooks/log.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useEffect } from 'react' 6 | import type { View } from 'vega' 7 | 8 | import type { LogLevel } from '../VegaHost.types.js' 9 | 10 | export function useLogLevel(view: View, logLevel: LogLevel): void { 11 | useEffect(() => { 12 | view.logLevel(logLevel) 13 | }, [view, logLevel]) 14 | } 15 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/hooks/signals.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useEffect } from 'react' 6 | import type { SignalListenerHandler, SignalValue, View } from 'vega' 7 | 8 | export function useSignals( 9 | view: View, 10 | signals: { [key: string]: SignalValue }, 11 | ): void { 12 | useEffect(() => { 13 | Object.entries(signals).forEach(([name, value]) => { 14 | view.signal(name, value) 15 | }) 16 | }, [view, signals]) 17 | } 18 | 19 | export function useSignalListeners( 20 | view: View, 21 | listeners: { [key: string]: SignalListenerHandler }, 22 | ): void { 23 | useEffect(() => { 24 | Object.entries(listeners).forEach(([name, value]) => { 25 | view.addSignalListener(name, value) 26 | }) 27 | return () => { 28 | if (view) { 29 | Object.entries(listeners).forEach(([name, value]) => { 30 | view.removeSignalListener(name, value) 31 | }) 32 | } 33 | } 34 | }, [view, listeners]) 35 | } 36 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/hooks/view.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useThematic } from '@thematic/react' 6 | import { vega } from '@thematic/vega' 7 | import { useEffect, useMemo } from 'react' 8 | import { parse, type Spec, View } from 'vega' 9 | 10 | export function useCreateView(spec: Spec, width: number, height: number): View { 11 | const theme = useThematic() 12 | return useMemo(() => { 13 | const themed = vega(theme, spec, { width, height }) 14 | const parsed = parse(themed) 15 | return new View(parsed).renderer('svg') 16 | }, [theme, spec, width, height]) 17 | } 18 | 19 | export function useInitializeView( 20 | ref: React.MutableRefObject, 21 | view: View, 22 | ): void { 23 | useEffect(() => { 24 | if (ref?.current !== null) { 25 | // eslint-disable-next-line 26 | view.initialize(ref.current as any).run() 27 | } 28 | }, [ref, view]) 29 | } 30 | 31 | export function useOnCreateView( 32 | view: View, 33 | onCreateView?: (view: View) => void, 34 | ): void { 35 | useEffect(() => { 36 | onCreateView?.(view) 37 | }, [view, onCreateView]) 38 | } 39 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/VegaHost/index.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | export * from './VegaHost.js' 6 | export * from './VegaHost.types.js' 7 | export * from './VegaHost.util.js' 8 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/AttributeCountsCountPercentRankRow.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | /*! 6 | * Copyright (c) Microsoft. All rights reserved. 7 | * Licensed under the MIT license. See LICENSE file in the project. 8 | */ 9 | 10 | import type { AttributeCountPercentRank } from '../../types.js' 11 | import { NumberCell, TextCell, Tr } from './styles.js' 12 | 13 | export interface AttributeCountsCountPercentRankRowProps { 14 | row: AttributeCountPercentRank 15 | } 16 | 17 | // TODO: the 100 - x math will be done in the service layer 18 | export const AttributeCountsCountPercentRankRow: React.FC< 19 | AttributeCountsCountPercentRankRowProps 20 | > = ({ row }) => ( 21 | 22 | {row[0]} 23 | {row[1]} 24 | {`${row[2]}%`} 25 | 26 | ) 27 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/AttributeCountsCountPercentRow.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import styled from 'styled-components' 6 | 7 | import type { EntityValues } from '../../types.js' 8 | import { 9 | Divider, 10 | FlexColumn, 11 | FlexRow, 12 | Key, 13 | Td, 14 | TextCell, 15 | Tr, 16 | Value, 17 | } from './styles.js' 18 | 19 | export interface AttributeCountsCountPercentRowProps { 20 | row: EntityValues 21 | } 22 | 23 | export const AttributeCountsCountPercentRow: React.FC< 24 | AttributeCountsCountPercentRowProps 25 | > = ({ row }) => ( 26 | 27 | {row.key} 28 | 29 | 30 | {row.value.map((activity, index) => { 31 | return ( 32 | 33 | 34 | EntityId {activity.entity_id} 35 | 36 | | 37 | 38 | Value 39 | {activity.value} 40 | 41 | 42 | ) 43 | })} 44 | 45 | 46 | 47 | ) 48 | 49 | const Pair = styled(FlexRow)` 50 | justify-content: space-between; 51 | ` 52 | 53 | const EntityRow = styled(FlexRow)` 54 | min-width: 160px; 55 | ` 56 | 57 | const ValueRow = styled(FlexRow)` 58 | width: 100px; 59 | ` 60 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/AttributeCountsCountRow.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { AttributeCount } from '../../types.js' 6 | import { NumberCell, TextCell, Tr } from './styles.js' 7 | 8 | export interface AttributeCountsCountRowProps { 9 | row: AttributeCount 10 | } 11 | 12 | export const AttributeCountsCountRow: React.FC = 13 | ({ row }) => ( 14 | 15 | {row[0]} 16 | {row[1]} 17 | 18 | ) 19 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/AttributeValuesListRow.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import styled from 'styled-components' 6 | 7 | import type { PrimitiveArray } from '../../types.js' 8 | import { FlexColumn, Td, TextCell, Tr } from './styles.js' 9 | 10 | export interface AttributeValuedListRowProps { 11 | row: PrimitiveArray[] 12 | } 13 | 14 | export const AttributeValuesListRow: React.FC = ({ 15 | row, 16 | }) => ( 17 | 18 | {row[0]} 19 | 20 | 21 | {row[1].map((value, index) => ( 22 | {value} 23 | ))} 24 | 25 | 26 | 27 | ) 28 | 29 | const Rows = styled(FlexColumn)` 30 | gap: 8px; 31 | ` 32 | 33 | const Row = styled.div`` 34 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/AttributeValuesOwnFlagsRow.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import styled from 'styled-components' 6 | 7 | import type { Flag } from '../../types.js' 8 | import { EvidenceCellContent } from './EvidenceCellContent.js' 9 | import { Td, TextCell, Tr } from './styles.js' 10 | 11 | export interface AttributeValuesOwnFlagsRowProps { 12 | row: Flag 13 | } 14 | 15 | export const AttributeValuesOwnFlagsRow: React.FC< 16 | AttributeValuesOwnFlagsRowProps 17 | > = ({ row }) => ( 18 | 19 | {row.flag} 20 | 21 | 22 | 23 | 24 | ) 25 | 26 | const FlagCell = styled(TextCell)` 27 | width: 50%; 28 | ` 29 | 30 | const EvidenceCell = styled(Td)` 31 | width: 50%; 32 | ` 33 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/AttributeValuesRelatedFlagsMeasurementsRow.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import styled from 'styled-components' 6 | 7 | import type { FlagMeasurement, FlagMeasurementEntry, Paths } from '../../types.js' 8 | import { FlexColumn, Key, Td, Tr, Value, MeasureName } from './styles.js' 9 | 10 | export interface AttributeValuesRelatedFlagsMeasurementsRowProps { 11 | row: FlagMeasurementEntry 12 | } 13 | 14 | export const AttributeValuesRelatedFlagsMeasurementsRow: React.FC< 15 | AttributeValuesRelatedFlagsMeasurementsRowProps 16 | > = ({ row }) => ( 17 | 18 | 19 | 20 | 21 | ) 22 | 23 | const PathsCell = ({ paths }: { paths: Paths }) => ( 24 | 25 | 26 | Direct paths 27 | 28 | {paths.direct_paths.map((path, index) => ( 29 | {path} 30 | ))} 31 | 32 | 33 | 34 | Indirect paths 35 | {paths.indirect_paths} 36 | 37 | 38 | ) 39 | 40 | const PathsContainer = styled(Td)`` 41 | 42 | const DirectPathsContainer = styled.div`` 43 | const DirectPathsList = styled.div` 44 | display: flex; 45 | flex-direction: column; 46 | gap: 8px; 47 | padding: 8px 8px 8px 20px; 48 | margin-bottom: 20px; 49 | ` 50 | const DirectPathsValue = styled.div`` 51 | 52 | const IndirectPathsContainer = styled.div` 53 | display: flex; 54 | gap: 8px; 55 | ` 56 | 57 | const MeasurementsCell = ({ measurements }: { measurements: FlagMeasurement[] }) => ( 58 | 59 | 60 | {measurements.map((measure, fidx) => ( 61 | 62 | {measure.key} 63 | {measure.value}% 64 | 65 | ))} 66 | 67 | 68 | ) 69 | 70 | const FlagsContainer = styled(Td)`` 71 | 72 | const FlagsColumn = styled(FlexColumn)` 73 | gap: 20px; 74 | ` 75 | 76 | const MeasureContainer = styled.div` 77 | gap: 8px; 78 | display: flex; 79 | ` 80 | 81 | const MeasureValue = styled.div` 82 | display: inline-block; 83 | ` 84 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/AttributeValuesRelatedFlagsRow.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import styled from 'styled-components' 6 | 7 | import type { Flag, FlagEntry, Paths } from '../../types.js' 8 | import { EvidenceCellContent } from './EvidenceCellContent.js' 9 | import { FlexColumn, Key, Td, Tr, Value } from './styles.js' 10 | 11 | export interface AttributeValuesRelatedFlagsRowProps { 12 | row: FlagEntry 13 | } 14 | 15 | export const AttributeValuesRelatedFlagsRow: React.FC< 16 | AttributeValuesRelatedFlagsRowProps 17 | > = ({ row }) => ( 18 | 19 | 20 | 21 | 22 | ) 23 | 24 | const PathsCell = ({ paths }: { paths: Paths }) => ( 25 | 26 | 27 | Direct paths 28 | 29 | {paths.direct_paths.map((path, index) => ( 30 | {path} 31 | ))} 32 | 33 | 34 | 35 | Indirect paths 36 | {paths.indirect_paths} 37 | 38 | 39 | ) 40 | 41 | const PathsContainer = styled(Td)`` 42 | 43 | const DirectPathsContainer = styled.div`` 44 | const DirectPathsList = styled.div` 45 | display: flex; 46 | flex-direction: column; 47 | gap: 8px; 48 | padding: 8px 8px 8px 20px; 49 | margin-bottom: 20px; 50 | ` 51 | const DirectPathsValue = styled.div`` 52 | 53 | const IndirectPathsContainer = styled.div` 54 | display: flex; 55 | gap: 8px; 56 | ` 57 | 58 | const FlagCell = ({ flags }: { flags: Flag[] }) => ( 59 | 60 | 61 | {flags.map((flag, fidx) => ( 62 | 63 | {flag.flag} 64 | 65 | 66 | ))} 67 | 68 | 69 | ) 70 | 71 | const FlagsContainer = styled(Td)`` 72 | 73 | const FlagsColumn = styled(FlexColumn)` 74 | gap: 20px; 75 | ` 76 | 77 | const FlagContainer = styled(FlexColumn)` 78 | gap: 8px; 79 | ` 80 | 81 | const FlagName = styled.div`` 82 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/EvidenceCellContent.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import styled from 'styled-components' 6 | 7 | import type { Evidence } from '../../types.js' 8 | import { FlexColumn, FlexRow, Key, Value } from './styles.js' 9 | 10 | export interface EvidenceCellContentProps { 11 | evidence: Evidence[] 12 | } 13 | export const EvidenceCellContent: React.FC = ({ 14 | evidence, 15 | }) => ( 16 | 17 | {evidence !== undefined && evidence.map((evidence, index) => ( 18 | 19 | {typeof evidence === 'string' && {evidence}} 20 | 21 | {typeof evidence !== 'string' && 22 | Object.entries(evidence).map(([key, value], eidx) => ( 23 | 24 | {key} 25 | {value} 26 | 27 | ))} 28 | 29 | ))} 30 | 31 | ) 32 | 33 | const Container = styled(FlexColumn)` 34 | gap: 20px; 35 | ` 36 | 37 | const EvidenceRow = styled(FlexColumn)` 38 | gap: 4px; 39 | ` 40 | 41 | const EvidenceEntry = styled(FlexRow)` 42 | font-size: 0.9em; 43 | justify-content: space-between; 44 | ` 45 | 46 | const EvidenceKey = styled(Key)` 47 | min-width: 160px; 48 | ` 49 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/TableHead.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { Th, Thead, Tr } from './styles.js' 6 | 7 | export interface TableHeadProps { 8 | columns: string[] 9 | } 10 | 11 | export const TableHead: React.FC = ({ columns }) => ( 12 | 13 | 14 | {columns?.map((col) => ( 15 | {col} 16 | ))} 17 | 18 | 19 | ) 20 | -------------------------------------------------------------------------------- /javascript/webapp/src/components/tables/styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ITheme } from '@fluentui/react' 6 | import styled from 'styled-components' 7 | 8 | export const Table = styled.table` 9 | border: 1px solid 10 | ${({ theme }: { theme: ITheme }) => theme.palette.neutralQuaternary}; 11 | border-collapse: collapse; 12 | ` 13 | 14 | export const Thead = styled.thead` 15 | background-color: ${({ theme }: { theme: ITheme }) => 16 | theme.palette.themePrimary}; 17 | color: ${({ theme }: { theme: ITheme }) => theme.palette.white}; 18 | ` 19 | 20 | export const Tbody = styled.tbody`` 21 | 22 | export const Th = styled.th` 23 | background-color: ${({ theme }: { theme: ITheme }) => 24 | theme.palette.themePrimary}; 25 | padding: 6px 10px; 26 | ` 27 | 28 | export const Tr = styled.tr` 29 | :nth-child(even) { 30 | background-color: ${({ theme }: { theme: ITheme }) => 31 | theme.palette.neutralLighter}; 32 | } 33 | ` 34 | 35 | export const Td = styled.td` 36 | white-space: pre-wrap; 37 | border-bottom: 1px solid 38 | ${({ theme }: { theme: ITheme }) => theme.palette.neutralQuaternaryAlt}; 39 | padding: 6px 10px; 40 | ` 41 | 42 | // semantic cell types for default formatting 43 | export const TextCell = styled(Td)`` 44 | export const NumberCell = styled(Td)` 45 | text-align: right; 46 | ` 47 | export const EntityIdCell = styled(TextCell)`` 48 | 49 | // useful content types for consistent visuals 50 | export const Divider = styled.div` 51 | color: ${({ theme }: { theme: ITheme }) => theme.palette.neutralTertiary}; 52 | ` 53 | 54 | export const MeasureName = styled.div` 55 | display: inline-block; 56 | font-weight: bold; 57 | color: ${({ theme }: { theme: ITheme }) => theme.palette.neutralSecondary}; 58 | &:after { 59 | content: ':'; 60 | } 61 | ` 62 | 63 | export const Key = styled.div` 64 | font-weight: bold; 65 | color: ${({ theme }: { theme: ITheme }) => theme.palette.neutralSecondary}; 66 | &:after { 67 | content: ':'; 68 | } 69 | ` 70 | 71 | export const Value = styled.div`` 72 | 73 | export const FlexRow = styled.div` 74 | display: flex; 75 | flex-direction: row; 76 | gap: 12px; 77 | ` 78 | 79 | export const FlexColumn = styled.div` 80 | display: flex; 81 | flex-direction: column; 82 | gap: 12px; 83 | ` 84 | -------------------------------------------------------------------------------- /javascript/webapp/src/declarations.d.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | 6 | declare module 'dom-to-pdf' { 7 | export interface PdfOptions { 8 | filename: string 9 | compression?: string 10 | overrideWidth?: number 11 | } 12 | export type Handler = () => void 13 | 14 | const pdf: ( 15 | element: Element | null, 16 | options: PdfOptions, 17 | onComplete?: Handler, 18 | ) => Promise 19 | 20 | export default pdf 21 | } 22 | -------------------------------------------------------------------------------- /javascript/webapp/src/hooks/useBarChartData.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useMemo } from 'react' 6 | 7 | import type { RelatedEntityActivity, TargetEntityActivity } from '../types.js' 8 | 9 | export interface TimedActivityEntry { 10 | position: number 11 | value: number 12 | time: string 13 | } 14 | 15 | /** 16 | * Takes the original service data and parses it to make sure it is ready for our charts. 17 | * @param target 18 | * @param related 19 | */ 20 | export function useBarChartData( 21 | target: TargetEntityActivity, 22 | related: RelatedEntityActivity, 23 | ): TimedActivityEntry[] { 24 | return useMemo(() => { 25 | const result: TimedActivityEntry[] = [] 26 | 27 | const times = new Set() 28 | 29 | target.value.forEach((targetElement) => { 30 | times.add(targetElement.time) 31 | }) 32 | 33 | related.activity.forEach((relatedElement) => { 34 | times.add(relatedElement.time) 35 | }) 36 | 37 | for (const time of times) { 38 | let targetCounter = 0 39 | let relatedCounter = 0 40 | let sharedCounter = 0 41 | 42 | target.value.forEach((targetElement) => { 43 | //means is target only 44 | if (targetElement.time === time) { 45 | if ( 46 | related.activity.find( 47 | (e) => e.value === targetElement.value && e.time === time, 48 | ) === undefined 49 | ) { 50 | targetCounter++ 51 | } 52 | } 53 | }) 54 | 55 | related.activity.forEach((relatedElement) => { 56 | //means is related only 57 | if (relatedElement.time === time) { 58 | if ( 59 | target.value.find( 60 | (e) => e.value === relatedElement.value && e.time === time, 61 | ) === undefined 62 | ) { 63 | relatedCounter++ 64 | } 65 | } 66 | }) 67 | 68 | if (target.value.filter((e) => e.time === time).length !== undefined) { 69 | const resultNumber = target.value.filter((e) => e.time === time).length 70 | 71 | sharedCounter = resultNumber - targetCounter 72 | } else { 73 | sharedCounter = 0 74 | } 75 | 76 | result.push( 77 | { position: 0, value: targetCounter, time }, 78 | { position: 1, value: relatedCounter, time }, 79 | { position: 2, value: sharedCounter, time }, 80 | ) 81 | } 82 | 83 | return result.sort((a, b) => a.time.localeCompare(b.time)) 84 | }, [target, related]) 85 | } 86 | -------------------------------------------------------------------------------- /javascript/webapp/src/hooks/useDonutChartData.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useMemo } from 'react' 6 | 7 | import type { RelatedEntityActivity, TargetEntityActivity } from '../types.js' 8 | 9 | export interface QuarterlyActivitySummary { 10 | id: string 11 | field: number 12 | } 13 | 14 | /** 15 | * Takes the original service data and parses it to make sure it is ready for our charts. 16 | * @param target 17 | * @param related 18 | */ 19 | export function useDonutChartData( 20 | target: TargetEntityActivity, 21 | related: RelatedEntityActivity, 22 | ): QuarterlyActivitySummary[] { 23 | return useMemo(() => { 24 | let targetCounter = 0 25 | let relatedCounter = 0 26 | let sharedCounter = 0 27 | 28 | target.value.forEach((targetElement) => { 29 | //means is target only 30 | if ( 31 | related.activity.find((e) => e.value === targetElement.value) === 32 | undefined 33 | ) { 34 | targetCounter++ 35 | } 36 | }) 37 | 38 | related.activity.forEach((relatedElement) => { 39 | //means is related only 40 | if ( 41 | target.value.find((e) => e.value === relatedElement.value) === undefined 42 | ) { 43 | relatedCounter++ 44 | } 45 | }) 46 | 47 | sharedCounter = target.value.length - targetCounter 48 | 49 | return [ 50 | { id: 'selected_only', field: targetCounter }, 51 | { id: 'related_only', field: relatedCounter }, 52 | { id: 'shared', field: sharedCounter }, 53 | ] 54 | }, [target, related]) 55 | } 56 | -------------------------------------------------------------------------------- /javascript/webapp/src/hooks/useGraphData.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useThematic } from '@thematic/react' 6 | import { useMemo } from 'react' 7 | 8 | import type { GraphData } from '../types.js' 9 | import { addGraphEncodings } from '../utils/graphs.js' 10 | /** 11 | * Takes the original service data and parses it to make sure it is ready for our graphs. 12 | * @param original 13 | */ 14 | export function useGraphData( 15 | data: GraphData | undefined, 16 | ): GraphData | undefined { 17 | const theme = useThematic() 18 | return useMemo(() => { 19 | if (data) { 20 | return addGraphEncodings(data, theme) 21 | } 22 | }, [theme, data]) 23 | } 24 | 25 | export function useGraphDataMap( 26 | data: Map | undefined, 27 | ): Map | undefined { 28 | const theme = useThematic() 29 | return useMemo(() => { 30 | if (data) { 31 | const transformed = new Map() 32 | data.forEach((value, key) => { 33 | transformed.set(key, addGraphEncodings(value, theme)) 34 | }) 35 | return transformed 36 | } 37 | return data 38 | }, [theme, data]) 39 | } 40 | -------------------------------------------------------------------------------- /javascript/webapp/src/index.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { App } from './App/App.js' 6 | import { bootstrap } from './bootstrap.js' 7 | 8 | void bootstrap() 9 | -------------------------------------------------------------------------------- /javascript/webapp/src/pages/EntityReportPage.styles.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ITheme } from '@fluentui/react' 6 | import styled from 'styled-components' 7 | 8 | export const Container = styled.div` 9 | overflow: auto; 10 | width: 100%; 11 | ` 12 | 13 | export const Header = styled.div` 14 | padding: 4px; 15 | ` 16 | 17 | export const Download = styled.div` 18 | display: flex; 19 | gap: 12px; 20 | align-items: center; 21 | ` 22 | 23 | export const Content = styled.div` 24 | display: flex; 25 | flex-direction: column; 26 | align-items: center; 27 | ` 28 | 29 | export const Report = styled.div<{ width: number }>` 30 | display: flex; 31 | flex-direction: column; 32 | gap: 40px; 33 | width: ${({ width }) => width}px; 34 | font-size: 1.3em; 35 | ` 36 | 37 | export const Intro = styled.div` 38 | display: flex; 39 | flex-direction: column; 40 | gap: 12px; 41 | ` 42 | export const IntroTitle = styled.h1` 43 | color: ${({ theme }: { theme: ITheme }) => theme.palette.themePrimary}; 44 | margin-bottom: 0; 45 | ` 46 | export const IntroText = styled.div`` 47 | export const IntroId = styled.div` 48 | font-weight: bold; 49 | color: ${({ theme }: { theme: ITheme }) => theme.palette.neutralSecondary}; 50 | ` 51 | 52 | export const GraphContainer = styled.div<{ width: number; height: number }>` 53 | display: flex; 54 | flex-direction: column; 55 | width: ${({ width }) => width}px; 56 | height: ${({ height }) => height}px; 57 | justify-content: center; 58 | align-items: center; 59 | ` 60 | export const Caption = styled.div` 61 | margin-top: 10px; 62 | font-style: italic; 63 | font-size: 0.7em; 64 | ` 65 | -------------------------------------------------------------------------------- /javascript/webapp/src/pages/HomePage.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { PrimaryButton, TextField } from '@fluentui/react' 6 | import { memo, useCallback, useEffect, useRef, useState } from 'react' 7 | import { useNavigate } from 'react-router-dom' 8 | import styled from 'styled-components' 9 | 10 | export const HomePage: React.FC = memo(function HomePage() { 11 | const navigate = useNavigate() 12 | const [id, setId] = useState('sample-220') 13 | const handleChange = useCallback((_e, value) => setId(value), []) 14 | const handleClick = useCallback(() => { 15 | navigate(`/report/${id}`) 16 | }, [navigate, id]) 17 | const handleKey = useCallback((e) => { 18 | e.keyCode === 13 && handleClick() 19 | }, [handleClick]) 20 | const ref = useRef() 21 | useEffect(() => { 22 | // HACK: TextField doesn't expose regular ref properly, so we have to get the root and drilldown to the input 23 | const input = ref.current.firstChild.firstChild.firstChild 24 | input.focus() 25 | },[]) 26 | return ( 27 | 28 |

29 | It looks like you were trying to find an entity report, but didn't 30 | supply an entity ID. Please enter one in the textbox below. 31 |

32 | 33 | 34 | 35 | Load 36 | 37 | 38 | ) 39 | }) 40 | 41 | const Container = styled.div` 42 | overflow: auto; 43 | width: 100%; 44 | padding: 20px; 45 | ` 46 | 47 | const Inputs = styled.div` 48 | width: 280px; 49 | display: flex; 50 | gap: 8px; 51 | ` 52 | -------------------------------------------------------------------------------- /javascript/webapp/src/styles/reports.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { ITheme } from '@fluentui/react' 6 | import styled from 'styled-components' 7 | 8 | export const SectionContainer = styled.section`` 9 | 10 | export const SectionTitle = styled.h2` 11 | color: ${({ theme }: { theme: ITheme }) => theme.palette.themePrimary}; 12 | ` 13 | 14 | export const SectionDescription = styled.p`` 15 | 16 | export const SubsectionContainer = styled.div` 17 | margin-bottom: 30px; 18 | ` 19 | 20 | export const SubsectionTitle = styled.h3` 21 | color: ${({ theme }: { theme: ITheme }) => theme.palette.neutralSecondaryAlt}; 22 | margin-bottom: 0; 23 | ` 24 | 25 | export const IntroId = styled.div` 26 | padding-top: 20px; 27 | font-weight: bold; 28 | color: ${({ theme }: { theme: ITheme }) => theme.palette.neutralSecondary}; 29 | ` 30 | 31 | export const SubsectionDescription = styled.p`` 32 | 33 | export const EntityId = styled.p` 34 | font-weight: bold; 35 | color: ${({ theme }: { theme: ITheme }) => theme.palette.neutralSecondary}; 36 | ` 37 | -------------------------------------------------------------------------------- /javascript/webapp/src/utils/graphs.ts: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import type { Theme } from '@thematic/core' 6 | 7 | import type { GraphData } from '../types' 8 | 9 | /** 10 | * Takes a set of graph data and fills in the encoding block for each node based on desired viz options. 11 | * @param data 12 | * @param theme 13 | * @returns 14 | */ 15 | export function addGraphEncodings(data: GraphData, theme: Theme): GraphData { 16 | const colors = getCategoricalColors(data, theme) 17 | const warn = theme.application().error().hex() 18 | const normal = theme.application().background().hex() 19 | const bold = theme.rule().stroke().hex() 20 | return { 21 | nodes: data.nodes.map((node) => { 22 | return { 23 | ...node, 24 | encoding: { 25 | shape: node.relationship === 'target' ? 'diamond' : 'circle', 26 | fill: colors.get(node.type), 27 | stroke: 28 | node.relationship === 'target' 29 | ? bold 30 | : node.flag === 1 31 | ? warn 32 | : normal, 33 | strokeWidth: 34 | node.flag === 1 || node.relationship === 'target' ? 3 : 1, 35 | opacity: 1.0, 36 | size: 37 | node.relationship === 'target' 38 | ? 25 39 | : node.relationship === 'related' 40 | ? 20 41 | : 10, 42 | }, 43 | } 44 | }), 45 | // quick edge dedup. this may be done server-side in the future 46 | edges: Object.values( 47 | data.edges.reduce((acc, cur) => { 48 | const key = `${cur.source}-${cur.target}` 49 | acc[key] = cur 50 | return acc 51 | }, {}), 52 | ), 53 | } 54 | } 55 | 56 | // only create the colors map once, so all graphs match. 57 | // TODO: this could be in context instead of a module var for better control 58 | const colors = new Map() 59 | 60 | function getCategoricalColors(data: GraphData, theme: Theme) { 61 | const types = new Set(data?.nodes.map((node) => node.type)) 62 | const nominal = theme.scales().nominal(10) 63 | // capture a couple of variants for the entity type 64 | colors.set('entity', theme.process().fill().hex()) 65 | colors.set('EntityID', theme.process().fill().hex()) 66 | Array.from(types).forEach((category, index) => { 67 | if (!colors.has(category)) { 68 | colors.set(category, nominal(index).hex()) 69 | } 70 | }) 71 | return colors 72 | } 73 | -------------------------------------------------------------------------------- /javascript/webapp/src/vega/VegaChart.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { memo, useMemo } from 'react' 6 | import styled from 'styled-components' 7 | import type { Spec } from 'vega' 8 | 9 | import { 10 | parseJsonPathSpecMerged, 11 | VegaHost, 12 | } from '../components/VegaHost/index.js' 13 | 14 | // this is intentional so the chart can flex to any data type 15 | /* eslint-disable @typescript-eslint/no-explicit-any*/ 16 | export const VegaChart: React.FC<{ 17 | data?: any[] 18 | template: Spec 19 | width: number 20 | height: number 21 | description: string 22 | }> = memo(function VegaChart({ data, template, width, height, description }) { 23 | const spec = useOverlay(data, template) 24 | return ( 25 | 26 | 27 | {description} 28 | 29 | ) 30 | }) 31 | 32 | function useOverlay(data: any[] | undefined, template: Spec): Spec { 33 | return useMemo(() => { 34 | const pathspec = { 35 | "$.data[?(@.name == 'chart-data')].values": data, 36 | } 37 | return parseJsonPathSpecMerged(template, pathspec) 38 | }, [data, template]) 39 | } 40 | 41 | const Container = styled.div` 42 | display: inline-block; 43 | flex-direction: column; 44 | ` 45 | 46 | const Description = styled.div` 47 | font-size: 0.6em; 48 | padding-left: 5px; 49 | width: 95%; 50 | ` 51 | -------------------------------------------------------------------------------- /javascript/webapp/src/vega/VegaGraph.tsx: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | import { useThematic } from '@thematic/react' 6 | import merge from 'lodash-es/merge.js' 7 | import { memo, useMemo } from 'react' 8 | import styled from 'styled-components' 9 | import type { Spec } from 'vega' 10 | 11 | import { 12 | parseJsonPathSpecMerged, 13 | VegaHost, 14 | } from '../components/VegaHost/index.js' 15 | import type { GraphData, GraphNode } from '../types.js' 16 | 17 | export const VegaGraph: React.FC<{ 18 | data: GraphData 19 | template: Spec 20 | width: number 21 | height: number 22 | onMouseClick?: (datum?: GraphNode, axis?: { x: number; y: number }) => void 23 | }> = memo(function VegaGraph({ data, template, width, height, onMouseClick }) { 24 | const spec = useOverlay(data, template) 25 | const themed = useThemed(spec) 26 | return ( 27 | 28 | 34 | 35 | ) 36 | }) 37 | 38 | function useOverlay(data: GraphData, template: Spec): Spec { 39 | return useMemo(() => { 40 | const nodeData = data.nodes.map((x, index) => ({ ...x, index })) 41 | const linkData = data.edges.map((x) => { 42 | const source = nodeData.find((a) => a.id === x.source)?.index 43 | const target = nodeData.find((a) => a.id === x.target)?.index 44 | return { 45 | source, 46 | target, 47 | } 48 | }) 49 | 50 | const pathspec = { 51 | "$.data[?(@.name == 'node-data')].values": nodeData, 52 | "$.data[?(@.name == 'link-data')].values": linkData, 53 | } 54 | return parseJsonPathSpecMerged(template, pathspec) 55 | }, [data, template]) 56 | } 57 | 58 | // add any theme defaults needed that may be missing. in this case, we want to map standard thematic graph links to the path mark used on the graph. 59 | function useThemed(spec: Spec): Spec { 60 | const theme = useThematic() 61 | return useMemo(() => { 62 | const link = theme.link() 63 | return merge( 64 | { 65 | config: { 66 | path: { 67 | stroke: link.stroke().hex(), 68 | strokeWidth: link.strokeWidth(), 69 | strokeOpacity: link.strokeOpacity(), 70 | }, 71 | }, 72 | }, 73 | spec, 74 | ) 75 | }, [theme, spec]) 76 | } 77 | 78 | const Container = styled.div` 79 | display: flex; 80 | flex-direction: column; 81 | ` 82 | -------------------------------------------------------------------------------- /javascript/webapp/src/vega/bar-chart.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega/v5.json", 3 | "description": "A basic grouped bar chart example.", 4 | "padding": 5, 5 | "data": [ 6 | { 7 | "name": "chart-data" 8 | } 9 | ], 10 | "scales": [ 11 | { 12 | "name": "xscale", 13 | "type": "band", 14 | "domain": { 15 | "data": "chart-data", 16 | "field": "time" 17 | }, 18 | "range": "width", 19 | "padding": 0.2 20 | }, 21 | { 22 | "name": "yscale", 23 | "type": "linear", 24 | "domain": { 25 | "data": "chart-data", 26 | "field": "value" 27 | }, 28 | "range": "height", 29 | "round": true, 30 | "zero": true, 31 | "nice": true 32 | }, 33 | { 34 | "name": "color", 35 | "type": "ordinal", 36 | "domain": { 37 | "data": "chart-data", 38 | "field": "position" 39 | }, 40 | "range": "category" 41 | } 42 | ], 43 | "axes": [ 44 | { 45 | "orient": "left", 46 | "scale": "yscale", 47 | "tickSize": 0, 48 | "labelPadding": 4, 49 | "zindex": 1 50 | }, 51 | { 52 | "orient": "bottom", 53 | "scale": "xscale", 54 | "tickSize": 4, 55 | "labelAngle": -90, 56 | "labelPadding": 20, 57 | "labelBaseline": "middle" 58 | } 59 | ], 60 | "marks": [ 61 | { 62 | "type": "group", 63 | "from": { 64 | "facet": { 65 | "data": "chart-data", 66 | "name": "facet", 67 | "groupby": "time" 68 | } 69 | }, 70 | "encode": { 71 | "enter": { 72 | "x": { 73 | "scale": "xscale", 74 | "field": "time" 75 | } 76 | } 77 | }, 78 | "signals": [ 79 | { 80 | "name": "width", 81 | "update": "bandwidth('xscale')" 82 | } 83 | ], 84 | "scales": [ 85 | { 86 | "name": "pos", 87 | "type": "band", 88 | "range": "width", 89 | "domain": { 90 | "data": "facet", 91 | "field": "position" 92 | } 93 | } 94 | ], 95 | "marks": [ 96 | { 97 | "name": "bars", 98 | "from": { 99 | "data": "facet" 100 | }, 101 | "type": "rect", 102 | "encode": { 103 | "enter": { 104 | "x": { 105 | "scale": "pos", 106 | "field": "position" 107 | }, 108 | "width": { 109 | "scale": "pos", 110 | "band": 1 111 | }, 112 | "y": { 113 | "scale": "yscale", 114 | "field": "value" 115 | }, 116 | "y2": { 117 | "scale": "yscale", 118 | "value": 0 119 | }, 120 | "fill": { 121 | "scale": "color", 122 | "field": "position" 123 | } 124 | } 125 | } 126 | } 127 | ] 128 | } 129 | ] 130 | } 131 | -------------------------------------------------------------------------------- /javascript/webapp/src/vega/donut-chart.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://vega.github.io/schema/vega/v5.json", 3 | "description": "A basic donut chart example.", 4 | "autosize": "none", 5 | "signals": [ 6 | { 7 | "name": "startAngle", 8 | "value": 0 9 | }, 10 | { 11 | "name": "endAngle", 12 | "value": 6.29 13 | }, 14 | { 15 | "name": "padAngle", 16 | "value": 0 17 | }, 18 | { 19 | "name": "innerRadius", 20 | "value": 60 21 | }, 22 | { 23 | "name": "cornerRadius", 24 | "value": 0 25 | }, 26 | { 27 | "name": "sort", 28 | "value": false 29 | } 30 | ], 31 | "data": [ 32 | { 33 | "name": "chart-data", 34 | "transform": [ 35 | { 36 | "type": "pie", 37 | "field": "field", 38 | "startAngle": { "signal": "startAngle" }, 39 | "endAngle": { "signal": "endAngle" }, 40 | "sort": { "signal": "sort" } 41 | } 42 | ] 43 | } 44 | ], 45 | 46 | "scales": [ 47 | { 48 | "name": "color", 49 | "type": "ordinal", 50 | "domain": { "data": "chart-data", "field": "id" }, 51 | "range": "category" 52 | } 53 | ], 54 | 55 | "legends": [ 56 | { 57 | "orient": "none", 58 | "fill": "color", 59 | "direction": "horizontal", 60 | "strokeColor": "none", 61 | "legendY": 240, 62 | "legendX": 8 63 | } 64 | ], 65 | 66 | "marks": [ 67 | { 68 | "name": "arcs", 69 | "type": "arc", 70 | "from": { "data": "chart-data" }, 71 | "encode": { 72 | "enter": { 73 | "fill": { "scale": "color", "field": "id" }, 74 | "x": { "signal": "width / 2" }, 75 | "y": { "signal": "height / 2" } 76 | }, 77 | "update": { 78 | "startAngle": { "field": "startAngle" }, 79 | "endAngle": { "field": "endAngle" }, 80 | "padAngle": { "signal": "padAngle" }, 81 | "innerRadius": { "signal": "innerRadius" }, 82 | "outerRadius": { "signal": "(width - 30) / 2" }, 83 | "cornerRadius": { "signal": "cornerRadius" } 84 | } 85 | } 86 | }, 87 | { 88 | "type": "text", 89 | "from": { "data": "arcs" }, 90 | "encode": { 91 | "enter": { 92 | "x": { "field": "x" }, 93 | "y": { "field": "y" }, 94 | "radius": { "value": 80 }, 95 | "theta": { 96 | "signal": "((datum.endAngle - datum.startAngle) / 2) + datum.startAngle" 97 | }, 98 | "align": { "value": "center" }, 99 | "baseline": { "value": "middle" }, 100 | "text": { "field": "datum.field" }, 101 | "fontWeight": { "value": "bold" } 102 | } 103 | } 104 | } 105 | ] 106 | } 107 | -------------------------------------------------------------------------------- /javascript/webapp/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "ESNext", 4 | "moduleResolution": "Node", 5 | "baseUrl": ".", 6 | "lib": ["ESNext", "DOM", "DOM.Iterable", "ES2020.BigInt"], 7 | "outDir": "lib", 8 | "jsx": "react-jsx", 9 | "resolveJsonModule": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "allowSyntheticDefaultImports": true, 12 | "strict": true, 13 | "declaration": true, 14 | "incremental": true, 15 | "noImplicitAny": false, 16 | "target": "ESNext", 17 | "allowJs": false, 18 | "skipLibCheck": true, 19 | "esModuleInterop": false, 20 | "isolatedModules": true, 21 | "noEmit": true 22 | }, 23 | "include": ["src/**/*"] 24 | } 25 | -------------------------------------------------------------------------------- /javascript/webapp/webpack.config.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) Microsoft. All rights reserved. 3 | * Licensed under the MIT license. See LICENSE file in the project. 4 | */ 5 | const { configure } = require('@essex/webpack-config') 6 | const HtmlWebpackPlugin = require('html-webpack-plugin') 7 | 8 | const configuration = configure({ 9 | aliases: () => ({ 10 | 'react-router-dom': require.resolve('react-router-dom'), 11 | }), 12 | environment: (env, mode) => { 13 | return { 14 | APP_MOUNT_PATH: process.env.APP_MOUNT_PATH ?? '/', 15 | } 16 | }, 17 | plugins: (env, mode) => [ 18 | new HtmlWebpackPlugin({ 19 | baseUrl: process.env.BASE_URL ?? '/', 20 | template: './config/index.hbs', 21 | title: 'Transparency Engine', 22 | appMountIds: ['root', 'cookie-banner'], 23 | devServer: mode === 'development', 24 | files: { 25 | js: ['https://wcpstatic.microsoft.com/mscc/lib/v2/wcp-consent.js'], 26 | }, 27 | }), 28 | ], 29 | devServer: () => ({ 30 | port: 3000, 31 | allowedHosts: 'all', 32 | }), 33 | }) 34 | 35 | // remove old html-plugin 36 | configuration.plugins.shift() 37 | delete configuration.baseUrl 38 | module.exports = configuration 39 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@transparency-engine/project", 3 | "private": true, 4 | "scripts": { 5 | "_fix_packages": "turbo run fix", 6 | "rome_fix": "rome check . --apply-suggested", 7 | "clean": "turbo run clean --parallel", 8 | "build": "turbo run build", 9 | "bundle": "turbo run bundle", 10 | "fix": "run-s _fix_packages rome_fix format", 11 | "format": "rome format . --write", 12 | "start": "SB_QUIET=true turbo run start --parallel", 13 | "update_sdks": "yarn dlx @yarnpkg/sdks vscode" 14 | }, 15 | "devDependencies": { 16 | "@essex/scripts": "^22.2.0", 17 | "@types/jest": "^29.4.0", 18 | "@types/node": "^18.14.4", 19 | "@types/semver": "^7.3.13", 20 | "eslint": "^8.33.0", 21 | "npm-run-all": "^4.1.5", 22 | "rome": "^11.0.0", 23 | "semver": "^7.3.8", 24 | "turbo": "1.8.3", 25 | "typescript": "^4.9.5" 26 | }, 27 | "workspaces": [ 28 | "javascript/*" 29 | ], 30 | "packageManager": "yarn@3.4.1" 31 | } 32 | -------------------------------------------------------------------------------- /powerbi/TransparencyEngine.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/powerbi/TransparencyEngine.pbit -------------------------------------------------------------------------------- /python/.gitattributes: -------------------------------------------------------------------------------- 1 | *.js text eol=lf 2 | *.jsx text eol=lf 3 | *.mjs text eol=lf 4 | *.cjs text eol=lf 5 | *.ts text eol=lf 6 | *.tsx text eol=lf 7 | *.mts text eol=lf 8 | *.cts text eol=lf 9 | *.json text eol=lf 10 | *.yml text eol=lf 11 | *.md text eol=lf 12 | *.sh text eol=lf -------------------------------------------------------------------------------- /python/api-backend/.dockerignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | *.pyo 4 | *.pyd 5 | .Python 6 | env 7 | pip-log.txt 8 | pip-delete-this-directory.txt 9 | .tox 10 | .coverage 11 | .coverage.* 12 | .cache 13 | nosetests.xml 14 | coverage.xml 15 | *.cover 16 | *.log 17 | .git 18 | .mypy_cache 19 | .pytest_cache 20 | .hypothesis 21 | .venv -------------------------------------------------------------------------------- /python/api-backend/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 127 3 | max-complexity = 10 -------------------------------------------------------------------------------- /python/api-backend/.gitattributes: -------------------------------------------------------------------------------- 1 | *.js text eol=lf 2 | *.jsx text eol=lf 3 | *.mjs text eol=lf 4 | *.cjs text eol=lf 5 | *.ts text eol=lf 6 | *.tsx text eol=lf 7 | *.mts text eol=lf 8 | *.cts text eol=lf 9 | *.json text eol=lf 10 | *.yml text eol=lf 11 | *.md text eol=lf 12 | *.sh text eol=lf -------------------------------------------------------------------------------- /python/api-backend/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://eng.ms/docs/more/containers-secure-supply-chain/approved-images 2 | FROM mcr.microsoft.com/oryx/python:3.9 3 | 4 | # Keeps Python from generating .pyc files in the container 5 | ENV PYTHONDONTWRITEBYTECODE 1 6 | # Turns off buffering for easier container logging 7 | ENV PYTHONUNBUFFERED 1 8 | 9 | # Install necessary dependencies to compile 10 | RUN apt-get update -y \ 11 | && apt-get install -y gcc \ 12 | && apt-get install -y --no-install-recommends curl \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | # Install Poetry 16 | RUN curl -sSL https://install.python-poetry.org | python - --version 1.3.2 17 | ENV PATH="${PATH}:/root/.local/bin" 18 | 19 | WORKDIR /api-backend 20 | 21 | # Copy backend assets 22 | # This is done separately so the base layers won't change that often 23 | # to speed up image pushing 24 | COPY ./poetry.lock . 25 | COPY ./pyproject.toml . 26 | COPY ./scripts ./scripts 27 | 28 | # Install python denpendencies 29 | RUN chmod +x ./scripts/install_python_deps.sh 30 | RUN ./scripts/install_python_deps.sh 31 | 32 | # Copy remaining files 33 | COPY . . 34 | 35 | # Start backend either in API or worker mode, depending on the WORKER env var 36 | RUN chmod +x ./scripts/start.sh 37 | ENTRYPOINT [ "./scripts/start.sh" ] -------------------------------------------------------------------------------- /python/api-backend/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/README.md -------------------------------------------------------------------------------- /python/api-backend/api_backend/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.0" 2 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/api_main.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import logging 7 | import os 8 | 9 | from fastapi import FastAPI 10 | from fastapi.middleware.cors import CORSMiddleware 11 | 12 | from api_backend.entities.api.router import entities_router 13 | from api_backend.report.api.router import report_router 14 | from api_backend.search.api.router import search_router 15 | 16 | app = FastAPI() 17 | 18 | enable_cors = os.getenv("ENABLE_CORS", "").strip() 19 | 20 | if enable_cors: 21 | origins = [c.strip() for c in enable_cors.split(",")] 22 | logging.info(f"Enabling CORS for {origins}") 23 | app.add_middleware( 24 | CORSMiddleware, 25 | allow_origins=origins, 26 | allow_credentials=True, 27 | allow_methods=["*"], 28 | allow_headers=["*"], 29 | ) 30 | 31 | app.include_router(search_router, prefix="/api/search") 32 | app.include_router(report_router, prefix="/api/report") 33 | app.include_router(entities_router, prefix="/api/entities") 34 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/entities/api/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.0.1" 2 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/entities/api/get_graph.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import logging 7 | from typing import Optional 8 | 9 | from fastapi import status 10 | from fastapi.responses import JSONResponse 11 | 12 | from api_backend.entities.api.get_graph_from_db import get_graph_from_db, get_graph_schema_from_db 13 | 14 | error_text = "No source parameter provided. Please provide valid source." 15 | 16 | 17 | async def get_graph(source: str, target: Optional[str] = None): 18 | """ 19 | Asynchronously retrieves a graph from the database using the given source and target nodes, and returns it. 20 | 21 | Args: 22 | source: A string representing the source node for the graph. 23 | target: An optional string representing the target node for the graph. 24 | 25 | Returns: 26 | The graph retrieved from the database. 27 | """ 28 | logging.info(f"Calling get_graph with params: {source}, {target}") 29 | 30 | if not source: 31 | return JSONResponse(content={"error": error_text}, status_code=status.HTTP_400_BAD_REQUEST) 32 | 33 | try: 34 | results = get_graph_from_db(source, target) 35 | return JSONResponse(content=results, status_code=status.HTTP_200_OK) 36 | except Exception as e: 37 | logging.error(e, exc_info=True) 38 | return JSONResponse( 39 | content={"error": f"Internal Server Error - {str(e)}"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR 40 | ) 41 | 42 | 43 | async def get_graph_schema(source: str, target: Optional[str] = None): 44 | """ 45 | Asynchronously retrieves a graph from the database using the given source and target nodes, and returns it. 46 | 47 | Args: 48 | source: A string representing the source node for the graph. 49 | target: An optional string representing the target node for the graph. 50 | 51 | Returns: 52 | The graph retrieved from the database. 53 | """ 54 | logging.info(f"Calling get_graph_schema with params: {source}, {target}") 55 | 56 | if not source: 57 | return JSONResponse(content={"error": error_text}, status_code=status.HTTP_400_BAD_REQUEST) 58 | 59 | try: 60 | results = get_graph_schema_from_db(source, target) 61 | 62 | if not results: 63 | return JSONResponse(content={"error": "No graph schema found."}, status_code=status.HTTP_404_NOT_FOUND) 64 | return JSONResponse(content=results, status_code=status.HTTP_200_OK) 65 | except Exception as e: 66 | logging.error(e, exc_info=True) 67 | return JSONResponse( 68 | content={"error": f"Internal Server Error - {str(e)}"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR 69 | ) 70 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/entities/api/router.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from fastapi import APIRouter, Query 7 | 8 | from api_backend.entities.api.get_graph import get_graph, get_graph_schema 9 | 10 | entities_router = APIRouter() 11 | 12 | 13 | @entities_router.get("/health") 14 | async def main(): 15 | return {"message": "entities api is healthy"} 16 | 17 | 18 | @entities_router.get("/graph/{source}") 19 | async def graph(source, target=Query(None)): 20 | return await get_graph(source, target) 21 | 22 | @entities_router.get("/graph/schema/{source}") 23 | async def graph_schema(source, target=Query(None)): 24 | return await get_graph_schema(source, target) 25 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/entities/constants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/entities/constants/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/entities/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/entities/util/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/entities/util/style_graph.py: -------------------------------------------------------------------------------- 1 | def style_graph(data, template): 2 | encoded_data = add_graph_encodings(data) 3 | node_data = [{**x, "index": index} for index, x in enumerate(encoded_data["nodes"])] 4 | 5 | link_data = [ 6 | { 7 | "source": next((i for i, a in enumerate(node_data) if a["id"] == x["source"]), None), 8 | "target": next((i for i, a in enumerate(node_data) if a["id"] == x["target"]), None), 9 | } 10 | for x in encoded_data["edges"] 11 | ] 12 | 13 | data_spec = { 14 | "node-data": node_data, 15 | "link-data": link_data, 16 | } 17 | return merge_data_to_template(template, data_spec) 18 | 19 | 20 | def merge_data_to_template(template, data_spec): 21 | node_data = data_spec["node-data"] 22 | link_data = data_spec["link-data"] 23 | data = template["data"] 24 | 25 | for d in data: 26 | if d["name"] == "node-data": 27 | d["values"] = node_data 28 | elif d["name"] == "link-data": 29 | d["values"] = link_data 30 | template["data"] = data 31 | return template 32 | 33 | 34 | def add_graph_encodings(data): 35 | colors = get_categorical_colors(data) 36 | warn = "#d13438" 37 | normal = "#fdfeff" 38 | bold = "#31302e" 39 | 40 | return { 41 | "nodes": [ 42 | { 43 | **node, 44 | "encoding": { 45 | "shape": "diamond" if node["relationship"] == "target" else "circle", 46 | "fill": colors.get(node["type"]), 47 | "stroke": bold if node["relationship"] == "target" else warn if node["flag"] == 1 else normal, 48 | "strokeWidth": 3 if node["flag"] == 1 or node["relationship"] == "target" else 1, 49 | "opacity": 1.0, 50 | "size": 25 if node["relationship"] == "target" else 20 if node["relationship"] == "related" else 10, 51 | }, 52 | } 53 | for node in data["nodes"] 54 | ], 55 | "edges": list({f"{edge['source']}-{edge['target']}": edge for edge in data["edges"]}.values()), 56 | } 57 | 58 | 59 | def nominal(index): 60 | colors = [ 61 | "#32bb51", 62 | "#f87f7f", 63 | "#37b4b4", 64 | "#baa332", 65 | "#f66dec", 66 | "#39b0d2", 67 | "#90af32", 68 | "#f778b7", 69 | "#35b796", 70 | "#e29132", 71 | ] 72 | return colors[index % 10] 73 | 74 | 75 | def get_categorical_colors(data): 76 | types = set(node["type"] for node in data.get("nodes", [])) 77 | colors = {"entity": "#787672", "EntityID": "#787672"} 78 | for index, category in enumerate(types): 79 | if category not in colors: 80 | colors[category] = nominal(index) 81 | return colors 82 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/model/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/model/graph_model.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from dataclasses import dataclass 7 | from typing import List 8 | 9 | 10 | @dataclass 11 | class Node: 12 | id: int 13 | name: str 14 | color: str 15 | labelColor: str 16 | nodeSize: int 17 | 18 | 19 | @dataclass 20 | class Edge: 21 | source: int 22 | target: int 23 | widthWeight: int 24 | targetColor: int 25 | 26 | 27 | @dataclass 28 | class Graph: 29 | nodes: List[Node] 30 | edges: List[Edge] 31 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/report/api/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/get_report.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import logging 7 | 8 | from fastapi import status 9 | from fastapi.responses import JSONResponse 10 | 11 | from api_backend.report.api.get_report_from_db import get_report_from_db 12 | 13 | 14 | async def get_report(id: str): 15 | """ 16 | Asynchronously retrieves a report with the specified entity id. 17 | 18 | The function queries a database to retrieve the data to build a report with the specified id and returns it. 19 | 20 | Args: 21 | id: A string representing the entity id. 22 | 23 | Returns: 24 | A report object. 25 | """ 26 | 27 | logging.info(f"Calling get_report with param: {id}") 28 | 29 | if not id: 30 | return JSONResponse( 31 | content={"error": "No id parameter provided. Please provide valid id."}, 32 | status_code=status.HTTP_400_BAD_REQUEST, 33 | ) 34 | 35 | try: 36 | results = get_report_from_db(id) 37 | return JSONResponse(content=results, status_code=status.HTTP_200_OK) 38 | except Exception as e: 39 | logging.error(e, exc_info=True) 40 | return JSONResponse( 41 | content={"error": f"Internal Server Error - {str(e)}"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR 42 | ) 43 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/get_report_from_db.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import os 7 | 8 | from sqlalchemy import MetaData, Table, select 9 | 10 | from api_backend.report.builders.report import build_report 11 | from api_backend.report.constants.attributes import async_activity, related, sync_activity 12 | from api_backend.report.util.parsers import parse_activity, parse_entity_details, parse_raw_report 13 | from api_backend.util.db_engine import get_engine 14 | 15 | 16 | def get_report_from_db(id=""): 17 | report_table_name = os.getenv("REPORT_TABLE", "") 18 | entity_table_name = os.getenv("ENTITY_TABLE", "") 19 | activity_table_name_name = os.getenv("ACTIVITY_TABLE", "") 20 | engine = get_engine() 21 | 22 | metadata = MetaData() 23 | report_table = Table(report_table_name, metadata, autoload_with=engine) 24 | entity_table = Table(entity_table_name, metadata, autoload_with=engine) 25 | activity_table_name = Table(activity_table_name_name, metadata, autoload_with=engine) 26 | 27 | query_report = select(report_table).where(report_table.c.EntityID == id) 28 | 29 | query_entity = select(entity_table).where(entity_table.c.EntityID == id) 30 | 31 | query_filtered_attributes = lambda id_list: select(activity_table_name).where( 32 | activity_table_name.c.EntityID.in_(id_list) 33 | ) 34 | 35 | conn = engine.connect() 36 | 37 | result_proxy = conn.execute(query_report) 38 | report_results = result_proxy.fetchall() 39 | column_names = result_proxy.keys() 40 | entity_results = conn.execute(query_entity).fetchall() 41 | entity_details = parse_entity_details(entity_results) 42 | raw_report = parse_raw_report(column_names, report_results) 43 | activity = {} 44 | 45 | if raw_report.get(sync_activity, None): 46 | entity_ids = [id] + [item[related] for item in raw_report[sync_activity]] 47 | activity_results = conn.execute(query_filtered_attributes(entity_ids)).fetchall() 48 | activity = {sync_activity: parse_activity(activity_results, id)} 49 | if raw_report.get(async_activity, None): 50 | entity_ids = [id] + [item[related] for item in raw_report[async_activity]] 51 | activity_results = conn.execute(query_filtered_attributes(entity_ids)).fetchall() 52 | activity = {async_activity: parse_activity(activity_results, id)} 53 | 54 | conn.close() 55 | return build_report(id, raw_report, entity_details, activity) 56 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/get_report_score.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import logging 7 | 8 | from fastapi import status 9 | from fastapi.responses import JSONResponse 10 | 11 | from api_backend.report.api.get_report_score_from_db import get_report_score_from_db 12 | 13 | 14 | async def get_report_score(id: str): 15 | """ 16 | Asynchronously retrieves the report score for the specified entity id. 17 | 18 | The function queries a database to retrieve the network score data. 19 | 20 | Args: 21 | id: A string representing the entity id. 22 | 23 | Returns: 24 | A network score object. 25 | """ 26 | 27 | logging.info(f"Calling get_report score with param: {id}") 28 | 29 | if not id: 30 | return JSONResponse( 31 | content={"error": "No id parameter provided. Please provide valid id."}, 32 | status_code=status.HTTP_400_BAD_REQUEST, 33 | ) 34 | 35 | try: 36 | result = get_report_score_from_db(id) 37 | if result is None: 38 | return JSONResponse( 39 | content={"error": f"Not found report score with entity id: {id}"}, 40 | status_code=status.HTTP_404_NOT_FOUND, 41 | ) 42 | return JSONResponse(content=result, status_code=status.HTTP_200_OK) 43 | except Exception as e: 44 | logging.error(e, exc_info=True) 45 | return JSONResponse( 46 | content={"error": f"Internal Server Error - {str(e)}"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR 47 | ) 48 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/get_report_score_from_db.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from sqlalchemy import MetaData, Table, select 4 | 5 | from api_backend.util.db_engine import get_engine 6 | 7 | 8 | def get_report_score_from_db(entity_id): 9 | table_name = os.getenv("NETWORK_SCORING_TABLE", "") 10 | engine = get_engine() 11 | 12 | metadata = MetaData() 13 | table = Table(table_name, metadata, autoload_with=engine) 14 | 15 | query = select(table).where(table.c.EntityID == entity_id) 16 | 17 | conn = engine.connect() 18 | result_proxy = conn.execute(query) 19 | result = result_proxy.fetchone() 20 | column_names = [col for col in result_proxy.keys()] 21 | conn.close() 22 | 23 | if "final_network_score_max_scaled" in column_names: 24 | i = column_names.index("final_network_score_max_scaled") 25 | score = result[i] 26 | return {"score": score} 27 | return None 28 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/get_report_url.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import logging 7 | 8 | from fastapi import status 9 | from fastapi.responses import JSONResponse 10 | 11 | from api_backend.report.api.get_report_url_from_db import get_report_url_from_db 12 | 13 | 14 | async def get_report_url(entity_id): 15 | """ 16 | Asynchronously retrieves the report url with the specified entity id. 17 | 18 | The function queries a database to retrieve the url of a report and returns it. 19 | Args: 20 | id: A string representing the entity id. 21 | 22 | Returns: 23 | An object with EntityID and ReportLink data 24 | """ 25 | 26 | logging.info(f"Calling get_report_url with param: {id}") 27 | 28 | if not id: 29 | return JSONResponse( 30 | content={"error": "No id parameter provided. Please provide valid id."}, 31 | status_code=status.HTTP_400_BAD_REQUEST, 32 | ) 33 | try: 34 | result = get_report_url_from_db(entity_id) 35 | if result is None: 36 | return JSONResponse( 37 | content={"error": f"Not found report url with entity id: {entity_id}"}, 38 | status_code=status.HTTP_404_NOT_FOUND, 39 | ) 40 | return JSONResponse(content=result, status_code=status.HTTP_200_OK) 41 | except Exception as e: 42 | logging.error(e, exc_info=True) 43 | return JSONResponse( 44 | content={"error": f"Internal Server Error - {str(e)}"}, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR 45 | ) 46 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/get_report_url_from_db.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from sqlalchemy import MetaData, Table, select 4 | 5 | from api_backend.util.db_engine import get_engine 6 | 7 | 8 | def get_report_url_from_db(entity_id): 9 | table_name = os.getenv("REPORT_URL_TABLE", "") 10 | engine = get_engine() 11 | 12 | metadata = MetaData() 13 | table = Table(table_name, metadata, autoload_with=engine) 14 | 15 | query = select(table).where(table.c.EntityID == entity_id) 16 | 17 | conn = engine.connect() 18 | result = conn.execute(query).fetchone() 19 | conn.close() 20 | 21 | response = None 22 | 23 | if result is not None: 24 | response = { 25 | "EntityID": result[0], 26 | "ReportLink": result[1], 27 | } 28 | 29 | return response 30 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/api/router.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from fastapi import APIRouter 7 | 8 | from api_backend.report.api.get_report import get_report 9 | from api_backend.report.api.get_report_score import get_report_score 10 | from api_backend.report.api.get_report_url import get_report_url 11 | 12 | report_router = APIRouter() 13 | 14 | 15 | @report_router.get("/health") 16 | async def main(): 17 | return {"message": "report api is healthy"} 18 | 19 | 20 | @report_router.get("/score/{id}") 21 | async def report_score(id): 22 | return await get_report_score(id) 23 | 24 | 25 | @report_router.get("/url/{id}") 26 | async def report_url(id): 27 | return await get_report_url(id) 28 | 29 | 30 | @report_router.get("/{id}") 31 | async def report(id): 32 | return await get_report(id) 33 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/report/builders/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/builders/activity_analysis.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | 7 | from api_backend.report.constants.attributes import link_summary, related 8 | from api_backend.report.util.util import build_activity_values_data 9 | 10 | 11 | def get_attribute_counts(raw_section): 12 | data = {} 13 | 14 | if raw_section is None: 15 | return [] 16 | 17 | for item in raw_section: 18 | entity_id = item[related] 19 | for key in item: 20 | if key == related or key == link_summary or "average" in key or key == "flag_count": 21 | continue 22 | new_key = key.split("_")[0] if "score" in key else key.replace("_", " ") 23 | value = data.get(new_key, []) 24 | new_value = { 25 | "entity_id": entity_id, 26 | "value": round(item[key] * 100, 2), 27 | } 28 | value.append(new_value) 29 | data[new_key] = value 30 | return [{"key": key, "value": value} for key, value in data.items()] 31 | 32 | 33 | def get_attribute_values(raw_section): 34 | return build_activity_values_data(raw_section) 35 | 36 | 37 | def get_attribute_charts(activity): 38 | target_entity_activity = [] 39 | related_entity_activity = [] 40 | if activity: 41 | for key in activity: 42 | values = activity[key].get("values", []) 43 | if activity[key]["is_target"]: 44 | target_entity_activity = values 45 | else: 46 | related_entity_activity.append({"entity_id": key, "activity": values}) 47 | 48 | return [ 49 | { 50 | "key": "target entity activity", 51 | "value": target_entity_activity, 52 | }, 53 | { 54 | "key": "related entity activity", 55 | "value": related_entity_activity, 56 | }, 57 | ] 58 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/builders/activity_summary.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from api_backend.report.util.util import build_count_percent_rank_data 7 | 8 | 9 | def get_attribute_counts(raw_section): 10 | return build_count_percent_rank_data(raw_section) 11 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/builders/entity_overview.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from api_backend.report.util.util import key_to_title 7 | 8 | 9 | def get_attribute_counts(raw_section): 10 | data = [] 11 | for key in raw_section: 12 | if isinstance(raw_section[key], int): 13 | data.append([key_to_title(key), raw_section[key]]) 14 | return data 15 | 16 | 17 | def get_attribute_values(entity_details): 18 | data = [] 19 | for key in entity_details: 20 | if key.lower() == "entityweight": 21 | continue 22 | data.append([key_to_title(key), entity_details[key]]) 23 | return data 24 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/builders/own_review_flag_analysis.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from api_backend.report.util.util import get_review_flags 7 | 8 | 9 | def get_attribute_values(raw_section): 10 | return get_review_flags(raw_section) 11 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/builders/related_entities.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | 7 | from api_backend.report.util.util import build_related_entities_data 8 | 9 | 10 | def get_attribute_values(raw_section): 11 | return build_related_entities_data(raw_section) 12 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/builders/review_flag_summary.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from api_backend.report.util.util import build_count_percent_rank_data 7 | 8 | 9 | def get_attribute_counts(raw_section): 10 | return build_count_percent_rank_data(raw_section) 11 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/constants/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/report/constants/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/constants/attributes.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | # Sections 7 | attribute_summary = "attributesummary" 8 | flag_summary = "flagsummary" 9 | activity_summary = "activitysummary" 10 | own_flags = "ownflags" 11 | direct_flags = "directflags" 12 | indirect_flags = "indirectflags" 13 | sync_activity = "syncactivity" 14 | async_activity = "asyncactivity" 15 | 16 | # Section attributes 17 | direct_links = "direct_links" 18 | evidence = "evidence" 19 | flag = "flag" 20 | indirect_link_count = "indirect_link_count" 21 | link_summary = "link_summary" 22 | related = "related" 23 | related_flag_details = "related_flag_details" 24 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/constants/report_data_mapping.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import api_backend.report.builders.activity_analysis as activity_analysis 7 | import api_backend.report.builders.activity_summary as activity_summary_builder 8 | import api_backend.report.builders.entity_overview as entity_overview 9 | import api_backend.report.builders.own_review_flag_analysis as own_review_flag_analysis 10 | import api_backend.report.builders.related_entities as related_entities 11 | import api_backend.report.builders.review_flag_summary as review_flag_summary 12 | from api_backend.report.constants.attributes import ( 13 | activity_summary, 14 | async_activity, 15 | attribute_summary, 16 | direct_flags, 17 | flag_summary, 18 | indirect_flags, 19 | own_flags, 20 | sync_activity, 21 | ) 22 | 23 | report_data_mapping = { 24 | attribute_summary: { 25 | "attribute_counts": {"method": entity_overview.get_attribute_counts, "args": "raw_section"}, 26 | "attribute_values": {"method": entity_overview.get_attribute_values, "args": "entity_details"}, 27 | }, 28 | flag_summary: {"attribute_counts": {"method": review_flag_summary.get_attribute_counts, "args": "raw_section"}}, 29 | activity_summary: { 30 | "attribute_counts": {"method": activity_summary_builder.get_attribute_counts, "args": "raw_section"} 31 | }, 32 | own_flags: {"attribute_values": {"method": own_review_flag_analysis.get_attribute_values, "args": "raw_section"}}, 33 | direct_flags: {"attribute_values": {"method": related_entities.get_attribute_values, "args": "raw_section"}}, 34 | indirect_flags: {"attribute_values": {"method": related_entities.get_attribute_values, "args": "raw_section"}}, 35 | sync_activity: { 36 | "attribute_values": {"method": activity_analysis.get_attribute_values, "args": "raw_section"}, 37 | "attribute_charts": { 38 | "method": activity_analysis.get_attribute_charts, 39 | "args": "activity", 40 | }, 41 | }, 42 | async_activity: { 43 | "attribute_values": {"method": activity_analysis.get_attribute_values, "args": "raw_section"}, 44 | "attribute_charts": { 45 | "method": activity_analysis.get_attribute_charts, 46 | "args": "activity", 47 | }, 48 | }, 49 | } 50 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/report/util/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/report/util/parsers.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import json 7 | 8 | from api_backend.report.util.util import title_to_key 9 | 10 | 11 | def parse_entity_details(entity_results): 12 | entity_details = {} 13 | for row in entity_results: 14 | attribute = row[0] 15 | value = row[2] 16 | values = entity_details.get(attribute, []) 17 | if value not in values: 18 | values.append(value) 19 | entity_details[attribute] = values 20 | return entity_details 21 | 22 | 23 | def parse_raw_report(column_names, report_results): 24 | raw_report = {} 25 | for row in report_results: 26 | for key, value in zip(column_names, row): 27 | formatted_key = title_to_key(key) 28 | if value is not None: 29 | raw_report[formatted_key] = parse_value(value) 30 | else: 31 | raw_report[formatted_key] = value 32 | return raw_report 33 | 34 | 35 | def parse_activity(activity_results, target_id): 36 | activity = {} 37 | for item in activity_results: 38 | entity = item[0] 39 | attribute = item[1] 40 | value = item[2] 41 | time = item[3] 42 | if entity not in activity: 43 | activity[entity] = {} 44 | values = activity[entity].get("values", []) 45 | values.append({"time": time, "attribute": attribute, "value": value}) 46 | activity[entity]["values"] = values 47 | activity[entity]["is_target"] = entity == target_id 48 | return activity 49 | 50 | 51 | def parse_value(value): 52 | if isinstance(value, str): 53 | try: 54 | value = json.loads(value) 55 | except: 56 | pass 57 | 58 | if isinstance(value, list): 59 | # Recursively process each element of the list 60 | return [parse_value(item) for item in value] 61 | elif isinstance(value, dict): 62 | # Recursively process each value of the dictionary 63 | return {title_to_key(key): parse_value(value) for key, value in value.items()} 64 | else: 65 | # Try convert the result to a JSON object, otherwise return the value 66 | try: 67 | return json.loads(value) 68 | except: 69 | return value 70 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/search/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/search/api/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/search/api/router.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from fastapi import APIRouter 7 | 8 | search_router = APIRouter() 9 | 10 | 11 | @search_router.get("/") 12 | def main(): 13 | return {"message": "search api is healthy"} 14 | -------------------------------------------------------------------------------- /python/api-backend/api_backend/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/api_backend/util/__init__.py -------------------------------------------------------------------------------- /python/api-backend/api_backend/util/db_engine.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | import os 6 | 7 | from sqlalchemy import create_engine 8 | from sqlalchemy.engine import Engine 9 | 10 | 11 | def get_engine() -> Engine: 12 | """ 13 | Get a SQLAlchemy engine for the database. 14 | 15 | Returns: 16 | A SQLAlchemy Engine object for connecting to the database. 17 | """ 18 | 19 | server = os.getenv("SQL_ENDPOINT", "") 20 | database = os.getenv("DB_NAME", "") 21 | username = os.getenv("SQL_USERNAME", "") 22 | password = os.getenv("SQL_PASSWORD", "") 23 | 24 | driver = "{ODBC Driver 17 for SQL Server}" 25 | 26 | odbc_connect = f"DRIVER={driver};SERVER=tcp:{server};PORT=1433;DATABASE={database};UID={username};PWD={password};Authentication=ActiveDirectoryServicePrincipal;" 27 | 28 | url = "mssql+pyodbc:///?odbc_connect={}".format(odbc_connect) 29 | 30 | engine = create_engine(url, echo=False) 31 | 32 | return engine 33 | -------------------------------------------------------------------------------- /python/api-backend/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "api-backend" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Alonso Guevara "] 6 | license = "MIT" 7 | readme = "README.md" 8 | packages = [{include = "api_backend"}] 9 | 10 | [tool.poetry.dependencies] 11 | python = ">=3.8,<3.10" 12 | fastapi = "^0.89.1" 13 | numpy = "^1.24.1" 14 | pandas = "^1.5.2" 15 | uvicorn = "^0.20.0" 16 | python-dotenv = "^0.21.1" 17 | sqlalchemy = "^2.0.4" 18 | pandas-stubs = "^1.5.3.230214" 19 | pyodbc = "^4.0.35" 20 | pydantic = "^1.10.5" 21 | 22 | [tool.poetry.group.utility.dependencies] 23 | gunicorn = "^20.1.0" 24 | pydantic = "^1.10.4" 25 | 26 | 27 | [tool.poetry.group.graph.dependencies] 28 | celery = "^5.2.7" 29 | networkx = "^3.0" 30 | scikit-learn = "^1.2.0" 31 | redis = {extras = ["hiredis"], version = "^4.4.1"} 32 | python-multipart = "^0.0.5" 33 | 34 | 35 | [tool.poetry.group.dev.dependencies] 36 | watchfiles = "^0.18.1" 37 | black = "^22.12.0" 38 | isort = "^5.11.4" 39 | flake8 = "^4.0.1" 40 | poethepoet = "^0.18.0" 41 | pytest = "^7.2.0" 42 | mypy = "^1.0.1" 43 | 44 | [build-system] 45 | requires = ["poetry-core"] 46 | build-backend = "poetry.core.masonry.api" 47 | 48 | [tool.poe.tasks] 49 | test = { shell = "pytest tests/" } 50 | _flake8 = { shell = "flake8 api_backend/ tests/" } 51 | _isort = { shell = "isort ." } 52 | _black = { shell = "black ." } 53 | _mypy = { shell = "mypy ./api_backend" } 54 | _black_check = 'black --check .' 55 | _isort_check = 'isort --check .' 56 | 57 | [tool.poe.tasks.format] 58 | sequence = ["_isort", "_black"] 59 | ignore_fail = "return_non_zero" 60 | 61 | [tool.poe.tasks.format_check] 62 | sequence = ["_isort_check", "_black_check"] 63 | ignore_fail = "return_non_zero" 64 | 65 | [tool.poe.tasks.lint] 66 | sequence = ["format_check", "_flake8"] 67 | ignore_fail = "return_non_zero" 68 | 69 | [tool.poe.tasks.mypy] 70 | sequence = ["_mypy"] 71 | ignore_fail = "return_non_zero" 72 | 73 | [tool.black] 74 | line-length = 120 75 | target-version = ['py38'] 76 | include = '\.pyi?$' 77 | extend-exclude = ''' 78 | ( 79 | __pycache__ 80 | | \.github 81 | ) 82 | ''' 83 | 84 | [tool.pylint] 85 | max-line-length = 120 86 | disable = ["W0511"] 87 | 88 | [tool.isort] 89 | profile = 'black' 90 | multi_line_output = 3 91 | line_length = 120 92 | py_version = 38 -------------------------------------------------------------------------------- /python/api-backend/scripts/install_python_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Partition the instal by group, so 5 | # it won't run out of memory 6 | # 7 | 8 | echo "Installing common dependencies..." 9 | poetry install --no-interaction --no-root --no-ansi --only main,dev 10 | 11 | echo "Installing utility dependencies..." 12 | poetry install --no-interaction --no-root --no-ansi --only utility 13 | 14 | echo "Installing graph dependencies..." 15 | poetry install --no-interaction --no-root --no-ansi --only graph 16 | -------------------------------------------------------------------------------- /python/api-backend/scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "${DEBUG}" ]]; then 4 | # API debug disabled 5 | echo "Starting backend api with debug disabled" 6 | poetry run python -m uvicorn api_backend.api_main:app --host 0.0.0.0 --port 8081 7 | else 8 | # API debug enabled 9 | echo "Starting backend api with debug enabled" 10 | poetry run python -m debugpy --listen 0.0.0.0:6900 -m uvicorn api_backend.api_main:app --host 0.0.0.0 --port 8081 11 | fi -------------------------------------------------------------------------------- /python/api-backend/setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | extend-ignore = 3 | # Indentation — black handles 4 | E1 5 | W1 6 | # Whitespace — black handles 7 | E2 8 | W2 9 | # Blank lines — black handles 10 | E3 11 | W3 12 | # Imports — isort handles 13 | E4 14 | W4 15 | # Line length — black handles 16 | E5 17 | W5 18 | # No lambdas — too strict 19 | E731 -------------------------------------------------------------------------------- /python/api-backend/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/api-backend/tests/__init__.py -------------------------------------------------------------------------------- /python/transparency-engine/.codespellignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/transparency-engine/.codespellignore -------------------------------------------------------------------------------- /python/transparency-engine/.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Dockerfile", 3 | "context": "../", 4 | "dockerFile": "../Dockerfile", 5 | "extensions": [ 6 | "ms-python.python", 7 | "ms-toolsai.jupyter" 8 | ], 9 | "settings": {}, 10 | "forwardPorts": [ 11 | 4050 12 | ] 13 | } -------------------------------------------------------------------------------- /python/transparency-engine/.dockerignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | *.pyo 4 | *.pyd 5 | .Python 6 | env 7 | pip-log.txt 8 | pip-delete-this-directory.txt 9 | .tox 10 | .coverage 11 | .coverage.* 12 | .cache 13 | nosetests.xml 14 | coverage.xml 15 | *.cover 16 | *.log 17 | .git 18 | .mypy_cache 19 | .pytest_cache 20 | .hypothesis 21 | .venv 22 | .devcontainer -------------------------------------------------------------------------------- /python/transparency-engine/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 127 3 | max-complexity = 10 -------------------------------------------------------------------------------- /python/transparency-engine/.gitattributes: -------------------------------------------------------------------------------- 1 | *.js text eol=lf 2 | *.jsx text eol=lf 3 | *.mjs text eol=lf 4 | *.cjs text eol=lf 5 | *.ts text eol=lf 6 | *.tsx text eol=lf 7 | *.mts text eol=lf 8 | *.cts text eol=lf 9 | *.json text eol=lf 10 | *.yml text eol=lf 11 | *.md text eol=lf 12 | *.sh text eol=lf -------------------------------------------------------------------------------- /python/transparency-engine/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | 7 | # install python and java required for Spark 8 | # Image details: 9 | # - Ubuntu 20.04.5 10 | # - Spark 3.2.2 11 | # - Python 3.9.15 12 | FROM mcr.microsoft.com/mmlspark/build-minimal:0.10.2 13 | 14 | # COPY --from=py3 / / 15 | 16 | # Keeps Python from generating .pyc files in the container 17 | ENV PYTHONDONTWRITEBYTECODE 1 18 | # Turns off buffering for easier container logging 19 | ENV PYTHONUNBUFFERED 1 20 | 21 | ENV SPARK_EXECUTOR_MEMORY=2g 22 | 23 | # Install necessary dependencies to compile 24 | # C++11 support needed for gensim 25 | RUN apt-get update -y &&\ 26 | apt-get install -y curl &&\ 27 | apt-get install -y git &&\ 28 | apt-get install -y gcc &&\ 29 | apt-get install --no-install-recommends --yes build-essential &&\ 30 | apt-get install -y libc-dev &&\ 31 | rm -rf /var/lib/apt/lists/* 32 | 33 | 34 | # Install Poetry 35 | RUN curl -sSL https://install.python-poetry.org | python - --version 1.3.2 36 | ENV PATH="${PATH}:/root/.local/bin" 37 | 38 | WORKDIR /workdir 39 | 40 | # Copy backend assets 41 | # This is done separately so the base layers won't change that often 42 | # to speed up image pushing 43 | COPY ./poetry.lock . 44 | COPY ./pyproject.toml . 45 | COPY ./scripts ./scripts 46 | 47 | 48 | # Install python dependencies 49 | RUN ./scripts/install_python_deps.sh 50 | 51 | # Copy remaining files 52 | COPY . . 53 | 54 | ENTRYPOINT ["bash"] -------------------------------------------------------------------------------- /python/transparency-engine/docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | import os 7 | import sys 8 | sys.path.insert(0, os.path.abspath('../../')) 9 | sys.path.insert(0, os.path.abspath('../../transparency_engine')) 10 | 11 | # -- Project information ----------------------------------------------------- 12 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 13 | 14 | project = 'Transparency Engine' 15 | copyright = '2023, Microsoft' 16 | 17 | # -- General configuration --------------------------------------------------- 18 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 19 | 20 | 21 | 22 | extensions = [ 23 | 'sphinx.ext.autodoc', 24 | 'sphinx.ext.napoleon', 25 | 'sphinx.ext.autosummary', 26 | 'sphinx.ext.viewcode', 27 | 'sphinx_rtd_theme', 28 | 'sphinx.ext.todo', 29 | 'sphinx_copybutton', 30 | "sphinx_design", 31 | ] 32 | 33 | templates_path = ['_templates'] 34 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 35 | 36 | # sphinx-panels shouldn't add bootstrap css since the pydata-sphinx-theme 37 | # already loads it 38 | panels_add_bootstrap_css = False 39 | 40 | # -- Options for HTML output ------------------------------------------------- 41 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 42 | 43 | html_theme = 'pydata_sphinx_theme' 44 | 45 | # Theme options are theme-specific and customize the look and feel of a theme 46 | # further. For a list of options available for each theme, see the 47 | # documentation. 48 | # 49 | html_theme_options = { 50 | "navbar_end": ["navbar-icon-links.html", "search-field.html"] 51 | } 52 | 53 | 54 | html_static_path = ['_static'] 55 | 56 | # -- Extension configuration ------------------------------------------------- 57 | 58 | # -- Options for todo extension ---------------------------------------------- 59 | 60 | # If true, `todo` and `todoList` produce output, else they produce nothing. 61 | todo_include_todos = True 62 | 63 | # init docstrings should also be included in class 64 | autoclass_content = "both" 65 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/contributing/index.rst: -------------------------------------------------------------------------------- 1 | Contributing to Transparency Engine 2 | =================================== 3 | 4 | Transparency Engine community project and welcomes contributions. 5 | 6 | There are multiple ways to contribute to DoWhy. 7 | Here are some examples: 8 | 9 | * Adding Jupyter notebooks that describe the use of Transparency Engine for different use cases. 10 | 11 | * Helping to implement a new embbeding method for any of the graph analysis steps of the Transparency Engine. 12 | 13 | * Helping to extend the Transparency Engine so that we can support new functionality such as out-of-sample link prediction and more. 14 | 15 | * Helping to update the documentation for Transparency Engine. 16 | 17 | If you would like to contribute, please raise a pull request. 18 | If you have questions before contributing, you can start by opening an issue on Github. -------------------------------------------------------------------------------- /python/transparency-engine/docs/getting_started/index.rst: -------------------------------------------------------------------------------- 1 | .. toctree:: 2 | :maxdepth: 3 3 | :hidden: 4 | :glob: 5 | 6 | install 7 | 8 | Getting Started 9 | =============== 10 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/getting_started/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ^^^^^^^^^^^^ 3 | 4 | 5 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Transparency Engine documentation master file, created by 2 | sphinx-quickstart on Thu Jan 19 17:05:02 2023. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Transparency Engine's documentation! 7 | =============================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 3 11 | :hidden: 12 | :glob: 13 | :caption: Contents: 14 | 15 | getting_started/index 16 | transparency_engine 17 | contributing/index 18 | modules 19 | 20 | **Date**: |today| 21 | 22 | **Related resources**:| 23 | `Source Repository `__ | 24 | `Issues & Ideas `__ | 25 | 26 | Transparency Engine aims to detect and communicate the implicit structure of complex activities in real-world problem areas, 27 | in ways that support both situational awareness and targeted action. 28 | 29 | Indices and tables 30 | ================== 31 | 32 | * :ref:`genindex` 33 | * :ref:`modindex` 34 | * :ref:`search` 35 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/modules.rst: -------------------------------------------------------------------------------- 1 | transparency_engine 2 | =================== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | transparency_engine 8 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.io.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.io package 2 | =============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.io.data\_handler module 8 | -------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.io.data_handler 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.io 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.graph.link_inference.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.graph.link\_inference package 2 | ========================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.modules.graph.link\_inference.ann\_search module 8 | --------------------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.modules.graph.link_inference.ann_search 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.modules.graph.link_inference 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.graph.preprocessing.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.graph.preprocessing package 2 | ======================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.modules.graph.preprocessing.graph\_edges module 8 | -------------------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.modules.graph.preprocessing.graph_edges 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.modules.graph.preprocessing 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.graph.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.graph package 2 | ========================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | transparency_engine.modules.graph.link_inference 11 | transparency_engine.modules.graph.preprocessing 12 | 13 | Module contents 14 | --------------- 15 | 16 | .. automodule:: transparency_engine.modules.graph 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.graphs.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.graphs package 2 | =========================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | transparency_engine.modules.graphs.uase 11 | 12 | Submodules 13 | ---------- 14 | 15 | transparency\_engine.modules.graphs.ann\_search module 16 | ------------------------------------------------------ 17 | 18 | .. automodule:: transparency_engine.modules.graphs.ann_search 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: transparency_engine.modules.graphs 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.graphs.uase.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.graphs.uase package 2 | ================================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.modules.graphs.uase.uase\_embedding module 8 | --------------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.modules.graphs.uase.uase_embedding 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | transparency\_engine.modules.graphs.uase.uase\_graph module 16 | ----------------------------------------------------------- 17 | 18 | .. automodule:: transparency_engine.modules.graphs.uase.uase_graph 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: transparency_engine.modules.graphs.uase 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.math.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.math package 2 | ========================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.modules.math.matrix\_operations module 8 | ----------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.modules.math.matrix_operations 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | transparency\_engine.modules.math.similarity\_score module 16 | ---------------------------------------------------------- 17 | 18 | .. automodule:: transparency_engine.modules.math.similarity_score 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: transparency_engine.modules.math 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules package 2 | ==================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | transparency_engine.modules.graphs 11 | transparency_engine.modules.math 12 | transparency_engine.modules.stats 13 | 14 | Module contents 15 | --------------- 16 | 17 | .. automodule:: transparency_engine.modules 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.stats.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.stats package 2 | ========================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.modules.stats.anomaly\_detection module 8 | ------------------------------------------------------------ 9 | 10 | .. automodule:: transparency_engine.modules.stats.anomaly_detection 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.modules.stats 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.modules.text.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.modules.text package 2 | ========================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.modules.text.fuzzy\_matching module 8 | -------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.modules.text.fuzzy_matching 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.modules.text 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.pipeline.config.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.pipeline.config package 2 | ============================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.pipeline.config.pipeline\_config module 8 | ------------------------------------------------------------ 9 | 10 | .. automodule:: transparency_engine.pipeline.config.pipeline_config 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.pipeline.config 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.pipeline.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.pipeline package 2 | ===================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | transparency_engine.pipeline.config 11 | transparency_engine.pipeline.step 12 | 13 | Submodules 14 | ---------- 15 | 16 | transparency\_engine.pipeline.transparency\_pipeline module 17 | ----------------------------------------------------------- 18 | 19 | .. automodule:: transparency_engine.pipeline.transparency_pipeline 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: transparency_engine.pipeline 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.pipeline.step.data_load.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.pipeline.step.data\_load package 2 | ===================================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.pipeline.step.data\_load.data\_load\_step module 8 | --------------------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.pipeline.step.data_load.data_load_step 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.pipeline.step.data_load 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.pipeline.step.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.pipeline.step package 2 | ========================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | transparency_engine.pipeline.step.data_load 11 | 12 | Submodules 13 | ---------- 14 | 15 | transparency\_engine.pipeline.step.step module 16 | ---------------------------------------------- 17 | 18 | .. automodule:: transparency_engine.pipeline.step.step 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: transparency_engine.pipeline.step 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.preprocessing.graph.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.preprocessing.graph package 2 | ================================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.preprocessing.graph.dynamic\_multipartite\_graph module 8 | ---------------------------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.preprocessing.graph.dynamic_multipartite_graph 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | transparency\_engine.preprocessing.graph.multiplex\_graph module 16 | ---------------------------------------------------------------- 17 | 18 | .. automodule:: transparency_engine.preprocessing.graph.multiplex_graph 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: transparency_engine.preprocessing.graph 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.preprocessing.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.preprocessing package 2 | ========================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | transparency_engine.preprocessing.graph 11 | transparency_engine.preprocessing.text 12 | 13 | Module contents 14 | --------------- 15 | 16 | .. automodule:: transparency_engine.preprocessing 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.preprocessing.text.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.preprocessing.text package 2 | =============================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.preprocessing.text.fuzzy\_matching module 8 | -------------------------------------------------------------- 9 | 10 | .. automodule:: transparency_engine.preprocessing.text.fuzzy_matching 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.preprocessing.text 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.rst: -------------------------------------------------------------------------------- 1 | Transparency Engine API 2 | ======================= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | transparency_engine.io 11 | transparency_engine.modules 12 | transparency_engine.pipeline 13 | transparency_engine.preprocessing 14 | 15 | Submodules 16 | ---------- 17 | 18 | transparency\_engine.main module 19 | -------------------------------- 20 | 21 | .. automodule:: transparency_engine.main 22 | :members: 23 | :undoc-members: 24 | :show-inheritance: 25 | 26 | Module contents 27 | --------------- 28 | 29 | .. automodule:: transparency_engine 30 | :members: 31 | :undoc-members: 32 | :show-inheritance: 33 | -------------------------------------------------------------------------------- /python/transparency-engine/docs/transparency_engine.spark.rst: -------------------------------------------------------------------------------- 1 | transparency\_engine.spark package 2 | ================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | transparency\_engine.spark.utils module 8 | --------------------------------------- 9 | 10 | .. automodule:: transparency_engine.spark.utils 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Module contents 16 | --------------- 17 | 18 | .. automodule:: transparency_engine.spark 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | -------------------------------------------------------------------------------- /python/transparency-engine/samples/config/pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "A pipeline", 3 | "description": "A pipeline", 4 | "storage": { 5 | "type": "parquet", 6 | "root": "output/demo/" 7 | }, 8 | "steps": [ 9 | "prep", 10 | "individual_link_prediction", 11 | "individual_link_filtering", 12 | "macro_link_prediction", 13 | "macro_link_filtering", 14 | "scoring", 15 | "report" 16 | ] 17 | } -------------------------------------------------------------------------------- /python/transparency-engine/scripts/install_python_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Partition the install by group, so 5 | # it won't run out of memory 6 | # 7 | 8 | echo "Installing common dependencies..." 9 | poetry config installer.max-workers 10 10 | poetry install --no-interaction --no-root --no-ansi --only main,dev 11 | -------------------------------------------------------------------------------- /python/transparency-engine/setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | extend-ignore = 3 | # Indentation — black handles 4 | E1 5 | W1 6 | # Whitespace — black handles 7 | E2 8 | W2 9 | # Blank lines — black handles 10 | E3 11 | W3 12 | # Imports — isort handles 13 | E4 14 | W4 15 | # Line length — black handles 16 | E5 17 | W5 18 | # No lambdas — too strict 19 | E731 -------------------------------------------------------------------------------- /python/transparency-engine/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/transparency-engine/tests/__init__.py -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.0" 2 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/analysis/link_filtering/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/analysis/link_filtering/base_link_filter.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | """Contains the base class for transformers that transform a Spark dataframe to a new dataframe.""" 6 | 7 | from abc import ABC, abstractmethod 8 | from typing import Any, Tuple, Union 9 | 10 | from pyspark.sql import DataFrame 11 | 12 | 13 | class BaseLinkFilter(ABC): 14 | """Base class for a filter that transforms a Spark dataframe to a new dataframe.""" 15 | 16 | def __init__(self, config: Any): 17 | """Initialize a filter.""" 18 | self.config = config 19 | 20 | @abstractmethod 21 | def filter( 22 | self, input_data: DataFrame 23 | ) -> Union[DataFrame, Tuple[DataFrame, DataFrame]]: 24 | """ 25 | Perform the gilters on the input dataframe and return a new dataframe. 26 | 27 | Params: 28 | input_data: Spark DataFrame The input dataset to apply filters 29 | 30 | Returns: 31 | output_data: Spark DataFrame The output of the filtering 32 | """ 33 | pass 34 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/analysis/link_inference/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.preprocessing.graph.multipartite_graph import ( 7 | MultipartiteGraphTransformer, 8 | ) 9 | from transparency_engine.preprocessing.graph.multiplex_graph import ( 10 | MultiplexGraphTransformer, 11 | ) 12 | 13 | 14 | __all__ = ["MultiplexGraphTransformer", "MultipartiteGraphTransformer"] 15 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/analysis/link_inference/base_link_estimator.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import logging 7 | 8 | from abc import abstractmethod 9 | from typing import Generic, List, TypeVar, Union 10 | 11 | from dataclasses import dataclass 12 | from pyspark.sql import DataFrame 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | @dataclass 19 | class BaseLinkConfig: 20 | """ 21 | Base config for an estimator that predicts links between entities based on static attributes. 22 | 23 | Params: 24 | n_links: int, default = 20 25 | Max number of links to return for each entity 26 | """ 27 | 28 | n_links: int = 20 29 | 30 | 31 | LinkConfig = TypeVar("LinkConfig", bound=BaseLinkConfig) 32 | 33 | 34 | class BaseLinkEstimator(Generic[LinkConfig]): 35 | """ 36 | An estimator that predicts links between entities based on static or dynamic relationships. 37 | """ 38 | 39 | def __init__( 40 | self, 41 | configs: LinkConfig, 42 | ): 43 | """ 44 | Params: 45 | configs: LinkConfig 46 | Configurations for the estimator that predicts entity links 47 | """ 48 | self.configs = configs 49 | 50 | @abstractmethod 51 | def predict(self, input_data: Union[DataFrame, List[DataFrame]]) -> DataFrame: 52 | """ 53 | Predict entity links based on entities' static/dynamic relationships. 54 | 55 | Params: 56 | input_data: Spark DataFrame 57 | Contains data related to entity relationships 58 | 59 | Returns: 60 | Spark DataFrame Dataframe containing predicted links 61 | """ 62 | pass 63 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/analysis/scoring/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.analysis.scoring.entity_scoring import compute_entity_score 7 | from transparency_engine.analysis.scoring.measures import ( 8 | EntityMeasures, 9 | NetworkMeasures, 10 | NormalizationTypes, 11 | ScoringTypes, 12 | ) 13 | from transparency_engine.analysis.scoring.network_scoring import ( 14 | compute_network_score, 15 | direct_dynamic_link_udf, 16 | direct_related_entity_udf, 17 | ) 18 | 19 | 20 | __all__ = [ 21 | "EntityMeasures", 22 | "NetworkMeasures", 23 | "ScoringTypes", 24 | "NormalizationTypes", 25 | "compute_entity_score", 26 | "compute_network_score", 27 | "direct_dynamic_link_udf", 28 | "direct_related_entity_udf", 29 | ] 30 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/base_transformer.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | """Contains the base class for transformers that transform a Spark dataframe to a new dataframe.""" 6 | 7 | import logging 8 | 9 | from abc import ABC, abstractmethod 10 | from typing import Any 11 | 12 | from pyspark.sql import DataFrame 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class BaseTransformer(ABC): 19 | """Base class for a transformer that transforms a Spark dataframe to a new dataframe.""" 20 | 21 | def __init__(self, configs: Any): 22 | """Initialize a transformer.""" 23 | self.configs = configs 24 | 25 | @abstractmethod 26 | def transform(self, input_data: DataFrame) -> DataFrame: 27 | """ 28 | Perform the transformations on the input dataframe and return a new dataframe. 29 | 30 | Params: 31 | input_data: Spark DataFrame The input dataset to apply transformations 32 | 33 | Returns: 34 | output_data: Spark DataFrame The output of the transformation 35 | """ 36 | pass 37 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/containers.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | """ 6 | Container definition. 7 | 8 | This module contains the dependency injection container for the transparency engine. 9 | """ 10 | 11 | 12 | from enum import Enum 13 | from typing import Any, Dict, List 14 | 15 | from dependency_injector import containers, providers 16 | from dependency_injector.wiring import register_loader_containers 17 | 18 | 19 | class ContainerKeys(str, Enum): 20 | """ 21 | Container keys. 22 | 23 | This class contains the keys for the dependency injection container. 24 | """ 25 | 26 | DATA_HANDLER = "data_handler" 27 | PIPELINE_CONFIG = "pipeline_config" 28 | STEP_CONFIG = "step_config" 29 | 30 | 31 | def build_container( 32 | attributes: Dict[ContainerKeys, Any], 33 | modules: List[str], 34 | packages: List[str], 35 | ) -> containers.DynamicContainer: 36 | """ 37 | Build the dependency injection container. 38 | 39 | Parameters 40 | ---------- 41 | attributes : Dict[ContainerKeys, Any] 42 | The attributes to add to the container. 43 | modules : List[str] 44 | The modules to add to the container. 45 | packages : List[str] 46 | The packages to add to the container. 47 | 48 | Returns 49 | ------- 50 | containers.DynamicContainer 51 | The dependency injection container. 52 | """ 53 | container = containers.DynamicContainer() 54 | register_loader_containers(container) 55 | 56 | for attribute, value in attributes.items(): 57 | if isinstance(value, Dict): 58 | setattr(container, attribute, providers.Dict(value)) 59 | elif isinstance(value, List): 60 | setattr(container, attribute, providers.List(value)) 61 | elif isinstance(value, tuple): 62 | # If it is a Tuple, we unpack and invoke the callable 63 | setattr(container, attribute, providers.ThreadSafeSingleton(*value)) 64 | container.wiring_config = containers.WiringConfiguration( 65 | modules=modules, packages=packages 66 | ) 67 | 68 | # Wire the container configuration 69 | container.wire() 70 | 71 | return container 72 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/io/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/main.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | """ 6 | Main module for the transparency engine. 7 | 8 | This module contains the main function for the transparency engine. 9 | 10 | Example 11 | ------- 12 | $ python transparency_engine/main.py --config pipeline.json --steps steps.json 13 | """ 14 | 15 | import argparse 16 | import json 17 | import logging 18 | 19 | from typing import Any, Dict 20 | 21 | from transparency_engine.containers import ContainerKeys, build_container 22 | from transparency_engine.io.data_handler import DataHandler, DataHandlerModes 23 | from transparency_engine.pipeline import TransparencyPipeline 24 | from transparency_engine.typing import PipelineSteps 25 | 26 | 27 | # Set to info for now. Pull from config later 28 | logging.basicConfig(level=logging.INFO) 29 | 30 | 31 | def main(config_path: str, steps_path: str) -> None: 32 | """ 33 | Run Main function for the transparency engine. 34 | 35 | Parameters 36 | ---------- 37 | config_path : str 38 | Path to the configuration file. 39 | 40 | Returns 41 | ------- 42 | None 43 | """ 44 | # Initialize the containers 45 | pipeline_config: Dict[str, Any] = dict() 46 | step_config: Dict[str, Any] = dict() 47 | 48 | with open(config_path) as config_file: 49 | pipeline_config = json.load(config_file) 50 | 51 | with open(steps_path) as steps_file: 52 | step_config = json.load(steps_file) 53 | 54 | storage_config: Dict[str, str] = pipeline_config.get("storage", dict()) 55 | 56 | build_container( 57 | { 58 | ContainerKeys.STEP_CONFIG: step_config, 59 | ContainerKeys.PIPELINE_CONFIG: pipeline_config, 60 | ContainerKeys.DATA_HANDLER: ( 61 | DataHandler, 62 | DataHandlerModes.from_string(storage_config.get("type", "")), 63 | storage_config.get("root", ""), 64 | ), 65 | }, 66 | modules=["transparency_engine.pipeline"], 67 | packages=[], 68 | ) 69 | 70 | pipeline = TransparencyPipeline() 71 | 72 | steps = PipelineSteps.from_string_list(pipeline_config.get("steps", [])) 73 | 74 | pipeline.run(steps=steps) 75 | 76 | 77 | if __name__ == "__main__": 78 | parser = argparse.ArgumentParser(description="Execute the transparency engine.") 79 | parser.add_argument( 80 | "--config", metavar="config", required=True, help="the path to config json file" 81 | ) 82 | parser.add_argument( 83 | "--steps", metavar="steps", required=True, help="path to the steps json file" 84 | ) 85 | args = parser.parse_args() 86 | main(args.config, args.steps) 87 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/data_generator/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.data_generator.synthetic_data_generator import ( 7 | DataGeneratorConfig, 8 | DataSourceConfig, 9 | DynamicAttributeConfig, 10 | EntityConfig, 11 | FlagConfig, 12 | StaticAttributeConfig, 13 | SyntheticDataGenerator, 14 | write_synthetic_data, 15 | ) 16 | 17 | 18 | __all__ = [ 19 | "DataGeneratorConfig", 20 | "SyntheticDataGenerator", 21 | "StaticAttributeConfig", 22 | "DynamicAttributeConfig", 23 | "FlagConfig", 24 | "EntityConfig", 25 | "DataSourceConfig", 26 | "write_synthetic_data", 27 | ] 28 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/data_shaper/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.data_shaper.spark_transform import column_to_list, melt 7 | 8 | 9 | __all__ = ["melt", "column_to_list"] 10 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/data_shaper/spark_transform.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from itertools import chain 7 | from typing import List, Union 8 | 9 | import pyspark.sql.functions as F 10 | 11 | from pyspark.sql import DataFrame 12 | 13 | from transparency_engine.pipeline.schemas import ATTRIBUTE_ID, VALUE 14 | 15 | 16 | """ 17 | This module contains Spark utility functions for transforming Spark dataframes. 18 | """ 19 | 20 | 21 | def melt( 22 | data: DataFrame, 23 | id_cols: Union[str, List[str]], 24 | value_cols: Union[List[str], None] = None, 25 | attribute_col: str = ATTRIBUTE_ID, 26 | value_col: str = VALUE, 27 | ) -> DataFrame: 28 | """ 29 | Convert dataframe from wide format to long format (equivalent of pd.melt). 30 | 31 | Params: 32 | ---------- 33 | data: Dataframe 34 | Dataframe in wide format 35 | id_cols: Union[str, List[str]] 36 | Name of the columns to keep as id columns in the long format 37 | value_cols: Union[List[str], None], default = None 38 | Name of the columns to be converted to [attribute, value] columns in the long format. 39 | If None, convert all columns that are not in the id_cols list 40 | attribute_col: str 41 | Name of the attribute column in the long format 42 | value_col: str 43 | name of the value column in the long format 44 | 45 | Returns: 46 | ---------- 47 | DataFrame: Long-formatted dataframe with schema [id_columns, attribute_col, value_col] 48 | """ 49 | if isinstance(id_cols, str): 50 | id_cols = [id_cols] 51 | if value_cols is None: 52 | value_cols = [column for column in data.columns if column not in id_cols] 53 | 54 | attribute_values = F.create_map( 55 | *list( 56 | chain.from_iterable( 57 | [[F.lit(column), F.col(column)] for column in value_cols] 58 | ) 59 | ) 60 | ) 61 | data = ( 62 | data.select(*id_cols, F.explode(attribute_values)) 63 | .withColumnRenamed("key", attribute_col) 64 | .withColumnRenamed("value", value_col) 65 | ) 66 | return data 67 | 68 | 69 | def column_to_list(data: DataFrame, column: str, drop_duplicates: bool = True) -> List: 70 | """ 71 | Convert distinct values of a column to list. 72 | 73 | Params: 74 | data(DataFrame): Contains the column values to be extracted 75 | column(str): Column name 76 | drop_duplicates(bool): Drop duplicate values, default = True 77 | 78 | Return: 79 | values: List Values in the column 80 | """ 81 | if drop_duplicates: 82 | return [data[0] for data in data.select(column).distinct().collect()] 83 | else: 84 | return [data[0] for data in data.select(column).collect()] 85 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/graph/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/graph/embed/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.graph.embed.base_embed import ( 7 | BaseNodeEmbedding, 8 | Graph, 9 | GraphNode, 10 | ) 11 | from transparency_engine.modules.graph.embed.use_embedding import UnfoldedSpectralEmbed 12 | from transparency_engine.modules.graph.embed.use_graph import ( 13 | UnfoldedGraph, 14 | UnfoldedGraphNode, 15 | UnfoldedNodeTypes, 16 | ) 17 | 18 | 19 | __all__ = [ 20 | "Graph", 21 | "GraphNode", 22 | "UnfoldedGraph", 23 | "UnfoldedGraphNode", 24 | "UnfoldedNodeTypes", 25 | "BaseNodeEmbedding", 26 | "UnfoldedSpectralEmbed", 27 | ] 28 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/graph/link_filtering/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/graph/link_filtering/normalizer.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import pyspark.sql.functions as F 7 | 8 | from pyspark.sql import DataFrame 9 | 10 | import transparency_engine.pipeline.schemas as schemas 11 | 12 | from transparency_engine.modules.similarity.similarity_score import ( 13 | set_intersection_size_udf, 14 | set_intersection_udf, 15 | ) 16 | 17 | 18 | def compute_normalizer( 19 | aggregated_activity_data: DataFrame, 20 | source_entity: str = schemas.SOURCE, 21 | target_entity: str = schemas.TARGET, 22 | source_all_activities: str = f"{schemas.SOURCE}_all_{schemas.ACTIVITIES}", 23 | target_all_activities: str = f"{schemas.TARGET}_all_{schemas.ACTIVITIES}", 24 | source_period_activities: str = f"{schemas.SOURCE}_period_{schemas.ACTIVITIES}", 25 | target_period_activities: str = f"{schemas.TARGET}_period_{schemas.ACTIVITIES}", 26 | activity_time: str = "time", 27 | ): 28 | """ 29 | Compute a normalizer factor to penalize links with small number of overlapping periods 30 | """ 31 | normalizer = aggregated_activity_data.withColumn( 32 | "all_intersections", 33 | set_intersection_udf(source_all_activities, target_all_activities), 34 | ) 35 | normalizer = normalizer.withColumn( 36 | f"{schemas.SOURCE}_intersection_size", 37 | set_intersection_size_udf(source_period_activities, "all_intersections"), 38 | ) 39 | normalizer = normalizer.withColumn( 40 | f"{schemas.TARGET}_intersection_size", 41 | set_intersection_size_udf(target_period_activities, "all_intersections"), 42 | ) 43 | normalizer = normalizer.withColumn( 44 | "intersection_exist", 45 | F.when( 46 | (F.col(f"{schemas.SOURCE}_intersection_size") > 0) 47 | | (F.col(f"{schemas.TARGET}_intersection_size") > 0), 48 | 1, 49 | ).otherwise(0), 50 | ) 51 | normalizer = normalizer.groupBy(source_entity, target_entity).agg( 52 | F.sum("intersection_exist").alias("intersection_periods") 53 | ) 54 | period_count = aggregated_activity_data.select(activity_time).distinct().count() 55 | normalizer = normalizer.withColumn( 56 | "normalizer", F.col("intersection_periods") / F.lit(period_count) 57 | ) 58 | return normalizer 59 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/graph/link_inference/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.graph.link_inference.base_link_prediction import ( 7 | BaseNodeLinkEstimator, 8 | ) 9 | from transparency_engine.modules.graph.link_inference.use_link_prediction import ( 10 | USENodeLinkEstimator, 11 | ) 12 | 13 | 14 | __all__ = ["BaseNodeLinkEstimator", "USENodeLinkEstimator"] 15 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/graph/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.graph.preprocessing.data_formatting import format_attribute_data 7 | from transparency_engine.modules.graph.preprocessing.graph_edges import ( 8 | convert_links_to_bipartite, 9 | generate_bipartite_edges, 10 | generate_unipartite_edges, 11 | ) 12 | 13 | 14 | __all__ = [ 15 | "format_attribute_data", 16 | "generate_unipartite_edges", 17 | "generate_bipartite_edges", 18 | "convert_links_to_bipartite", 19 | ] 20 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/graph/preprocessing/data_formatting.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import pyspark.sql.functions as F 7 | from pyspark.sql import DataFrame 8 | 9 | def format_attribute_data( 10 | entity_attribute_data: DataFrame, 11 | value_col: str = "Value", 12 | ) -> DataFrame: 13 | """ 14 | Format the entity attribute values in the input data. 15 | 16 | Params: 17 | entity_attribute_data: DataFrame 18 | Input dataframe in the format of [EntityID, AttributeID, Value] 19 | 20 | Returns: 21 | Dataframe: Cleaned up attribute data. 22 | """ 23 | cleaned_data = entity_attribute_data.na.drop() 24 | cleaned_data = cleaned_data.withColumn(value_col, F.upper(F.trim(value_col))) 25 | return cleaned_data 26 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/math/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/transparency-engine/transparency_engine/modules/math/__init__.py -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/similarity/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.similarity.ann_search import ( 7 | get_all_closet_pairs, 8 | get_nearest_neigbors, 9 | index_vectors_ivf, 10 | ) 11 | 12 | 13 | __all__ = ["get_all_closet_pairs", "get_nearest_neigbors", "index_vectors_ivf"] 14 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/stats/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.stats.aggregation import ( 7 | mean_list_udf, 8 | normalize_max_scale, 9 | normalize_rank, 10 | percent_rank, 11 | ) 12 | from transparency_engine.modules.stats.arithmetic_operators import ( 13 | ArithmeticOperators, 14 | weighted_multiplication, 15 | weighted_sum, 16 | ) 17 | 18 | 19 | __all__ = [ 20 | "ArithmeticOperators", 21 | "weighted_sum", 22 | "weighted_multiplication", 23 | "percent_rank", 24 | "normalize_rank", 25 | "normalize_max_scale", 26 | "mean_list_udf", 27 | ] 28 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/stats/anomaly_detection.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | import pyspark.sql.functions as F 7 | 8 | from pyspark.sql import DataFrame 9 | 10 | 11 | """ 12 | This module contains functions for detecting outliers in a DataFrame. 13 | The functions in this module are designed to be used with PySpark DataFrames. 14 | """ 15 | 16 | 17 | def detect_anomaly_zscore( 18 | df: DataFrame, col: str, outlier_flag_col: str = "is_anomaly", min_zscore: float = 3 19 | ): 20 | """ 21 | A PySpark function to detect outliers in a DataFrame using the Z-Score method. 22 | 23 | Parameters: 24 | df (DataFrame): The DataFrame containing the column to check for outliers. 25 | col (str): The name of the column to check for outliers. 26 | outlier_flag_col (str, optional): The name of the column to add to the DataFrame indicating whether a row is an outlier. Defaults to "is_anomaly". 27 | min_zscore (float, optional): The minimum z-score required for a value to be considered an outlier. Defaults to 3. 28 | 29 | Returns: 30 | DataFrame: The input DataFrame with an added column indicating whether each row is an outlier. 31 | """ 32 | stats = df.agg( 33 | F.stddev(col).alias("std"), F.avg(col).alias("mean"), F.max(col).alias("max") 34 | ) 35 | df = df.join(F.broadcast(stats)) 36 | df = df.withColumn("z_score", (F.col(col) - F.col("mean")) / F.col("std")).drop( 37 | *["std", "mean", "max"] 38 | ) 39 | df = df.withColumn( 40 | outlier_flag_col, F.when(F.col("z_score") >= min_zscore, 1).otherwise(0) 41 | ).drop("z_score") 42 | return df 43 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/modules/text/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.modules.text.fuzzy_matching import lsh_match_text 7 | 8 | 9 | __all__ = ["lsh_match_text"] 10 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.pipeline.transparency_pipeline import TransparencyPipeline 7 | 8 | 9 | __all__ = ["TransparencyPipeline"] 10 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.preprocessing.graph.multipartite_graph import ( 7 | MultipartiteGraphTransformer, 8 | ) 9 | from transparency_engine.preprocessing.graph.multiplex_graph import ( 10 | MultiplexGraphTransformer, 11 | ) 12 | from transparency_engine.preprocessing.text.lsh_fuzzy_matching import ( 13 | LSHFuzzyMatchTransformer, 14 | ) 15 | 16 | 17 | __all__ = [ 18 | "LSHFuzzyMatchTransformer", 19 | "MultiplexGraphTransformer", 20 | "MultipartiteGraphTransformer", 21 | ] 22 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/preprocessing/flag/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.preprocessing.flag.flag_filtering import ( 7 | FlagFilterConfig, 8 | FlagFilterTransformer, 9 | generate_flag_metadata, 10 | validate_flag_metadata, 11 | ) 12 | 13 | 14 | __all__ = [ 15 | "FlagFilterConfig", 16 | "FlagFilterTransformer", 17 | "generate_flag_metadata", 18 | "validate_flag_metadata", 19 | ] 20 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/preprocessing/graph/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.preprocessing.graph.multipartite_graph import ( 7 | MultipartiteGraphTransformer, 8 | ) 9 | from transparency_engine.preprocessing.graph.multiplex_graph import ( 10 | MultiplexGraphTransformer, 11 | ) 12 | 13 | 14 | __all__ = ["MultiplexGraphTransformer", "MultipartiteGraphTransformer"] 15 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/preprocessing/text/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.preprocessing.text.lsh_fuzzy_matching import ( 7 | LSHFuzzyMatchTransformer, 8 | ) 9 | 10 | 11 | __all__ = ["LSHFuzzyMatchTransformer"] 12 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/reporting/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.reporting.entity_report import ( 7 | ReportConfig, 8 | ReportOutput, 9 | generate_report, 10 | ) 11 | 12 | 13 | __all__ = ["generate_report", "ReportConfig", "ReportOutput"] 14 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/spark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/transparency-engine/88b4617380351d7fcba289ee83f51d1afeae9932/python/transparency-engine/transparency_engine/spark/__init__.py -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/spark/utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | 7 | """ 8 | This script initializes a SparkSession and SparkContext, and creates a SparkConf object. 9 | The SparkSession is created using either an active SparkSession or a new SparkSession builder. 10 | The SparkContext is retrieved using the getOrCreate method. 11 | Finally, the SparkConf object is created using the getConf method on the SparkContext object. 12 | """ 13 | 14 | # Import required modules 15 | from pyspark import SparkConf, SparkContext 16 | from pyspark.sql import SparkSession 17 | 18 | 19 | # Initialize SparkSession and SparkContext 20 | config = SparkConf().setAll([("spark.executor.allowSparkContext", "true"), ("spark.port.maxRetries", "200")]) 21 | 22 | spark: SparkSession = ( 23 | SparkSession.builder.appName("Transparency Engine") 24 | .enableHiveSupport() 25 | .config(conf=config) 26 | .getOrCreate() 27 | ) 28 | 29 | sc: SparkContext = spark.sparkContext 30 | 31 | # Create SparkConf object 32 | conf: SparkConf = sc.getConf() 33 | -------------------------------------------------------------------------------- /python/transparency-engine/transparency_engine/synthetic_data/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) Microsoft. All rights reserved. 3 | # Licensed under the MIT license. See LICENSE file in the project. 4 | # 5 | 6 | from transparency_engine.synthetic_data.public_procurement import ( 7 | generate_procurement_data, 8 | ) 9 | 10 | 11 | __all__ = ["generate_procurement_data"] 12 | -------------------------------------------------------------------------------- /rome.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": { 3 | "ignore": [ 4 | "dist", 5 | "lib", 6 | "build", 7 | "python", 8 | "storybook-static", 9 | ".turbo", 10 | ".pnp.cjs", 11 | ".pnp.loader.mjs" 12 | ] 13 | }, 14 | "linter": { 15 | "enabled": true, 16 | "rules": { 17 | "recommended": true, 18 | "suspicious": { 19 | "noExplicitAny": "off", 20 | "noShadowRestrictedNames": "off" 21 | }, 22 | "a11y": { 23 | "useKeyWithClickEvents": "off" 24 | }, 25 | "performance": { 26 | "noDelete": "off" 27 | } 28 | } 29 | }, 30 | "javascript": { 31 | "formatter": { 32 | "semicolons": "asNeeded", 33 | "quoteStyle": "single", 34 | "trailingComma": "all", 35 | "quoteProperties": "asNeeded" 36 | } 37 | }, 38 | "formatter": { 39 | "indentStyle": "tab" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /scripts/build-backend-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | custom_docker_registry=$1 3 | 4 | if [ -z "$custom_docker_registry" ]; then 5 | echo "using default registry" 6 | else 7 | echo "using registry: $custom_docker_registry" 8 | fi 9 | 10 | echo "building backend image..." 11 | docker build -t backend python/api-backend --build-arg REGISTRY=$custom_docker_registry 12 | -------------------------------------------------------------------------------- /scripts/build-frontend-images.sh: -------------------------------------------------------------------------------- 1 | echo "building frontend..." 2 | docker build -t frontend javascript/webapp -------------------------------------------------------------------------------- /scripts/install-charts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | values_file=$1 3 | backend_version=$(awk '/^version:[^\n]*$/ {split($0, a); print a[2]}' helm/Chart.yaml) 4 | 5 | echo "Packaging transparency-engine..." 6 | helm package helm/ --destination helm/dist 7 | 8 | if [ -z "$values_file" ]; then 9 | echo "Installing transparency-engine with local values..." 10 | helm upgrade --install transparency-engine helm/dist/transparency-engine-$backend_version.tgz -f helm/values.local.yaml 11 | else 12 | echo "Installing transparency-engine with values from file '$values_file'..." 13 | helm upgrade --install transparency-engine helm/dist/transparency-engine-$backend_version.tgz -f $values_file --wait 14 | fi -------------------------------------------------------------------------------- /scripts/list-resources.sh: -------------------------------------------------------------------------------- 1 | namespaces="default backend frontend oauth-proxy ingress-nginx" 2 | 3 | for n in $namespaces 4 | do 5 | echo "------------------ $n namespace ------------------" 6 | kubectl get all,ingress -n $n 7 | echo 8 | done -------------------------------------------------------------------------------- /scripts/push-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | registry=$1 3 | 4 | docker tag backend $registry/backend 5 | docker push $registry/backend 6 | 7 | docker tag app-shell $registry/app-shell 8 | docker push $registry/app-shell -------------------------------------------------------------------------------- /scripts/start-backend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd python/api-backend 4 | echo "Installing common dependencies..." 5 | poetry install --no-interaction --no-root --no-ansi --only main,dev 6 | 7 | echo "Installing utility dependencies..." 8 | poetry install --no-interaction --no-root --no-ansi --only utility 9 | 10 | echo "Installing graph dependencies..." 11 | poetry install --no-interaction --no-root --no-ansi --only graph 12 | 13 | if [[ -z "${DEBUG}" ]]; then 14 | # API debug disabled 15 | echo "Starting backend api with debug disabled" 16 | poetry run python -m uvicorn api_backend.api_main:app --host 0.0.0.0 --port 8081 17 | else 18 | # API debug enabled 19 | echo "Starting backend api with debug enabled" 20 | poetry run python -m debugpy --listen 0.0.0.0:6900 -m uvicorn api_backend.api_main:app --host 0.0.0.0 --port 8081 21 | fi -------------------------------------------------------------------------------- /scripts/start-frontend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd javascript/webapp 4 | yarn install 5 | yarn start -------------------------------------------------------------------------------- /scripts/uninstall-charts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | helm uninstall transparency-engine -------------------------------------------------------------------------------- /turbo.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://turborepo.org/schema.json", 3 | "pipeline": { 4 | "build": { 5 | "dependsOn": [ 6 | "^build" 7 | ], 8 | "outputs": [ 9 | "dist/**", 10 | "docs/**" 11 | ] 12 | }, 13 | "test": { 14 | "dependsOn": [ 15 | "build" 16 | ], 17 | "outputs": [], 18 | "cache": false 19 | }, 20 | "check": { 21 | "dependsOn": [], 22 | "outputs": [] 23 | }, 24 | "check-deps": { 25 | "dependsOn": [], 26 | "outputs": [] 27 | }, 28 | "fix": { 29 | "dependsOn": [], 30 | "outputs": [] 31 | }, 32 | "bundle": { 33 | "dependsOn": [ 34 | "^build", 35 | "build" 36 | ], 37 | "outputs": [ 38 | "build/**", 39 | "storybook-static/**" 40 | ] 41 | }, 42 | "ci": { 43 | "dependsOn": [ 44 | "build", 45 | "check", 46 | "test", 47 | "check-deps", 48 | "bundle" 49 | ], 50 | "outputs": [ 51 | "dist/**", 52 | "docs/**", 53 | "build/**", 54 | "storybook-static/**" 55 | ] 56 | }, 57 | "prestart": { 58 | "dependsOn": [ 59 | "^prestart" 60 | ], 61 | "outputs": [ 62 | "dist/**" 63 | ] 64 | }, 65 | "start": { 66 | "cache": false, 67 | "dependsOn": [ 68 | "prestart", 69 | "^build" 70 | ], 71 | "outputs": [] 72 | }, 73 | "clean": { 74 | "cache": false 75 | }, 76 | "release": { 77 | "cache": false 78 | }, 79 | "deploy": { 80 | "cache": false 81 | } 82 | } 83 | } --------------------------------------------------------------------------------