2 |
3 |
4 |
6 |
7 |
9 |
10 | Compare manual and automatic labeling
11 |
12 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/browserslist:
--------------------------------------------------------------------------------
1 | # This file is currently used by autoprefixer to adjust CSS to support the below specified browsers
2 | # For additional information regarding the format and rule options, please see:
3 | # https://github.com/browserslist/browserslist#queries
4 | #
5 | # For IE 9-11 support, please remove 'not' from the last line of the file and adjust as needed
6 |
7 | > 0.5%
8 | last 2 versions
9 | Firefox ESR
10 | not dead
11 | not IE 9-11
--------------------------------------------------------------------------------
/deid_app/frontend/src/main.ts:
--------------------------------------------------------------------------------
1 | import 'hammerjs';
2 |
3 | import {enableProdMode} from '@angular/core';
4 | import {platformBrowserDynamic} from '@angular/platform-browser-dynamic';
5 |
6 | import {AppModule} from './app/app.module';
7 | import {environment} from './environments/environment';
8 |
9 | if (environment.production) {
10 | enableProdMode();
11 | }
12 |
13 | platformBrowserDynamic().bootstrapModule(AppModule).catch(
14 | err => console.log(err));
15 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/deidentify/compare-data/compare-data.component.css:
--------------------------------------------------------------------------------
1 | #compare-text-row {
2 | margin: 10px;
3 | display:flex;
4 | flex-direction:row;
5 | justify-content: space-around;
6 | }
7 |
8 | .compare-text-col {
9 | display: flex;
10 | flex-direction: column;
11 | justify-content: space-around;
12 | width: 45%;
13 | background-color: white;
14 | padding: 10px;
15 | }
16 |
17 | mat-form-field {
18 | width: 100%;
19 | }
20 |
21 | .data {
22 | flex-direction: row;
23 | }
24 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/common/submit_component.ts:
--------------------------------------------------------------------------------
1 | import {Component, Input, OnInit} from '@angular/core';
2 | import {FormGroup} from '@angular/forms';
3 |
4 | /**
5 | * Submit with spinner while running.
6 | */
7 | @Component({
8 | selector: 'app-submit',
9 | templateUrl: './submit_component.html',
10 | styleUrls: [
11 | './submit_component.css',
12 | ]
13 | })
14 | export class SubmitComponent {
15 | @Input() submitPlaceholder: string;
16 | @Input() submitForm: FormGroup;
17 | waiting = false;
18 | }
19 |
--------------------------------------------------------------------------------
/deid_app/frontend/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compileOnSave": false,
3 | "compilerOptions": {
4 | "baseUrl": "./",
5 | "outDir": "./dist/out-tsc",
6 | "sourceMap": true,
7 | "declaration": false,
8 | "module": "es2015",
9 | "moduleResolution": "node",
10 | "emitDecoratorMetadata": true,
11 | "experimentalDecorators": true,
12 | "target": "es5",
13 | "typeRoots": [
14 | "node_modules/@types"
15 | ],
16 | "lib": [
17 | "es2017",
18 | "dom"
19 | ]
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/deidentify/deidentify.component.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
6 |
7 |
9 |
10 |
12 |
13 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/common/submit_component.html:
--------------------------------------------------------------------------------
1 |
2 |
9 |
10 |
11 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/evaluate/evaluate.component.css:
--------------------------------------------------------------------------------
1 | input[type=number]::-webkit-inner-spin-button,
2 | input[type=number]::-webkit-outer-spin-button {
3 | -webkit-appearance: none;
4 | margin: 0;
5 | }
6 |
7 | .dlp-container {
8 | margin: 20px auto;
9 | padding: 0 15px;
10 | background-color: white;
11 | }
12 | @media (min-width: 768px) {
13 | .dlp-container {
14 | width: 750px;
15 | }
16 | }
17 | @media (min-width: 992px) {
18 | .dlp-container {
19 | width: 970px;
20 | }
21 |
22 | }
23 | @media (min-width: 1200px) {
24 | .dlp-container {
25 | width: 1170px;
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/evaluate/evaluate.component.ts:
--------------------------------------------------------------------------------
1 | import {Component, OnInit} from '@angular/core';
2 | import {DlpDemoService} from '../services/dlp-demo.service';
3 |
4 | @Component({
5 | selector: 'app-evaluate',
6 | templateUrl: './evaluate.component.html',
7 | styleUrls: ['./evaluate.component.css']
8 | })
9 | export class EvaluateComponent implements OnInit {
10 | constructor(private dlpDemoService: DlpDemoService) {}
11 |
12 | ngOnInit() {
13 | this.dlpDemoService.refreshEvalJobs();
14 | this.dlpDemoService.refreshDatasets();
15 | this.dlpDemoService.refreshProject();
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/deidentify/deidentify.component.ts:
--------------------------------------------------------------------------------
1 | import {Component, OnInit} from '@angular/core';
2 | import {DlpDemoService} from '../services/dlp-demo.service';
3 |
4 | @Component({
5 | selector: 'app-deidentify',
6 | templateUrl: './deidentify.component.html',
7 | styleUrls: ['./deidentify.component.css']
8 | })
9 | export class DeidentifyComponent implements OnInit {
10 | constructor(private dlpDemoService: DlpDemoService) {}
11 |
12 | ngOnInit() {
13 | this.dlpDemoService.refreshDeidJobs();
14 | this.dlpDemoService.refreshDatasets();
15 | this.dlpDemoService.refreshProject();
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/dlp/mae_testdata/sample.dtd:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/evaluate/eval_job.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Represents an Eval job along with the information needed for that job.
3 | */
4 | export interface EvalJob {
5 | id: number;
6 | name: string;
7 | findings: string;
8 | goldens: string;
9 | stats: string;
10 | debug: string;
11 | status: number;
12 | logTrace: string;
13 | timestamp: Date;
14 | }
15 |
16 | /**
17 | * Represents an Eval stats entry.
18 | */
19 | export interface EvalStats {
20 | infoType: string;
21 | recall?: number;
22 | precision?: number;
23 | fScore?: number;
24 | truePositives?: number;
25 | falsePositives?: number;
26 | falseNegatives?: number;
27 | timestamp: Date;
28 | }
29 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/deidentify/deidentify.component.css:
--------------------------------------------------------------------------------
1 | input[type=number]::-webkit-inner-spin-button,
2 | input[type=number]::-webkit-outer-spin-button {
3 | -webkit-appearance: none;
4 | margin: 0;
5 | }
6 |
7 | .dlp-container {
8 | margin: 20px 0;
9 | padding-right: 15px;
10 | padding-left: 15px;
11 | margin-right: auto;
12 | margin-left: auto;
13 | background-color: white;
14 | }
15 | @media (min-width: 768px) {
16 | .dlp-container {
17 | width: 750px;
18 | }
19 | }
20 | @media (min-width: 992px) {
21 | .dlp-container {
22 | width: 970px;
23 | }
24 |
25 | }
26 | @media (min-width: 1200px) {
27 | .dlp-container {
28 | width: 1170px;
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
2 |
3 |
5 |
7 | {{dataset}}
8 |
9 |
10 |
11 |
12 |
13 |
15 |
17 | {{table}}
18 |
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/requirements/BUILD:
--------------------------------------------------------------------------------
1 | # Copyright 2017 Google Inc. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | licenses(["notice"]) # Apache License 2.0
16 |
17 | exports_files(["requirements.txt"])
18 |
--------------------------------------------------------------------------------
/offline_tools/redactor/settings.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | rootProject.name = 'redactor'
18 | include 'lib', 'examples:tag_remover'
--------------------------------------------------------------------------------
/deid_app/frontend/src/evaluate/eval-stats/eval-stats.component.spec.ts:
--------------------------------------------------------------------------------
1 | import {async, ComponentFixture, TestBed} from '@angular/core/testing';
2 |
3 | import {EvalStatsComponent} from './eval-stats.component';
4 |
5 | describe('EvalStatsComponent', () => {
6 | let component: EvalStatsComponent;
7 | let fixture: ComponentFixture
13 |
26 |
27 |
--------------------------------------------------------------------------------
/deid_app/frontend/e2e/protractor.conf.js:
--------------------------------------------------------------------------------
1 | // Protractor configuration file, see link for more information
2 | // https://github.com/angular/protractor/blob/master/lib/config.ts
3 |
4 | const { SpecReporter } = require('jasmine-spec-reporter');
5 |
6 | exports.config = {
7 | allScriptsTimeout: 11000,
8 | specs: [
9 | './src/**/*.e2e-spec.ts'
10 | ],
11 | capabilities: {
12 | 'browserName': 'chrome'
13 | },
14 | directConnect: true,
15 | baseUrl: 'http://localhost:4200/',
16 | framework: 'jasmine',
17 | jasmineNodeOpts: {
18 | showColors: true,
19 | defaultTimeoutInterval: 30000,
20 | print: function() {}
21 | },
22 | onPrepare() {
23 | require('ts-node').register({
24 | project: require('path').join(__dirname, './tsconfig.e2e.json')
25 | });
26 | jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } }));
27 | }
28 | };
--------------------------------------------------------------------------------
/deid_app/frontend/src/common/bigquery-new-table.html:
--------------------------------------------------------------------------------
1 |
14 |
19 | Original Image
15 |
16 | ![]()
17 |
18 |
20 |
25 | Redacted Image
21 |
22 | ![]()
23 |
24 |
2 |
3 |
5 |
7 | {{dataset}}
8 |
9 |
10 |
11 |
12 |
13 |
18 |
19 |
21 | {{table}}
22 |
23 |
24 |
25 |
26 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/deidentify/deidentify.component.spec.ts:
--------------------------------------------------------------------------------
1 | import {NO_ERRORS_SCHEMA} from '@angular/core';
2 | import {async, ComponentFixture, TestBed} from '@angular/core/testing';
3 |
4 | import {DeidentifyComponent} from './deidentify.component';
5 |
6 | describe('DeidentifyComponent', () => {
7 | let component: DeidentifyComponent;
8 | let fixture: ComponentFixture
22 | {{title}}
23 |
24 |
2 |
32 |
--------------------------------------------------------------------------------
/offline_tools/redactor/examples/tag_remover/build.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | plugins {
18 | id 'java'
19 | id 'application'
20 | }
21 |
22 | buildDir = "/tmp/gradle_build/redactor/examples"
23 |
24 | dependencies {
25 | implementation project(':lib')
26 | implementation 'commons-cli:commons-cli:1.4'
27 |
28 | testImplementation 'junit:junit:4.12'
29 | testImplementation 'com.google.truth:truth:0.39'
30 | testImplementation 'commons-io:commons-io:2.6'
31 | }
32 |
33 | mainClassName = 'com.google.cloud.healthcare.deid.remover.TagRemover'
34 |
35 | jar {
36 | manifest {
37 | attributes 'Main-Class': mainClassName
38 | }
39 | }
--------------------------------------------------------------------------------
/deid_app/frontend/src/services/error_handler.ts:
--------------------------------------------------------------------------------
1 | import {Injectable} from '@angular/core';
2 | import {MatSnackBar} from '@angular/material/snack-bar';
3 |
4 | /**
5 | * Represents an error returned from the DlpDemo backend.
6 | */
7 | interface DlpServiceError {
8 | text: string;
9 | error: number;
10 | }
11 |
12 | /**
13 | * Handles errors and display their message in a snack bar.
14 | */
15 | @Injectable()
16 | export class ErrorHandler {
17 | constructor(
18 | public snackbar: MatSnackBar,
19 | ) {}
20 |
21 | /**
22 | * Opens a SnackBar with the error details for the user.
23 | */
24 | // tslint:disable-next-line:no-any error can be of any type by definition.
25 | handleError(err: any) {
26 | let message = 'Server Error';
27 | if (this.isDlpServiceError(err.error) && err.error.text) {
28 | message = err.error.text;
29 | }
30 | this.snackbar.open(message, 'close');
31 | }
32 |
33 | // tslint:disable-next-line:no-any error can be of any type by definition.
34 | private isDlpServiceError(errorObject: any): errorObject is DlpServiceError {
35 | return errorObject.text !== undefined && errorObject.error !== undefined &&
36 | typeof errorObject.text === 'string' &&
37 | typeof errorObject.error === 'number';
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/mist/bigquery_to_gcs.py:
--------------------------------------------------------------------------------
1 | # Copyright 2017 Google Inc. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Stand-alone executable version of bigquery_to_gcs_lib."""
16 |
17 | from __future__ import absolute_import
18 |
19 | import argparse
20 | import logging
21 | import sys
22 |
23 | from mist import bigquery_to_gcs_lib
24 |
25 |
26 | def main():
27 | logging.getLogger().setLevel(logging.INFO)
28 |
29 | parser = argparse.ArgumentParser(
30 | description=('Read from BigQuery to MIST format.'))
31 | bigquery_to_gcs_lib.add_all_args(parser)
32 | args, extra_args = parser.parse_known_args(sys.argv[1:])
33 |
34 | bigquery_to_gcs_lib.run_pipeline(
35 | args.input_query, args.output_path, extra_args)
36 |
37 |
38 | if __name__ == '__main__':
39 | main()
40 |
--------------------------------------------------------------------------------
/mist/gcs_to_bigquery.py:
--------------------------------------------------------------------------------
1 | # Copyright 2017 Google Inc. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Stand-alone executable version of gcs_to_bigquery_lib."""
16 |
17 | from __future__ import absolute_import
18 |
19 | import argparse
20 | import logging
21 | import sys
22 |
23 | from mist import gcs_to_bigquery_lib
24 |
25 |
26 | def main():
27 | logging.getLogger().setLevel(logging.INFO)
28 |
29 | parser = argparse.ArgumentParser(
30 | description=('Read from MIST files in GCS to BigQuery.'))
31 | gcs_to_bigquery_lib.add_all_args(parser)
32 | args, extra_args = parser.parse_known_args(sys.argv[1:])
33 |
34 | gcs_to_bigquery_lib.run_pipeline(
35 | args.input_pattern, args.output_table, extra_args)
36 |
37 | if __name__ == '__main__':
38 | main()
39 |
--------------------------------------------------------------------------------
/physionet/bigquery_to_gcs.py:
--------------------------------------------------------------------------------
1 | # Copyright 2017 Google Inc. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Stand-alone executable version of bigquery_to_gcs_lib."""
16 |
17 | from __future__ import absolute_import
18 |
19 | import argparse
20 | import logging
21 | import sys
22 |
23 | from physionet import bigquery_to_gcs_lib
24 |
25 |
26 | def main():
27 | logging.getLogger().setLevel(logging.INFO)
28 |
29 | parser = argparse.ArgumentParser(
30 | description=('Read from BigQuery to PhysioNet format.'))
31 | bigquery_to_gcs_lib.add_all_args(parser)
32 | args, extra_args = parser.parse_known_args(sys.argv[1:])
33 |
34 | bigquery_to_gcs_lib.run_pipeline(
35 | args.input_query, args.output_file, extra_args)
36 |
37 |
38 | if __name__ == '__main__':
39 | main()
40 |
--------------------------------------------------------------------------------
/physionet/gcs_to_bigquery.py:
--------------------------------------------------------------------------------
1 | # Copyright 2017 Google Inc. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Stand-alone executable version of gcs_to_bigquery_lib."""
16 |
17 | from __future__ import absolute_import
18 |
19 | import argparse
20 | import logging
21 | import sys
22 |
23 | from physionet import gcs_to_bigquery_lib
24 |
25 |
26 | def main():
27 | logging.getLogger().setLevel(logging.INFO)
28 |
29 | parser = argparse.ArgumentParser(
30 | description=('Read from PhysioNet files in GCS to BigQuery.'))
31 | gcs_to_bigquery_lib.add_all_args(parser)
32 | args, extra_args = parser.parse_known_args(sys.argv[1:])
33 |
34 | gcs_to_bigquery_lib.run_pipeline(
35 | args.input_pattern, args.output_table, extra_args)
36 |
37 | if __name__ == '__main__':
38 | main()
39 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/deidentify/current-jobs/current-jobs.component.ts:
--------------------------------------------------------------------------------
1 | import {Component, OnDestroy, OnInit} from '@angular/core';
2 | import {MatTableDataSource} from '@angular/material/table';
3 | import {Subscription} from 'rxjs';
4 |
5 | import {DlpDemoService} from '../../services/dlp-demo.service';
6 | import {DeidJob} from '../deid_job';
7 |
8 | /**
9 | * Displays the deidentify jobs that the user have created and ran.
10 | */
11 | @Component({
12 | selector: 'app-current-jobs',
13 | templateUrl: './current-jobs.component.html',
14 | styleUrls: [
15 | './current-jobs.component.css',
16 | '../deidentify.component.css',
17 | ]
18 | })
19 | export class CurrentJobsComponent implements OnInit, OnDestroy {
20 | private readonly subscriptions = new Subscription();
21 |
22 | dataSource = new MatTableDataSource();
23 | displayedColumns: string[] =
24 | ['id', 'name', 'originalQuery', 'deidTable', 'status', 'timestamp'];
25 |
26 | constructor(private dlpDemoService: DlpDemoService) {}
27 |
28 | ngOnInit() {
29 | /* Updates the dataSource whenever the with Deid Jobs. */
30 | this.subscriptions.add(
31 | this.dlpDemoService.deidJobs.subscribe((jobs: DeidJob[]) => {
32 | this.dataSource.data = jobs;
33 | }));
34 | }
35 |
36 | ngOnDestroy() {
37 | this.subscriptions.unsubscribe();
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/dlp-demo/dlp-image-demo/dlp-image-demo.component.spec.ts:
--------------------------------------------------------------------------------
1 | import {HttpClientTestingModule} from '@angular/common/http/testing';
2 | import {async, ComponentFixture, TestBed} from '@angular/core/testing';
3 | import {By} from '@angular/platform-browser';
4 |
5 | import {DlpImageDemoComponent} from './dlp-image-demo.component';
6 |
7 | describe('DlpImageDemoComponent', () => {
8 | let component: DlpImageDemoComponent;
9 | let fixture: ComponentFixtureUpload CSV to BigQuery
3 | 31 |
2 |
5 |
6 |
7 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/dlp/testdata/batch_config.json:
--------------------------------------------------------------------------------
1 | {
2 | "columns": {
3 | "passThrough": [
4 | {"name": "patient_id", "type": "stringValue"},
5 | {"name": "record_number", "type": "integerValue"}
6 | ],
7 | "inspect": [
8 | {"name": "note", "type": "stringValue"}
9 | ]
10 | },
11 | "infoTypeTransformations": [
12 | {
13 | "infoTypes": [
14 | {"name": "PHONE_NUMBER"},
15 | {"name": "US_CENSUS_NAME"},
16 | {"name": "US_FEMALE_NAME"},
17 | {"name": "US_MALE_NAME"},
18 | {"name": "KnownFirstName"},
19 | {"name": "KnownLastName"},
20 | {"name": "DatasetFirstName"}
21 | ],
22 | "primitiveTransformation": {
23 | "replaceWithInfoTypeConfig": {}
24 | }
25 | }
26 | ],
27 | "keyColumns": ["patient_id", "record_number"],
28 | "tagCategories": [
29 | {
30 | "name": "NAME",
31 | "infoTypes": ["US_CENSUS_NAME"]
32 | },
33 | {
34 | "name": "FIRST_NAME",
35 | "infoTypes": [
36 | "US_FEMALE_NAME", "US_MALE_NAME", "KnownFirstName", "DatasetFirstName"
37 | ]
38 | },
39 | {
40 | "name": "LAST_NAME",
41 | "infoTypes": ["KnownLastName"]
42 | },
43 | {
44 | "name": "PHONE",
45 | "infoTypes": ["PHONE_NUMBER"]
46 | },
47 | {
48 | "name": "UNKNOWN_CLASSIFICATION_TYPE",
49 | "infoTypes": []
50 | }
51 | ],
52 | "perRowTypes": [
53 | {
54 | "columnName": "first_name",
55 | "infoTypeName": "KnownFirstName"
56 | },
57 | {
58 | "columnName": "last_name",
59 | "infoTypeName": "KnownLastName"
60 | }
61 | ],
62 | "perDatasetTypes": [
63 | {
64 | "infoTypes": [
65 | {
66 | "columnName": "first_name",
67 | "infoTypeName": "DatasetFirstName"
68 | }
69 | ]
70 | }
71 | ]
72 | }
73 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/deidentify/run-deidentify/run-deidentify.component.spec.ts:
--------------------------------------------------------------------------------
1 | import {HttpClientTestingModule} from '@angular/common/http/testing';
2 | import {async, ComponentFixture, TestBed} from '@angular/core/testing';
3 | import {FormsModule, ReactiveFormsModule} from '@angular/forms';
4 | import {BrowserAnimationsModule} from '@angular/platform-browser/animations';
5 |
6 | import {AppMaterialModule} from '../../app/material.module';
7 |
8 | import {RunDeidentifyComponent} from './run-deidentify.component';
9 |
10 | describe('RunDeidentifyComponent', () => {
11 | let component: RunDeidentifyComponent;
12 | let fixture: ComponentFixtureDLP Deid Jobs
3 | 4 || ID | 9 |{{job.id}} | 10 |Name | 15 |{{job.name}} | 16 |Input Data | 21 |{{job.originalQuery}} | 22 |Redacted Table | 27 |{{job.deidTable}} | 28 |Status | 33 |{{job.status || "pending"}} | 34 |Timestamp | 39 |{{job.timestamp}} | 40 |
|---|
2 |
30 |
31 | Compare Jobs
3 | 29 |
33 |
58 |
--------------------------------------------------------------------------------
/mae/remove_invalid_characters.py:
--------------------------------------------------------------------------------
1 | # Copyright 2017 Google Inc. All rights reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | r"""Remove characters that aren't valid in XML 1.0 so MAE can read them.
15 |
16 | Usage:
17 | PYTHONPATH="."
18 | python mae/remove_invalid_characters.py --input_pattern="input/*.xml" \
19 | --output_dir="output"
20 | """
21 |
22 | from __future__ import absolute_import
23 |
24 | import argparse
25 | import glob
26 | import logging
27 | import os
28 | import sys
29 |
30 | from common import mae
31 |
32 |
33 | def run(file_pattern, output_dir):
34 | """Re-write the files with invalid XML characters removed."""
35 | logging.info('Matched files: %s', glob.glob(file_pattern))
36 | for filename in glob.glob(file_pattern):
37 | logging.info('Loading file: "%s"', filename)
38 | with open(filename) as f:
39 | contents = f.read()
40 | with open(os.path.join(output_dir, os.path.basename(filename)), 'w') as o:
41 | o.write(mae.remove_invalid_characters(contents))
42 |
43 |
44 | def main(argv):
45 | logging.getLogger().setLevel(logging.INFO)
46 |
47 | parser = argparse.ArgumentParser(
48 | description='Remove characters that aren\'t valid in XML 1.0.')
49 | parser.add_argument('--input_pattern', type=str, required=True)
50 | parser.add_argument('--output_dir', type=str, required=True)
51 | args, _ = parser.parse_known_args(argv[1:])
52 | run(args.input_pattern, args.output_dir)
53 | logging.info('Complete. Output is in "%s".', args.output_dir)
54 |
55 |
56 | if __name__ == '__main__':
57 | main(sys.argv)
58 |
--------------------------------------------------------------------------------
/dlp/testdata/config.json:
--------------------------------------------------------------------------------
1 | {
2 | "columns": {
3 | "passThrough": [
4 | {"name": "patient_id", "type": "stringValue"},
5 | {"name": "record_number", "type": "integerValue"}
6 | ],
7 | "inspect": [
8 | {"name": "note", "type": "stringValue"}
9 | ],
10 | "fieldTransform": [
11 | {"name": "field_transform_col", "type": "stringValue"}
12 | ]
13 | },
14 | "fieldTransformations": [
15 | {
16 | "fields": [{"name": "field_transform_col"}],
17 | "primitiveTransformation": {"redactConfig": {}}
18 | }
19 | ],
20 | "infoTypeTransformations": [
21 | {
22 | "infoTypes": [
23 | {"name": "PHONE_NUMBER"},
24 | {"name": "US_CENSUS_NAME"},
25 | {"name": "US_FEMALE_NAME"},
26 | {"name": "US_MALE_NAME"},
27 | {"name": "KnownFirstName"},
28 | {"name": "KnownLastName"},
29 | {"name": "DatasetFirstName"}
30 | ],
31 | "primitiveTransformation": {
32 | "replaceWithInfoTypeConfig": {}
33 | }
34 | }
35 | ],
36 | "keyColumns": ["patient_id", "record_number"],
37 | "tagCategories": [
38 | {
39 | "name": "NAME",
40 | "infoTypes": ["US_CENSUS_NAME"]
41 | },
42 | {
43 | "name": "FIRST_NAME",
44 | "infoTypes": [
45 | "US_FEMALE_NAME", "US_MALE_NAME", "KnownFirstName", "DatasetFirstName"
46 | ]
47 | },
48 | {
49 | "name": "LAST_NAME",
50 | "infoTypes": ["KnownLastName"]
51 | },
52 | {
53 | "name": "PHONE",
54 | "infoTypes": ["PHONE_NUMBER"]
55 | },
56 | {
57 | "name": "UNKNOWN_CLASSIFICATION_TYPE",
58 | "infoTypes": []
59 | }
60 | ],
61 | "perRowTypes": [
62 | {
63 | "columnName": "first_name",
64 | "infoTypeName": "KnownFirstName"
65 | },
66 | {
67 | "columnName": "last_name",
68 | "infoTypeName": "KnownLastName"
69 | }
70 | ],
71 | "perDatasetTypes": [
72 | {
73 | "infoTypes": [
74 | {
75 | "columnName": "first_name",
76 | "infoTypeName": "DatasetFirstName"
77 | }
78 | ]
79 | }
80 | ]
81 | }
82 |
--------------------------------------------------------------------------------
/offline_tools/redactor/lib/src/main/proto/DicomConfig.proto:
--------------------------------------------------------------------------------
1 | // Copyright 2019 Google Inc.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // https://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | syntax = "proto2";
16 |
17 | package redactor;
18 |
19 | option java_package = "com.google.cloud.healthcare.deid.redactor.protos";
20 | option java_outer_classname = "DicomConfigProtos";
21 |
22 | // Specifies the parameters needed for de-identification of DICOM stores.
23 | message DicomConfig {
24 | // List of tags to be filtered.
25 | message TagFilterList {
26 | // Tags to be filtered. Tags must be DICOM Data Elements, as defined at:
27 | // http://dicom.nema.org/medical/dicom/current/output/html/part06.html#table_6-1.
28 | // They may be provided by "Keyword" or "Tag". For example "PatientID",
29 | // "00100010".
30 | repeated string tags = 1;
31 | }
32 |
33 | // Profile that determines which tags to keep/remove.
34 | enum TagFilterProfile {
35 | // No tag filtration profile provided. Same as CHC_BASIC.
36 | TAG_FILTER_PROFILE_UNSPECIFIED = 0;
37 | // Redacts basic tags which commonly contain sensitive data.
38 | CHC_BASIC = 1;
39 | }
40 |
41 | // Determines tag filtering method (meaning which tags to keep/remove).
42 | oneof tag_filter {
43 | // List of tags to keep. Remove all other tags.
44 | TagFilterList keep_list = 1;
45 | // List of tags to remove. Keep all other tags.
46 | TagFilterList remove_list = 2;
47 | // Tag filtering profile that determines which tags to keep/remove.
48 | TagFilterProfile filter_profile = 3;
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/common/bigquery-new-table.ts:
--------------------------------------------------------------------------------
1 | import {Component} from '@angular/core';
2 | import {FormBuilder, FormControl, FormGroup, Validators} from '@angular/forms';
3 | import {combineLatest, Observable} from 'rxjs';
4 | import {map, startWith} from 'rxjs/operators';
5 |
6 | import {DlpDemoService} from '../services/dlp-demo.service';
7 |
8 | import {BigQueryTable} from './bigquery-table';
9 |
10 |
11 | /**
12 | * A selector for a new table from the user's BQ tables.
13 | */
14 | @Component({
15 | selector: 'bigquery-new-table',
16 | templateUrl: './bigquery-new-table.html',
17 | styleUrls: [
18 | './bigquery-new-table.css',
19 | ]
20 | })
21 | export class BigQueryNewTable extends BigQueryTable {
22 | private tableFilter: Observable
34 |
45 | Original Text
35 |
36 |
44 |
37 |
38 |
41 |
42 |
43 |
46 |
57 | Deidentified Text
47 |
48 |
56 |
49 |
50 |
53 |
54 |
55 |
2 |
4 |
8 |
10 |
12 | ID: {{job.id}} - Name: {{job.name}}
13 |
14 |
15 |
16 |
17 |
76 |
--------------------------------------------------------------------------------
/deid_app/frontend/src/evaluate/eval-stats/eval-stats.component.ts:
--------------------------------------------------------------------------------
1 | import {Component, OnDestroy, OnInit} from '@angular/core';
2 | import {FormControl, FormGroup} from '@angular/forms';
3 | import {MatTableDataSource} from '@angular/material/table';
4 | import {combineLatest, Observable, Subscription} from 'rxjs';
5 | import {filter, map} from 'rxjs/operators';
6 | import {startWith, switchMap} from 'rxjs/operators';
7 |
8 | import {DlpDemoService} from '../../services/dlp-demo.service';
9 | import {EvalJob, EvalStats} from '../eval_job';
10 |
11 | /**
12 | * This component displays the results generated in the statistics table after
13 | * running the Evaluation pipeline.
14 | */
15 | @Component({
16 | selector: 'app-eval-stats',
17 | templateUrl: './eval-stats.component.html',
18 | styleUrls: ['./eval-stats.component.css', '../evaluate.component.css']
19 | })
20 | export class EvalStatsComponent implements OnInit, OnDestroy {
21 | private readonly subscriptions = new Subscription();
22 |
23 | jobsFilter: ObservableStatistics For Evaluation Jobs
3 |
18 |
20 |
23 |
24 |
25 |
28 |
29 |
30 |
74 |
75 | Stats Table
19 || Info Type | 32 |{{stat.infoType}} | 33 |Recall | 38 |{{(stat.recall | percent:'1.0-2') || 'undefined'}} | 39 |Precision | 44 |{{(stat.precision | percent:'1.0-2') || 'undefined'}} | 45 |FScore | 50 |{{(stat.fScore | percent:'1.0-2') || 'undefined'}} | 51 |True Positives | 56 |{{stat.truePositives}} | 57 |False Positives | 62 |{{stat.falsePositives}} | 63 |False Negatives | 68 |{{stat.falseNegatives}} | 69 |
|---|