├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── custom.md │ └── feature_request.md ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── codecov.ynl ├── hash.txt ├── nbactions.xml ├── pom.xml ├── sample_projects ├── anonymizer │ ├── DataDefender-jar-file-here │ ├── anonymize.config │ ├── database.config │ ├── datadefender │ ├── datadefender.bat │ ├── extensions │ │ ├── EmployeeNumber.class │ │ └── EmployeeNumber.java │ └── requirement.xml ├── database-logging │ ├── DataDefender-jar-file-here │ ├── column-discovery.config │ ├── common-discovery.config │ ├── create-logging-table.sql │ ├── data-discovery.config │ ├── database.config │ ├── datadefender │ ├── datadefender.bat │ └── log4j2.properties ├── database_discovery │ ├── mysql │ │ ├── DataDefender-jar-file-here │ │ ├── column-discovery.config │ │ ├── common-discovery.config │ │ ├── data-discovery.config │ │ ├── database.config │ │ ├── datadefender │ │ └── datadefender.bat │ └── postgresql │ │ ├── DataDefender-jar-file-here │ │ ├── column-discovery.config │ │ ├── common-discovery.config │ │ ├── data-discovery.config │ │ ├── database.config │ │ ├── datadefender │ │ └── datadefender.bat └── file_discovery │ ├── DataDefender-jar-file-here │ ├── datadefender │ ├── datadefender.bat │ └── file-discovery.config └── src ├── main ├── java │ └── com │ │ └── strider │ │ └── datadefender │ │ ├── Anonymize.java │ │ ├── DataDefender.java │ │ ├── DataDefenderException.java │ │ ├── DbConfig.java │ │ ├── Discover.java │ │ ├── DiscoverColumns.java │ │ ├── DiscoverData.java │ │ ├── DiscoverFiles.java │ │ ├── Extract.java │ │ ├── IRequirementCommand.java │ │ ├── ModelDiscoveryConfig.java │ │ ├── TestRequirement.java │ │ ├── anonymizer │ │ ├── DatabaseAnonymizer.java │ │ ├── DatabaseAnonymizerException.java │ │ ├── IAnonymizer.java │ │ └── functions │ │ │ ├── Address.java │ │ │ ├── Bio.java │ │ │ ├── Core.java │ │ │ ├── Lipsum.java │ │ │ └── Table.java │ │ ├── database │ │ ├── DatabaseException.java │ │ ├── DbConnection.java │ │ ├── IDbConnection.java │ │ ├── IDbFactory.java │ │ ├── MsSqlDbConnection.java │ │ ├── metadata │ │ │ ├── IMetaData.java │ │ │ ├── MetaData.java │ │ │ ├── MySqlMetaData.java │ │ │ ├── SqlTypeToClass.java │ │ │ └── TableMetaData.java │ │ └── sqlbuilder │ │ │ ├── ISqlBuilder.java │ │ │ ├── MsSqlBuilder.java │ │ │ ├── OracleSqlBuilder.java │ │ │ └── SqlBuilder.java │ │ ├── discoverer │ │ ├── ColumnDiscoverer.java │ │ ├── DatabaseDiscoverer.java │ │ ├── Discoverer.java │ │ ├── FileDiscoverer.java │ │ ├── FileDiscoveryException.java │ │ ├── IDiscoverer.java │ │ ├── Model.java │ │ └── Probability.java │ │ ├── extensions │ │ ├── BiographicFunctions.java │ │ └── ExtensionExample.java │ │ ├── extractor │ │ ├── DataExtractor.java │ │ └── IExtractor.java │ │ ├── file │ │ └── metadata │ │ │ └── FileMatchMetaData.java │ │ ├── functions │ │ ├── NamedParameter.java │ │ └── Utils.java │ │ ├── report │ │ └── ReportUtil.java │ │ ├── requirement │ │ ├── ClassAdapter.java │ │ ├── Column.java │ │ ├── Exclude.java │ │ ├── Requirement.java │ │ ├── RequirementException.java │ │ ├── Table.java │ │ ├── TypeConverter.java │ │ ├── file │ │ │ ├── Generator.java │ │ │ ├── JaxbValidationEventHandler.java │ │ │ └── Loader.java │ │ ├── package-info.java │ │ ├── plan │ │ │ ├── Argument.java │ │ │ ├── ArrayElement.java │ │ │ ├── Function.java │ │ │ ├── FunctionAttributeAdapter.java │ │ │ ├── GlobalPlan.java │ │ │ ├── Invokable.java │ │ │ ├── Plan.java │ │ │ ├── PlanRef.java │ │ │ └── package-info.java │ │ └── registry │ │ │ ├── ClassAndFunctionRegistry.java │ │ │ ├── DatabaseAwareRequirementFunction.java │ │ │ └── RequirementFunction.java │ │ ├── specialcase │ │ ├── EmailDetector.java │ │ ├── PhiDetector.java │ │ ├── SinDetector.java │ │ └── SpecialCase.java │ │ └── utils │ │ ├── ApplicationLock.java │ │ ├── Encoder.java │ │ ├── ICloseableNoException.java │ │ ├── IConsumerWithException.java │ │ ├── ISupplierWithException.java │ │ ├── LikeMatcher.java │ │ ├── Score.java │ │ ├── Xeger.java │ │ └── XegerUtils.java └── resources │ ├── com │ └── strider │ │ └── datadefender │ │ ├── anonymizer │ │ └── functions │ │ │ ├── cities.txt │ │ │ ├── countries.txt │ │ │ ├── dictionary.txt │ │ │ ├── first_names.txt │ │ │ ├── last_names.txt │ │ │ ├── lipsum.txt │ │ │ ├── provinces_states.txt │ │ │ ├── provinces_states_codes.txt │ │ │ └── streets.txt │ │ ├── discoverer │ │ ├── en-ner-date.bin │ │ ├── en-ner-location.bin │ │ ├── en-ner-money.bin │ │ ├── en-ner-organization.bin │ │ ├── en-ner-person.bin │ │ ├── en-ner-time.bin │ │ ├── en-sent.bin │ │ └── en-token.bin │ │ └── requirement │ │ └── file │ │ └── requirement.xsd │ ├── filediscovery.properties │ ├── log4j2.properties │ ├── names.dict │ └── phi.txt ├── run-scripts ├── datadefender └── datadefender.bat └── test ├── java └── com │ └── strider │ └── datadefender │ ├── anonymizer │ └── functions │ │ ├── BioTest.java │ │ ├── CoreTest.java │ │ └── LipsumTest.java │ ├── database │ ├── DbConnectionTest.java │ ├── IDbFactoryTest.java │ ├── metadata │ │ └── MetaDataTest.java │ └── sqlbuilder │ │ └── SqlBuilderTest.java │ ├── functions │ └── UtilsTest.java │ └── utils │ ├── EncoderTest.java │ ├── LikeMatcherTest.java │ ├── RequirementUtilsTest.java │ └── XegerTest.java └── resources ├── AppPropertiesTest.properties ├── Requirement-H2DB-DG.xml ├── Requirement-H2DB.xml ├── Requirement.xml ├── core-test.txt ├── log4j2.properties └── names.txt /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | nbactions.xml 3 | *.log 4 | nb-configuration.xml 5 | sample_projects/database_discovery/logs/anonymizer.log.2 6 | sample_projects/database_discovery/logs/anonymizer.log.3 7 | /nbproject/ 8 | sample_projects/file_discovery/files/000b2798-b3a2-4231-aab1-6a5da3999a11.xls 9 | sample_projects/file_discovery/files/0074014f-b378-4e51-a1ea-56499560f1e3.pdf 10 | sample_projects/file_discovery/files/026fafb2-364e-4e04-b006-3b4ab5808e87.pdf 11 | sample_projects/file_discovery/files/0379be65-9ead-420e-93d5-8f3d4ef69287.pdf 12 | sample_projects/file_discovery/files/0a87d04c-8f4d-404f-85ab-bc9da56a7c04.docx 13 | sample_projects/file_discovery/files/0a965acb-e8e7-43f7-9aec-dde7b15eeca6.docx 14 | sample_projects/file_discovery/files/0aa0a91c-a325-4101-a8db-17ef92510da0.docx 15 | sample_projects/file_discovery/files/0aa28454-3280-4cd6-9f9c-d1331efe895d.pdf 16 | sample_projects/file_discovery/files/0b8da254-9dd1-48c6-9224-af656dbec616.docx 17 | sample_projects/file_discovery/files/0be3924f-c04b-4cf9-9258-f75fe2f91b9c.pdf 18 | sample_projects/file_discovery/files/0bebf028-07b2-4489-a08e-d9a6b1f4fbb8.docx 19 | sample_projects/file_discovery/files/0c309926-72e4-49c3-9f00-92617e8cf648.pdf 20 | sample_projects/file_discovery/files/1c0ee6a2-a86b-4fd7-9942-e20cb64de58c.xls 21 | sample_projects/file_discovery/files/1fba4e10-f1a5-4c42-b1cc-f6bf22136f39.xls 22 | sample_projects/file_discovery/files/2e7a4ced-dd04-4977-9b3c-1581d091d311.xls 23 | sample_projects/file_discovery/files/8d6b912d-a5ee-467e-ba82-4440f3fe816f.xls 24 | 25 | # Ignore IntelliJ files. 26 | .idea/ 27 | *.iml 28 | *.ipr 29 | *.iws 30 | 31 | # Ignore Eclipse files. 32 | .classpath 33 | .project 34 | .factorypath 35 | .checkstyle 36 | .fbExcludeFilterFile 37 | .apt_generated/ 38 | .settings/ 39 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | dist: bionic 3 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at agstrider@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | You can contribute by clicking the star button and using this software! 4 | 5 | Contributors are encouraged to fork this repository and issue pull requests. If you would like to contribute, please 6 | 7 | * Fork it 8 | * Create your feature branch (git checkout -b new-feature) 9 | * Commit your changes (git commit -am 'Add new feature') 10 | * Push to the branch (git push origin new-feature) 11 | * Create new Pull Request 12 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Versions 1.0.x and above are currently being supported with security updates. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | 1.0.x | :white_check_mark: | 10 | | < 1.0.x | :x: | 11 | 12 | ## Reporting a Vulnerability 13 | 14 | Please report a vulnerability by creating a new issue. The issue will be fixed in one business day. 15 | -------------------------------------------------------------------------------- /codecov.ynl: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "pom.xml" 3 | - "sample_projects/.*" 4 | - "DADependencyInjection/Libraries/.*" 5 | - "DADependencyInjectionTests/.*" 6 | - "**/*ViewController.swift" 7 | -------------------------------------------------------------------------------- /hash.txt: -------------------------------------------------------------------------------- 1 | qDZagyGH3z8fBDzAGhSPNgGV 2 | -------------------------------------------------------------------------------- /nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | run 5 | 6 | jar 7 | 8 | 9 | process-classes 10 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 11 | 12 | 13 | -classpath %classpath com.strider.datadefender.DataDefender anonymize 14 | java 15 | /Users/strider/work/strider/DataDefender 16 | 17 | 18 | 19 | debug 20 | 21 | jar 22 | 23 | 24 | process-classes 25 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 26 | 27 | 28 | -agentlib:jdwp=transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath com.strider.datadefender.DataDefender anonymize 29 | java 30 | true 31 | /Users/strider/work/strider/DataDefender 32 | 33 | 34 | 35 | profile 36 | 37 | jar 38 | 39 | 40 | process-classes 41 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 42 | 43 | 44 | -classpath %classpath com.strider.datadefender.DataDefender anonymize 45 | java 46 | /Users/strider/work/strider/DataDefender 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /sample_projects/anonymizer/DataDefender-jar-file-here: -------------------------------------------------------------------------------- 1 | DataDefender.jar would need to be copied to this folder. 2 | -------------------------------------------------------------------------------- /sample_projects/anonymizer/anonymize.config: -------------------------------------------------------------------------------- 1 | -r=requirement.xml 2 | -------------------------------------------------------------------------------- /sample_projects/anonymizer/database.config: -------------------------------------------------------------------------------- 1 | --url=jdbc:mysql://localhost:3306/mydb?zeroDateTimeBehavior=convertToNull 2 | --password 3 | --user=root 4 | -------------------------------------------------------------------------------- /sample_projects/anonymizer/datadefender: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -cp ".:extensions/*:extensions:lib:DataDefender.jar" com.strider.datadefender.DataDefender "$@" 3 | -------------------------------------------------------------------------------- /sample_projects/anonymizer/datadefender.bat: -------------------------------------------------------------------------------- 1 | java -cp ".;extensions/*;extensions;lib;DataDefender.jar" com.strider.datadefender.DataDefender %* 2 | -------------------------------------------------------------------------------- /sample_projects/anonymizer/extensions/EmployeeNumber.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/sample_projects/anonymizer/extensions/EmployeeNumber.class -------------------------------------------------------------------------------- /sample_projects/anonymizer/extensions/EmployeeNumber.java: -------------------------------------------------------------------------------- 1 | import org.apache.commons.lang3.RandomUtils; 2 | public class EmployeeNumber { 3 | public static String randomNumber() { 4 | return String.format("%086d", RandomUtils.nextInt(0, 1000000) * 18); 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /sample_projects/database-logging/DataDefender-jar-file-here: -------------------------------------------------------------------------------- 1 | DataDefender.jar would need to be copied to this folder. 2 | -------------------------------------------------------------------------------- /sample_projects/database-logging/column-discovery.config: -------------------------------------------------------------------------------- 1 | --column-pattern=(?i).*first.* 2 | --column-pattern=(?i).*last.* 3 | --column-pattern=(?i).*middle.* 4 | --column-pattern=(?i).*name.* 5 | --column-pattern=(?i).*birth.* 6 | --column-pattern=(?i)date* 7 | --column-pattern=(?i).*phone.* 8 | --column-pattern=(?i).*email.* 9 | --column-pattern=(?i).*sin.* 10 | -------------------------------------------------------------------------------- /sample_projects/database-logging/common-discovery.config: -------------------------------------------------------------------------------- 1 | -o=generated-requirement.xml 2 | -------------------------------------------------------------------------------- /sample_projects/database-logging/create-logging-table.sql: -------------------------------------------------------------------------------- 1 | create table log_table_name ( 2 | id int(11) AUTO_INCREMENT PRIMARY KEY, 3 | eventDate TIMESTAMP, 4 | level VARCHAR(20), 5 | logger VARCHAR(100), 6 | message TEXT, 7 | exception TEXT 8 | ); 9 | -------------------------------------------------------------------------------- /sample_projects/database-logging/data-discovery.config: -------------------------------------------------------------------------------- 1 | -m date 2 | -m location 3 | -m money 4 | -m organization 5 | -m person 6 | --score-calculation 7 | -e com.strider.datadefender.specialcase.SinDetector.detectSin 8 | -------------------------------------------------------------------------------- /sample_projects/database-logging/database.config: -------------------------------------------------------------------------------- 1 | --url=jdbc:mysql://localhost:3306/mydb?zeroDateTimeBehavior=convertToNull 2 | --password 3 | --user=root 4 | -------------------------------------------------------------------------------- /sample_projects/database-logging/datadefender: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -cp ".:extensions/*:extensions:lib:DataDefender.jar" com.strider.datadefender.DataDefender "$@" 3 | -------------------------------------------------------------------------------- /sample_projects/database-logging/datadefender.bat: -------------------------------------------------------------------------------- 1 | java -cp ".;extensions/*;extensions;lib;DataDefender.jar" com.strider.datadefender.DataDefender %* 2 | -------------------------------------------------------------------------------- /sample_projects/database-logging/log4j2.properties: -------------------------------------------------------------------------------- 1 | 2 | appender.console.type=Console 3 | appender.console.name=STDOUT 4 | appender.console.layout.type=PatternLayout 5 | appender.console.layout.pattern=[%highlight{%p}{FATAL=red, ERROR=red, WARN=yellow bright, INFO=blue bright, DEBUG=green bright}] %m%n 6 | 7 | # using a DynamicThresholdFilter keeps the --debug and --verbose console options 8 | # working. Otherwise setting the rootLogger level appropriately causes those 9 | # flags to not work. 10 | appender.console.filter.threshold.type=DynamicThresholdFilter 11 | appender.console.filter.threshold.defaultThreshold=warn 12 | appender.console.filter.threshold.key=console-level 13 | appender.console.filter.threshold.keyValuePair.type=KeyValuePair 14 | appender.console.filter.threshold.keyValuePair.key=info 15 | appender.console.filter.threshold.keyValuePair.value=info 16 | appender.console.filter.threshold.keyValuePair2.type=KeyValuePair 17 | appender.console.filter.threshold.keyValuePair2.key=debug 18 | appender.console.filter.threshold.keyValuePair2.value=debug 19 | appender.console.filter.threshold.onMatch=ACCEPT 20 | appender.console.filter.threshold.onMismatch=DENY 21 | 22 | appender.jdbc.type=JDBC 23 | appender.jdbc.name=DatabaseAppender 24 | appender.jdbc.tableName=log_table_name 25 | appender.jdbc.connectionSource.type=DriverManager 26 | appender.jdbc.connectionSource.connectionString=jdbc:mariadb://localhost:3306/database_name 27 | appender.jdbc.connectionSource.username=user 28 | appender.jdbc.connectionSource.password=password 29 | appender.jdbc.connectionSource.driverClassName=org.mariadb.jdbc.Driver 30 | 31 | # using a DynamicThresholdFilter keeps the --debug and --verbose console options 32 | # working. Otherwise setting the rootLogger level appropriately causes those 33 | # flags to not work. Note how this is configured with the "file-level" 34 | # threshold. Note also that this is completely optional... if the command-line 35 | # options are not needed, this can be skipped and set on rootLevel for example 36 | appender.jdbc.filter.threshold.type=DynamicThresholdFilter 37 | appender.jdbc.filter.threshold.defaultThreshold=info 38 | appender.jdbc.filter.threshold.key=file-level 39 | appender.jdbc.filter.threshold.keyValuePair.type=KeyValuePair 40 | appender.jdbc.filter.threshold.keyValuePair.key=debug 41 | appender.jdbc.filter.threshold.keyValuePair.value=debug 42 | appender.jdbc.filter.threshold.onMatch=ACCEPT 43 | appender.jdbc.filter.threshold.onMismatch=DENY 44 | 45 | appender.jdbc.columnConfigs[0].type=Column 46 | appender.jdbc.columnConfigs[0].name=eventDate 47 | appender.jdbc.columnConfigs[0].isEventTimestamp=true 48 | 49 | appender.jdbc.columnConfigs[1].type=Column 50 | appender.jdbc.columnConfigs[1].name=level 51 | appender.jdbc.columnConfigs[1].pattern=%level 52 | 53 | appender.jdbc.columnConfigs[2].type=Column 54 | appender.jdbc.columnConfigs[2].name=logger 55 | appender.jdbc.columnConfigs[2].pattern=%logger 56 | 57 | appender.jdbc.columnConfigs[3].type=Column 58 | appender.jdbc.columnConfigs[3].name=message 59 | appender.jdbc.columnConfigs[3].pattern=%m 60 | 61 | appender.jdbc.columnConfigs[4].type=Column 62 | appender.jdbc.columnConfigs[4].name=exception 63 | appender.jdbc.columnConfigs[4].pattern=%ex{full} 64 | 65 | rootLogger.level=all 66 | rootLogger.appenderRef.database.ref=DatabaseAppender 67 | rootLogger.appenderRef.console.ref=STDOUT 68 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/mysql/DataDefender-jar-file-here: -------------------------------------------------------------------------------- 1 | DataDefender.jar would need to be copied to this folder. 2 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/mysql/column-discovery.config: -------------------------------------------------------------------------------- 1 | --column-pattern=(?i).*first.* 2 | --column-pattern=(?i).*last.* 3 | --column-pattern=(?i).*middle.* 4 | --column-pattern=(?i).*name.* 5 | --column-pattern=(?i).*birth.* 6 | --column-pattern=(?i)date* 7 | --column-pattern=(?i).*phone.* 8 | --column-pattern=(?i).*email.* 9 | --column-pattern=(?i).*sin.* 10 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/mysql/common-discovery.config: -------------------------------------------------------------------------------- 1 | -o=generated-requirement.xml 2 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/mysql/data-discovery.config: -------------------------------------------------------------------------------- 1 | -m date 2 | -m location 3 | -m money 4 | -m organization 5 | -m person 6 | --score-calculation 7 | -e com.strider.datadefender.specialcase.SinDetector.detectSin 8 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/mysql/database.config: -------------------------------------------------------------------------------- 1 | --url=jdbc:mysql://localhost:3306/mydb?zeroDateTimeBehavior=convertToNull 2 | --password 3 | --user=root 4 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/mysql/datadefender: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -cp ".:extensions/*:extensions:lib:DataDefender.jar" com.strider.datadefender.DataDefender "$@" 3 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/mysql/datadefender.bat: -------------------------------------------------------------------------------- 1 | java -cp ".;extensions/*;extensions;lib;DataDefender.jar" com.strider.datadefender.DataDefender %* 2 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/postgresql/DataDefender-jar-file-here: -------------------------------------------------------------------------------- 1 | DataDefender.jar would need to be copied to this folder. 2 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/postgresql/column-discovery.config: -------------------------------------------------------------------------------- 1 | --column-pattern=.*f.*rst.* 2 | --column-pattern=.*l.*st.* 3 | --column-pattern=.*name.* 4 | --column-pattern=.*birth.* 5 | --column-pattern=.*address.* 6 | --column-pattern=.*phone.* 7 | --column-pattern=.*number.* 8 | --column-pattern=.*email.* 9 | --column-pattern=.*st.*d.*nt.* 10 | --column-pattern=.*sin.* 11 | --column-pattern=.*stud.* 12 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/postgresql/common-discovery.config: -------------------------------------------------------------------------------- 1 | -o=generated-requirement.xml 2 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/postgresql/data-discovery.config: -------------------------------------------------------------------------------- 1 | -m person 2 | --probability-threshold=0.55 3 | --limit=0 4 | --threshold-count=6 5 | --threshold-high=3 6 | --score-calculation 7 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/postgresql/database.config: -------------------------------------------------------------------------------- 1 | --url=jdbc:postgresql://localhost/example 2 | --schema=public 3 | --password 4 | --user=root 5 | --no-skip-empty-tables-metadata 6 | --exclude-table-pattern-metadata=".*db_meta_info.*" 7 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/postgresql/datadefender: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -cp ".:extensions/*:extensions:lib:DataDefender.jar" com.strider.datadefender.DataDefender "$@" 3 | -------------------------------------------------------------------------------- /sample_projects/database_discovery/postgresql/datadefender.bat: -------------------------------------------------------------------------------- 1 | java -cp ".;extensions/*;extensions;lib;DataDefender.jar" com.strider.datadefender.DataDefender %* 2 | -------------------------------------------------------------------------------- /sample_projects/file_discovery/DataDefender-jar-file-here: -------------------------------------------------------------------------------- 1 | DataDefender.jar would need to be copied to this folder. 2 | -------------------------------------------------------------------------------- /sample_projects/file_discovery/datadefender: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -cp ".:extensions/*:extensions:lib:DataDefender.jar" com.strider.datadefender.DataDefender "$@" 3 | -------------------------------------------------------------------------------- /sample_projects/file_discovery/datadefender.bat: -------------------------------------------------------------------------------- 1 | java -cp ".;extensions/*;extensions;lib;DataDefender.jar" com.strider.datadefender.DataDefender %* 2 | -------------------------------------------------------------------------------- /sample_projects/file_discovery/file-discovery.config: -------------------------------------------------------------------------------- 1 | -m date 2 | -m location 3 | -m money 4 | -m organization 5 | -m person 6 | --score-calculation 7 | --directory=/path/to/directory/to/scan 8 | --directory=/another/path/to/scan 9 | --exclude-extension=jar 10 | --exclude-extension=exe 11 | --exclude-extension=dll 12 | --exclude-extension=so 13 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/Anonymize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.anonymizer.DatabaseAnonymizer; 19 | import com.strider.datadefender.anonymizer.IAnonymizer; 20 | import com.strider.datadefender.database.IDbFactory; 21 | import com.strider.datadefender.requirement.Requirement; 22 | import com.strider.datadefender.requirement.registry.ClassAndFunctionRegistry; 23 | 24 | import java.util.concurrent.Callable; 25 | import java.util.List; 26 | 27 | import org.apache.commons.collections4.CollectionUtils; 28 | import org.apache.commons.lang3.StringUtils; 29 | 30 | import picocli.CommandLine.ArgGroup; 31 | import picocli.CommandLine.Option; 32 | import picocli.CommandLine.Command; 33 | import picocli.CommandLine.Parameters; 34 | 35 | import lombok.extern.log4j.Log4j2; 36 | 37 | /** 38 | * Anonymize picocli subcommand, configures and executes the database 39 | * anonymizer. 40 | * 41 | * @author Zaahid Bateson 42 | */ 43 | @Command( 44 | name = "anonymize", 45 | version = "1.0", 46 | mixinStandardHelpOptions = true, 47 | description = "Run anonymization utility" 48 | ) 49 | @Log4j2 50 | public class Anonymize implements Callable { 51 | 52 | @Option(names = { "-r", "--requirement-file" }, paramLabel = "", description = "Requirement XML file", required = true) 53 | private Requirement requirement; 54 | 55 | @Option(names = { "-b", "--batch-size" }, description = "Number of update queries to batch together", defaultValue = "1000") 56 | private Integer batchSize; 57 | 58 | @ArgGroup(exclusive = false, multiplicity = "1", heading = "Database connection settings%n") 59 | private DbConfig dbConfig; 60 | 61 | @Parameters(paramLabel = "tables", description = "Limit anonymization to specified tables") 62 | private List tables; 63 | 64 | @Override 65 | public Integer call() throws Exception { 66 | System.out.println(""); 67 | System.out.println("Starting anonymizer"); 68 | log.info("Datasource URL: {}, vendor: {}, schema: {}", dbConfig.getUrl(), dbConfig.getVendor(), dbConfig.getSchema()); 69 | log.info("Username: {}, Password provided: {}", dbConfig.getUsername(), (StringUtils.isNotBlank(dbConfig.getPassword()) ? "yes" : "no")); 70 | log.info("Batch size: {}", batchSize); 71 | log.info("Limiting to tables: {}", CollectionUtils.isEmpty(tables) ? "" : StringUtils.join(tables, ", ")); 72 | 73 | IDbFactory factory = IDbFactory.get(dbConfig); 74 | ClassAndFunctionRegistry.singleton().initialize(factory); 75 | 76 | final IAnonymizer anonymizer = new DatabaseAnonymizer( 77 | factory, 78 | dbConfig, 79 | batchSize, 80 | requirement, 81 | tables 82 | ); 83 | try { 84 | anonymizer.anonymize(); 85 | } catch (DataDefenderException e) { 86 | log.error(e.getMessage()); 87 | log.debug("Exception occurred during anonymization", e); 88 | return 1; 89 | } catch (Throwable e) { 90 | log.error(e.getMessage(), e); 91 | return 1; 92 | } 93 | return 0; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/DataDefenderException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | /** 19 | * Application-level exception. 20 | * 21 | * @author Armenak Grigoryan 22 | */ 23 | public class DataDefenderException extends Exception { 24 | 25 | private static final long serialVersionUID = 1L; 26 | 27 | public DataDefenderException(final String msg) { 28 | super(msg); 29 | } 30 | 31 | public DataDefenderException(final String msg, final Throwable t) { 32 | super(msg, t); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/DbConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.regex.Matcher; 21 | import java.util.regex.Pattern; 22 | import picocli.CommandLine.Option; 23 | 24 | import lombok.Getter; 25 | 26 | /** 27 | * Database configuration options for picocli. 28 | * 29 | * @author Zaahid Bateson 30 | */ 31 | @Getter 32 | public class DbConfig { 33 | 34 | public enum Vendor { 35 | H2, MYSQL, POSTGRESQL, SQLSERVER, ORACLE, MARIADB; 36 | } 37 | 38 | private static final Map VENDOR_MAP = Map.of( 39 | "h2", Vendor.H2, 40 | "mysql", Vendor.MYSQL, 41 | "mariadb", Vendor.MARIADB, 42 | "postgresql", Vendor.POSTGRESQL, 43 | "sqlserver", Vendor.SQLSERVER, 44 | "mssql", Vendor.SQLSERVER, 45 | "oracle", Vendor.ORACLE 46 | ); 47 | 48 | private Vendor vendor; 49 | 50 | @Option(names = { "-u", "--user" }, description = "The username to connect with") 51 | private String username; 52 | 53 | @Option(names = { "-p", "--password" }, description = "The password to connect with", arity = "0..1", interactive = true) 54 | private String password; 55 | 56 | @Option(names = { "--schema" }, description = "The schema to connect to") 57 | private String schema; 58 | 59 | @Option(names = { "--no-skip-empty-tables-metadata" }, description = "Include generating metadata for empty tables (defaults to skipping)", defaultValue = "true", negatable = true) 60 | private boolean skipEmptyTables = true; 61 | 62 | @Option(names = { "--include-table-pattern-metadata" }, description = "Pattern(s) matching table names to include for metadata analysis") 63 | private List includeTablePatterns; 64 | @Option(names = { "--exclude-table-pattern-metadata" }, description = "Pattern(s) matching table names to exclude for metadata analysis") 65 | private List excludeTablePatterns; 66 | 67 | private String url; 68 | 69 | @Option( 70 | names = { "--vendor" }, 71 | description = "Database vendor, available options are: h2, mysql, mariadb, postgresql, sqlserver, oracle. " 72 | + "If not specified, vendor will attempt to be extracted from the datasource url for a jdbc scheme." 73 | ) 74 | public void setVendor(String vendor) { 75 | String c = vendor.trim().toLowerCase(); 76 | if (!VENDOR_MAP.containsKey(c)) { 77 | throw new IllegalArgumentException( 78 | "Invalid value for option '--vendor': Valid options are: " 79 | + "h2, mysql, mariadb, postgresql, sqlserver and oracle." 80 | ); 81 | } 82 | this.vendor = VENDOR_MAP.get(c); 83 | } 84 | 85 | @Option(names = { "--url" }, description = "The datasource URL") 86 | public void setUrl(String url) { 87 | Pattern p = Pattern.compile("\\s*jdbc:([^:]+):.*"); 88 | if (vendor == null) { 89 | Matcher m = p.matcher(url); 90 | if (m.find()) { 91 | setVendor(m.group(1)); 92 | } 93 | } 94 | this.url = url; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/Discover.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 19 | import com.strider.datadefender.requirement.file.Generator; 20 | import java.io.File; 21 | import java.util.List; 22 | 23 | import java.util.Set; 24 | import java.util.concurrent.Callable; 25 | import java.util.stream.Collectors; 26 | 27 | import picocli.CommandLine; 28 | import picocli.CommandLine.ArgGroup; 29 | import picocli.CommandLine.Command; 30 | import picocli.CommandLine.Model.CommandSpec; 31 | import picocli.CommandLine.Option; 32 | import picocli.CommandLine.Spec; 33 | 34 | import lombok.extern.log4j.Log4j2; 35 | import lombok.Getter; 36 | import lombok.Setter; 37 | 38 | /** 39 | * "discover" picocli subcommand, configures and executes the data discoverer. 40 | * 41 | * TODO(ZB): Look into setting up command mixins sp dbconfig options don't have 42 | * to appear before subcommands [https://picocli.info/#_mixins] 43 | * 44 | * @author Zaahid Bateson 45 | */ 46 | @Command( 47 | name = "discover", 48 | version = "1.0", 49 | description = "Run data discovery utility", 50 | mixinStandardHelpOptions = true, 51 | subcommands = { 52 | DiscoverColumns.class, 53 | DiscoverData.class, 54 | DiscoverFiles.class 55 | }, 56 | subcommandsRepeatable = true 57 | ) 58 | @Log4j2 59 | public class Discover implements Callable { 60 | 61 | @Getter 62 | @Setter 63 | @ArgGroup(exclusive = false, multiplicity = "0..1", heading = "Database connection settings%n") 64 | private DbConfig dbConfig; 65 | 66 | @Option(names = { "-o", "--output" }, description = "Generate a requirements xml file and write it out to the specified file") 67 | @Setter 68 | private File outputFile; 69 | 70 | @Spec 71 | private CommandSpec spec; 72 | 73 | public void afterSubcommand() { 74 | List subcommands = spec 75 | .commandLine() 76 | .getParseResult() 77 | .subcommands() 78 | .stream() 79 | .map((p) -> p.commandSpec().userObject()) 80 | .filter((u) -> (u instanceof IRequirementCommand)) 81 | .map((r) -> (IRequirementCommand) r) 82 | .collect(Collectors.toList()); 83 | if (subcommands.stream().allMatch((r) -> r.getColumnMetaData() != null)) { 84 | Set combined = subcommands.stream().flatMap((r) -> r.getColumnMetaData().stream()).collect(Collectors.toSet()); 85 | try { 86 | Generator.write(Generator.create(combined), outputFile); 87 | } catch (Exception e) { 88 | log.error("Error creating or writing to an output xml file", e); 89 | } 90 | } 91 | } 92 | 93 | @Override 94 | public Integer call() throws Exception { 95 | CommandLine.usage(this, System.out); 96 | return 0; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/DiscoverColumns.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.database.IDbFactory; 19 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 20 | import com.strider.datadefender.discoverer.ColumnDiscoverer; 21 | import java.io.File; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.Callable; 25 | import java.util.regex.Pattern; 26 | 27 | import picocli.CommandLine.ArgGroup; 28 | import picocli.CommandLine.Command; 29 | import picocli.CommandLine.Model.CommandSpec; 30 | import picocli.CommandLine.Option; 31 | import picocli.CommandLine.ParentCommand; 32 | import picocli.CommandLine.Spec; 33 | 34 | import lombok.extern.log4j.Log4j2; 35 | import org.apache.commons.lang3.StringUtils; 36 | 37 | 38 | /** 39 | * "discover" picocli subcommand, configures and executes the data discoverer. 40 | * 41 | * @author Zaahid Bateson 42 | */ 43 | @Command( 44 | name = "columns", 45 | version = "1.0", 46 | mixinStandardHelpOptions = true, 47 | description = "Run column discovery utility" 48 | ) 49 | @Log4j2 50 | public class DiscoverColumns implements Callable, IRequirementCommand { 51 | 52 | private File outputFile; 53 | 54 | @Option(names = { "--column-pattern" }, description = "Regex pattern(s) to match column names", required = true) 55 | private List patterns; 56 | 57 | @ArgGroup(exclusive = false, multiplicity = "0..1", heading = "Database connection settings%n") 58 | private DbConfig dbConfig; 59 | 60 | @ParentCommand 61 | private Discover discover; 62 | 63 | @Spec 64 | private CommandSpec spec; 65 | 66 | private List results; 67 | 68 | @Option(names = { "-o", "--output" }, description = "Generate a requirements xml file and write it out to the specified file") 69 | public void setOutputFile(File f) { 70 | discover.setOutputFile(f); 71 | } 72 | 73 | @Override 74 | public Integer call() throws Exception { 75 | 76 | if (dbConfig == null) { 77 | dbConfig = discover.getDbConfig(); 78 | } else { 79 | discover.setDbConfig(dbConfig); 80 | } 81 | 82 | System.out.println("Starting column discovery"); 83 | log.warn("Discovery writes personal data to log files."); 84 | log.info("Datasource URL: {}, vendor: {}, schema: {}", dbConfig.getUrl(), dbConfig.getVendor(), dbConfig.getSchema()); 85 | log.info("Username: {}, Password provided: {}", dbConfig.getUsername(), (StringUtils.isNotBlank(dbConfig.getPassword()) ? "yes" : "no")); 86 | 87 | IDbFactory factory = IDbFactory.get(dbConfig); 88 | ColumnDiscoverer discoverer = new ColumnDiscoverer(factory, patterns); 89 | 90 | try { 91 | results = discoverer.discover(); 92 | } catch (Exception e) { 93 | log.error(e.getMessage()); 94 | log.debug("Exception occurred during anonymization", e); 95 | return 1; 96 | } 97 | 98 | discover.afterSubcommand(); 99 | 100 | return 0; 101 | } 102 | 103 | @Override 104 | public List getColumnMetaData() { 105 | return results; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/DiscoverData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.database.IDbFactory; 19 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 20 | import com.strider.datadefender.discoverer.DatabaseDiscoverer; 21 | 22 | import java.io.File; 23 | import java.util.List; 24 | import java.util.concurrent.Callable; 25 | 26 | import picocli.CommandLine.ArgGroup; 27 | import picocli.CommandLine.Command; 28 | import picocli.CommandLine.Model.CommandSpec; 29 | import picocli.CommandLine.Option; 30 | import picocli.CommandLine.ParentCommand; 31 | import picocli.CommandLine.Spec; 32 | 33 | import org.apache.commons.lang3.StringUtils; 34 | 35 | import lombok.extern.log4j.Log4j2; 36 | 37 | /** 38 | * "discover" picocli subcommand, configures and executes the data discoverer. 39 | * 40 | * @author Zaahid Bateson 41 | */ 42 | @Command( 43 | name = "data", 44 | version = "1.0", 45 | mixinStandardHelpOptions = true, 46 | description = "Run data discovery utility" 47 | ) 48 | @Log4j2 49 | public class DiscoverData implements Callable, IRequirementCommand { 50 | 51 | @ArgGroup(exclusive = false, multiplicity = "1", heading = "Model discovery settings%n") 52 | private ModelDiscoveryConfig modelDiscoveryConfig; 53 | 54 | @ArgGroup(exclusive = false, multiplicity = "0..1", heading = "Database connection settings%n") 55 | private DbConfig dbConfig; 56 | 57 | @ParentCommand 58 | private Discover discover; 59 | 60 | @Spec 61 | private CommandSpec spec; 62 | 63 | private List results; 64 | 65 | @Option(names = { "-o", "--output" }, description = "Generate a requirements xml file and write it out to the specified file") 66 | public void setOutputFile(File f) { 67 | discover.setOutputFile(f); 68 | } 69 | 70 | @Override 71 | public Integer call() throws Exception { 72 | 73 | if (dbConfig == null) { 74 | dbConfig = discover.getDbConfig(); 75 | } else { 76 | discover.setDbConfig(dbConfig); 77 | } 78 | 79 | System.out.println(""); 80 | System.out.println("Starting data discovery"); 81 | log.warn("Discovery writes personal data to log files."); 82 | 83 | log.info("Datasource URL: {}, vendor: {}, schema: {}", dbConfig.getUrl(), dbConfig.getVendor(), dbConfig.getSchema()); 84 | log.info("Username: {}, Password provided: {}", dbConfig.getUsername(), (StringUtils.isNotBlank(dbConfig.getPassword()) ? "yes" : "no")); 85 | log.info("Probability threshold: {}", modelDiscoveryConfig.getProbabilityThreshold()); 86 | log.info("Calculate score: {}", (modelDiscoveryConfig.getCalculateScore()) ? "yes" : "no"); 87 | log.info("Threshold count: {}", modelDiscoveryConfig.getThresholdCount()); 88 | log.info("Threshold high-risk count: {}", modelDiscoveryConfig.getThresholdHighRisk()); 89 | log.info("Limit: {}", modelDiscoveryConfig.getLimit()); 90 | log.info("Built-in models: {}", StringUtils.join(modelDiscoveryConfig.getModels(), ", ")); 91 | log.info("Custom models: {}", StringUtils.join(modelDiscoveryConfig.getFileModels(), ", ")); 92 | log.info("Custom token model: {}", modelDiscoveryConfig.getTokenModel()); 93 | log.info("Extensions: {}", StringUtils.join(modelDiscoveryConfig.getExtensions(), ", ")); 94 | 95 | IDbFactory factory = IDbFactory.get(dbConfig); 96 | DatabaseDiscoverer dd = new DatabaseDiscoverer(modelDiscoveryConfig, factory); 97 | results = dd.discover(); 98 | 99 | discover.afterSubcommand(); 100 | return 0; 101 | } 102 | 103 | @Override 104 | public List getColumnMetaData() { 105 | return results; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/DiscoverFiles.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.discoverer.FileDiscoverer; 19 | import java.io.File; 20 | import java.util.List; 21 | import java.util.concurrent.Callable; 22 | 23 | import picocli.CommandLine.ArgGroup; 24 | import picocli.CommandLine.Command; 25 | import picocli.CommandLine.Option; 26 | import picocli.CommandLine.ParentCommand; 27 | 28 | import lombok.extern.log4j.Log4j2; 29 | import org.apache.commons.lang3.StringUtils; 30 | 31 | 32 | /** 33 | * "discover" picocli subcommand, configures and executes the data discoverer. 34 | * 35 | * @author Zaahid Bateson 36 | */ 37 | @Command( 38 | name = "files", 39 | version = "1.0", 40 | mixinStandardHelpOptions = true, 41 | description = "Run file discovery utility" 42 | ) 43 | @Log4j2 44 | public class DiscoverFiles implements Callable { 45 | 46 | @ArgGroup(exclusive = false, multiplicity = "1", heading = "Model discovery settings%n") 47 | private ModelDiscoveryConfig modelDiscoveryConfig; 48 | 49 | @Option(names = { "-d", "--directory" }, description = "Adds a directory to list of directories to be scanned", required = true) 50 | private List directories; 51 | 52 | @Option(names = { "-x", "--exclude-extension" }, description = "Adds an extension to exclude from data discovery") 53 | private List excludeExtensions; 54 | 55 | @ParentCommand 56 | private Discover discover; 57 | 58 | @Override 59 | public Integer call() throws Exception { 60 | System.out.println(""); 61 | System.out.println("Starting file discovery"); 62 | log.warn("Discovery writes personal data to log files."); 63 | 64 | log.info("Probability threshold: {}", modelDiscoveryConfig.getProbabilityThreshold()); 65 | log.info("Calculate score: {}", (modelDiscoveryConfig.getCalculateScore()) ? "yes" : "no"); 66 | log.info("Threshold count: {}", modelDiscoveryConfig.getThresholdCount()); 67 | log.info("Threshold high-risk count: {}", modelDiscoveryConfig.getThresholdHighRisk()); 68 | log.info("Limit: {}", modelDiscoveryConfig.getLimit()); 69 | log.info("Built-in models: {}", StringUtils.join(modelDiscoveryConfig.getModels(), ", ")); 70 | log.info("Custom models: {}", StringUtils.join(modelDiscoveryConfig.getFileModels(), ", ")); 71 | log.info("Custom token model: {}", modelDiscoveryConfig.getTokenModel()); 72 | log.info("Extensions: {}", StringUtils.join(modelDiscoveryConfig.getExtensions(), ", ")); 73 | log.info("Directories: {}", StringUtils.join(directories, ", ")); 74 | log.info("File types not considered for analysis: {}", excludeExtensions); 75 | 76 | FileDiscoverer fd = new FileDiscoverer(modelDiscoveryConfig, directories, excludeExtensions); 77 | fd.discover(); 78 | 79 | return 0; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/Extract.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.extractor.IExtractor; 19 | import com.strider.datadefender.extractor.DataExtractor; 20 | import com.strider.datadefender.database.IDbFactory; 21 | 22 | import java.util.concurrent.Callable; 23 | import java.util.List; 24 | 25 | import org.apache.commons.lang3.StringUtils; 26 | 27 | import picocli.CommandLine.ArgGroup; 28 | import picocli.CommandLine.Command; 29 | import picocli.CommandLine.Model.CommandSpec; 30 | import picocli.CommandLine.ParameterException; 31 | import picocli.CommandLine.Parameters; 32 | import picocli.CommandLine.Spec; 33 | 34 | import lombok.extern.log4j.Log4j2; 35 | 36 | /** 37 | * "extract" picocli subcommand, configures and executes the data extractor. 38 | * 39 | * @author Zaahid Bateson 40 | */ 41 | @Command( 42 | name = "extract", 43 | version = "1.0", 44 | mixinStandardHelpOptions = true, 45 | description = "Run data extraction utility -- generates files out of table " 46 | + "columns with the name 'table_columnName.txt' for each column " 47 | + "requested." 48 | ) 49 | @Log4j2 50 | public class Extract implements Callable { 51 | 52 | @ArgGroup(exclusive = false, multiplicity = "1", heading = "Database connection settings%n") 53 | private DbConfig dbConfig; 54 | 55 | @Spec 56 | private CommandSpec spec; 57 | 58 | private List tableColumns; 59 | 60 | @Parameters(paramLabel = "columns", description = "Generate data for the specified table.columName(s)") 61 | public void setTableColumns(List tableColumns) { 62 | for (String tableColumn : tableColumns) { 63 | int loc = tableColumn.indexOf("."); 64 | if (loc < 1 || loc >= tableColumn.length() - 1) { 65 | throw new ParameterException( 66 | spec.commandLine(), 67 | String.format( 68 | "Columns must be specified in the form [table].[columnName], found: %s", 69 | tableColumn 70 | ) 71 | ); 72 | } 73 | } 74 | this.tableColumns = tableColumns; 75 | } 76 | 77 | @Override 78 | public Integer call() throws Exception { 79 | System.out.println(""); 80 | System.out.println("Starting data extractor"); 81 | log.info("Datasource URL: {}, vendor: {}, schema: {}", dbConfig.getUrl(), dbConfig.getVendor(), dbConfig.getSchema()); 82 | log.info("Username: {}, Password provided: {}", dbConfig.getUsername(), (StringUtils.isNotBlank(dbConfig.getPassword()) ? "yes" : "no")); 83 | log.info("Extracting data from: {}", tableColumns); 84 | 85 | IDbFactory factory = IDbFactory.get(dbConfig); 86 | 87 | final IExtractor extractor = new DataExtractor( 88 | factory, 89 | dbConfig, 90 | tableColumns 91 | ); 92 | try { 93 | extractor.extract(); 94 | } catch (DataDefenderException e) { 95 | log.error(e.getMessage()); 96 | log.debug("Exception occurred during data extraction", e); 97 | return 1; 98 | } catch (Throwable e) { 99 | log.error(e.getMessage(), e); 100 | return 1; 101 | } 102 | return 0; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/IRequirementCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 19 | import java.util.List; 20 | 21 | /** 22 | * 23 | * @author Zaahid Bateson 24 | */ 25 | public interface IRequirementCommand { 26 | List getColumnMetaData(); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/ModelDiscoveryConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2020, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.discoverer.Discoverer; 19 | import java.io.File; 20 | import java.util.List; 21 | import java.util.Optional; 22 | 23 | import picocli.CommandLine.Option; 24 | 25 | import lombok.Getter; 26 | import lombok.extern.log4j.Log4j2; 27 | 28 | /** 29 | * Database configuration options for picocli. 30 | * 31 | * @author Zaahid Bateson 32 | */ 33 | @Getter 34 | @Log4j2 35 | public class ModelDiscoveryConfig { 36 | 37 | @Option(names = { "-l", "--limit" }, description = "Limit discovery to a set number of rows in a table", defaultValue = "1000") 38 | private Integer limit; 39 | 40 | private List models; 41 | 42 | @Option(names = { "-e", "--extension" }, description = "Adds a call to an extension method " 43 | + "(e.g. com.strider.datadefender.specialcase.SinDetector.detectSin)") 44 | private List extensions; 45 | 46 | @Option(names = { "--model-file" }, description = "Adds a custom made opennlp TokenizerME file for data discovery.") 47 | private List fileModels; 48 | 49 | @Option(names = { "--token-model" }, description = "Override the default built-in token model (English tokens, " 50 | + "en-token.bin) with a custom token file for use by opennlp's TokenizerModel") 51 | private File tokenModel; 52 | 53 | @Option(names = { "--probability-threshold" }, description = "Minimum NLP match score to return results for", defaultValue = "0.55") 54 | private Double probabilityThreshold; 55 | 56 | @Option(names = { "--no-score-calculation" }, description = "If set, includes a column score", negatable = true) 57 | private Boolean calculateScore = true; 58 | 59 | @Option(names = { "--threshold-count" }, description = "Reports if number of rows found are greater than the defined threshold", defaultValue = "6") 60 | private Integer thresholdCount; 61 | 62 | @Option(names = { "--threshold-high" }, description = "Reports if number of high risk columns found are greater than the defined threshold", defaultValue = "3") 63 | private Integer thresholdHighRisk; 64 | 65 | @Option(names = { "-m", "--model" }, description = "Adds a built-in configured opennlp TokenizerME model for data discovery. " 66 | + "Available models are: ${AVAILABLE-MODELS}") 67 | public void setModels(List models) { 68 | Optional unmatched = models.stream().filter((m) -> !Discoverer.BUILT_IN_MODELS.containsKey(m)).findFirst(); 69 | if (unmatched.isPresent()) { 70 | log.error( 71 | "A built-in model with the name \"{}\" does not exist. Please specify one of: {}", 72 | unmatched.get(), 73 | System.getProperty("AVAILABLE-MODELS") 74 | ); 75 | throw new IllegalArgumentException("Unmatched built-in model."); 76 | } 77 | this.models = models; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/TestRequirement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender; 17 | 18 | import com.strider.datadefender.anonymizer.DatabaseAnonymizer; 19 | import com.strider.datadefender.anonymizer.IAnonymizer; 20 | import com.strider.datadefender.database.IDbFactory; 21 | import com.strider.datadefender.requirement.Requirement; 22 | import com.strider.datadefender.requirement.registry.ClassAndFunctionRegistry; 23 | 24 | import java.util.concurrent.Callable; 25 | import java.util.List; 26 | 27 | import org.apache.commons.collections4.CollectionUtils; 28 | import org.apache.commons.lang3.StringUtils; 29 | 30 | import picocli.CommandLine.ArgGroup; 31 | import picocli.CommandLine.Option; 32 | import picocli.CommandLine.Command; 33 | import picocli.CommandLine.Parameters; 34 | 35 | import lombok.extern.log4j.Log4j2; 36 | 37 | /** 38 | * test-requirement cli subcommand for testing the requirements file. 39 | * 40 | * @author Zaahid Bateson 41 | */ 42 | @Command( 43 | name = "test-requirement", 44 | version = "1.0", 45 | mixinStandardHelpOptions = true, 46 | description = "Loads the requirement file without attempting to anonymize or process anything to check for syntax issues" 47 | ) 48 | @Log4j2 49 | public class TestRequirement implements Callable { 50 | 51 | @Option(names = { "-r", "--requirement-file" }, paramLabel = "", description = "Requirement XML file", required = true) 52 | private Requirement requirement; 53 | 54 | @Override 55 | public Integer call() throws Exception { 56 | System.out.println(""); 57 | System.out.println("Requirements file loaded successfully"); 58 | return 0; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/anonymizer/DatabaseAnonymizerException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.anonymizer; 17 | 18 | /** 19 | * Package-level exception 20 | * @author Armenak Grigoryan 21 | */ 22 | public class DatabaseAnonymizerException extends Exception { 23 | private static final long serialVersionUID = 1L; 24 | 25 | public DatabaseAnonymizerException(final String msg) { 26 | super(msg); 27 | } 28 | 29 | public DatabaseAnonymizerException(final String msg, final Throwable t) { 30 | super(msg, t); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/anonymizer/IAnonymizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.anonymizer; 17 | 18 | import com.strider.datadefender.DataDefenderException; 19 | 20 | /** 21 | * Defines contract for all Anonymizes 22 | * @author strider 23 | */ 24 | public interface IAnonymizer { 25 | 26 | /** 27 | * Anonymizes data. 28 | * 29 | * @throws DataDefenderException 30 | */ 31 | void anonymize() throws DataDefenderException, InstantiationException; 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/anonymizer/functions/Address.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.anonymizer.functions; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.commons.lang3.RandomUtils; 21 | 22 | import lombok.extern.log4j.Log4j2; 23 | 24 | /** 25 | * Built-in anonymization helper functions for address field anonymization. 26 | * 27 | * @author Armenak Grigoryan 28 | */ 29 | @Log4j2 30 | public class Address extends Core { 31 | 32 | public String randomCountry() throws IOException { 33 | return randomStringFromStream( 34 | "resource:countries.txt", 35 | () -> Lipsum.class.getResourceAsStream("countries.txt") 36 | ); 37 | } 38 | 39 | public String randomCity() throws IOException { 40 | return randomStringFromStream( 41 | "resource:cities.txt", 42 | () -> Lipsum.class.getResourceAsStream("cities.txt") 43 | ); 44 | } 45 | 46 | public String randomStreet() throws IOException { 47 | return randomStringFromStream( 48 | "resource:streets.txt", 49 | () -> Lipsum.class.getResourceAsStream("streets.txt") 50 | ); 51 | } 52 | 53 | public String randomProvinceState() throws IOException { 54 | return randomStringFromStream( 55 | "resource:provinces_states.txt", 56 | () -> Lipsum.class.getResourceAsStream("provinces_states.txt") 57 | ); 58 | } 59 | 60 | public String randomProvinceStateCode() throws IOException { 61 | return randomStringFromStream( 62 | "resource:provinces_states_codes.txt", 63 | () -> Lipsum.class.getResourceAsStream("provinces_states_codes.txt") 64 | ); 65 | } 66 | 67 | public String randomCanadianPostalCode() { 68 | return randomStringFromPattern("[A-Z][0-9][A-Z] [0-9][A-Z][0-9]"); 69 | } 70 | 71 | public String randomUsZipCode() { 72 | return randomStringFromPattern("[0-9]{5}"); 73 | } 74 | 75 | public String randomUsZipCodeNineDigit() { 76 | return randomStringFromPattern("[0-9]{5}-[0-9]{4}"); 77 | } 78 | 79 | public String randomCanadianOrUsFiveDigitPostalCode() { 80 | if (RandomUtils.nextBoolean()) { 81 | return randomCanadianPostalCode(); 82 | } 83 | return randomUsZipCode(); 84 | } 85 | 86 | public String randomCanadianOrUsFiveOrNineDigitPostalCode() { 87 | if (RandomUtils.nextBoolean()) { 88 | return randomCanadianPostalCode(); 89 | } else if (RandomUtils.nextBoolean()) { 90 | return randomUsZipCode(); 91 | 92 | } 93 | return randomUsZipCodeNineDigit(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/DatabaseException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | */ 17 | package com.strider.datadefender.database; 18 | 19 | import com.strider.datadefender.DataDefenderException; 20 | 21 | /** 22 | * Package level exception. Extends application level exception. 23 | * @author Armenak Grigoryan 24 | */ 25 | public class DatabaseException extends DataDefenderException { 26 | private static final long serialVersionUID = 1L; 27 | 28 | public DatabaseException(final String msg) { 29 | super(msg); 30 | } 31 | 32 | public DatabaseException(final String msg, final Throwable t) { 33 | super(msg, t); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/DbConnection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database; 17 | 18 | import com.strider.datadefender.DbConfig; 19 | import com.strider.datadefender.utils.ISupplierWithException; 20 | 21 | import java.sql.Connection; 22 | import java.sql.SQLException; 23 | import java.sql.DriverManager; 24 | 25 | import org.apache.commons.lang3.StringUtils; 26 | 27 | import lombok.extern.log4j.Log4j2; 28 | 29 | /** 30 | * Handles standard database connections with username/password provided. 31 | */ 32 | @Log4j2 33 | public class DbConnection implements IDbConnection { 34 | 35 | protected final DbConfig config; 36 | 37 | /** 38 | * Default constructor, initializes config. 39 | * @param config 40 | */ 41 | public DbConnection(final DbConfig config) { 42 | this.config = config; 43 | } 44 | 45 | /** 46 | * Handles the actual creating of the connection, by running the supplier 47 | * method provided by subclasses. 48 | * 49 | * @param supplier 50 | * @return 51 | * @throws DatabaseException 52 | */ 53 | protected Connection doConnect( 54 | final ISupplierWithException supplier 55 | ) throws DatabaseException { 56 | 57 | Connection conn = null; 58 | try { 59 | log.info("Establishing database connection"); 60 | conn = supplier.get(); 61 | conn.setAutoCommit(false); 62 | } catch (SQLException e) { 63 | if (conn != null) { 64 | try { 65 | conn.close(); 66 | } catch (SQLException e2) { 67 | // ignore 68 | } 69 | } 70 | throw new DatabaseException(e.getMessage(), e); 71 | } 72 | return conn; 73 | } 74 | 75 | /** 76 | * Default implementation calls DriverManager.getConnection with the 77 | * provided jdbc url, username and password. 78 | * 79 | * @return Connection 80 | * @throws DatabaseAnonymizerException 81 | */ 82 | @Override 83 | public Connection connect() throws DatabaseException { 84 | return doConnect(() -> DriverManager.getConnection( 85 | config.getUrl(), 86 | StringUtils.trimToNull(config.getUsername()), 87 | StringUtils.trimToNull(config.getPassword()) 88 | )); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/IDbConnection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database; 17 | 18 | import java.sql.Connection; 19 | 20 | /** 21 | * Interface for all classes implementing database connection 22 | * 23 | * @author Armenak Grigoryan 24 | */ 25 | public interface IDbConnection { 26 | Connection connect() throws DatabaseException; 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/MsSqlDbConnection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database; 17 | 18 | import java.sql.Connection; 19 | import static java.sql.DriverManager.getConnection; 20 | 21 | import com.strider.datadefender.DataDefenderException; 22 | import com.strider.datadefender.DbConfig; 23 | import org.apache.commons.lang3.StringUtils; 24 | 25 | /** 26 | * 27 | * @author sdi 28 | */ 29 | public class MsSqlDbConnection extends DbConnection { 30 | 31 | public MsSqlDbConnection(DbConfig config) throws DataDefenderException { 32 | super(config); 33 | } 34 | 35 | /** 36 | * Establishes database connection 37 | * 38 | * @return Connection 39 | * @throws DatabaseException 40 | */ 41 | @Override 42 | public Connection connect() throws DatabaseException { 43 | return doConnect(() -> getConnection(this.getUrl())); 44 | } 45 | 46 | /** 47 | * Get connection URL. 48 | * @return String 49 | */ 50 | protected String getUrl() { 51 | StringBuilder sqlServerUrl = new StringBuilder(config.getUrl()); 52 | if (!StringUtils.isBlank(config.getUsername())) { 53 | sqlServerUrl.append(";user=").append(config.getUsername()); 54 | } 55 | if (!StringUtils.isBlank(config.getPassword())) { 56 | sqlServerUrl.append(";password=").append(config.getPassword()); 57 | } 58 | return sqlServerUrl.toString(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/metadata/IMetaData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database.metadata; 17 | 18 | import java.sql.ResultSet; 19 | import java.sql.SQLException; 20 | import java.util.List; 21 | 22 | /** 23 | * Public interface to define the contract for all database-specific 24 | * metadata classes. 25 | * 26 | * @author armenak 27 | */ 28 | public interface IMetaData { 29 | List getMetaData() throws SQLException; 30 | TableMetaData getMetaDataFor(final ResultSet rs) throws SQLException; 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/metadata/MySqlMetaData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database.metadata; 17 | 18 | import com.strider.datadefender.DbConfig; 19 | import java.sql.Connection; 20 | import java.sql.DatabaseMetaData; 21 | import java.sql.ResultSet; 22 | import java.sql.SQLException; 23 | 24 | /** 25 | * Overridden to omit using schema in some cases, as it's not supported. 26 | * @author armenak 27 | */ 28 | public class MySqlMetaData extends MetaData { 29 | 30 | public MySqlMetaData(DbConfig config, Connection connection) { 31 | super(config, connection); 32 | } 33 | 34 | public MySqlMetaData(DbConfig config, Connection connection, SqlTypeToClass sqlTypeMap) { 35 | super(config, connection, sqlTypeMap); 36 | } 37 | 38 | @Override 39 | protected ResultSet getTableResultSet(final DatabaseMetaData md) throws SQLException { 40 | return md.getTables(null, null, "%", new String[] { "TABLE" }); 41 | } 42 | 43 | @Override 44 | protected ResultSet getColumnResultSet(final DatabaseMetaData md, final String tableName) throws SQLException { 45 | return md.getColumns(null, null, tableName, null); 46 | } 47 | 48 | @Override 49 | protected ResultSet getForeignKeysResultSet(final DatabaseMetaData md, final String tableName) throws SQLException { 50 | return md.getImportedKeys(null, null, tableName); 51 | } 52 | 53 | @Override 54 | protected ResultSet getPrimaryKeysResultSet(final DatabaseMetaData md, final String tableName) throws SQLException { 55 | return md.getPrimaryKeys(null, null, tableName); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/metadata/SqlTypeToClass.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database.metadata; 17 | 18 | import java.math.BigDecimal; 19 | import java.sql.Array; 20 | import java.sql.Blob; 21 | import java.sql.Clob; 22 | import java.sql.Date; 23 | import java.sql.Time; 24 | import java.sql.Timestamp; 25 | import java.sql.Types; 26 | 27 | /** 28 | * Maps java.sql.Types to Class objects. 29 | * 30 | * @author Zaahid Bateson 31 | */ 32 | public class SqlTypeToClass { 33 | public Class getTypeFrom(int type) { 34 | switch (type) { 35 | case Types.CHAR: 36 | case Types.VARCHAR: 37 | case Types.LONGVARCHAR: 38 | case Types.NCHAR: 39 | case Types.NVARCHAR: 40 | case Types.LONGNVARCHAR: 41 | case Types.OTHER: 42 | return String.class; 43 | case Types.BINARY: 44 | case Types.VARBINARY: 45 | case Types.LONGVARBINARY: 46 | return Byte[].class; 47 | case Types.NUMERIC: 48 | case Types.DECIMAL: 49 | return BigDecimal.class; 50 | case Types.BIT: 51 | return Boolean.class; 52 | case Types.TINYINT: 53 | case Types.SMALLINT: 54 | return Short.class; 55 | case Types.INTEGER: 56 | return Integer.class; 57 | case Types.BIGINT: 58 | return Long.class; 59 | case Types.REAL: 60 | return Float.class; 61 | case Types.FLOAT: 62 | case Types.DOUBLE: 63 | return Double.class; 64 | case Types.DATE: 65 | return Date.class; 66 | case Types.TIME: 67 | return Time.class; 68 | case Types.TIMESTAMP: 69 | return Timestamp.class; 70 | case Types.BLOB: 71 | return Blob.class; 72 | case Types.CLOB: 73 | return Clob.class; 74 | case Types.ARRAY: 75 | return Array.class; 76 | case Types.NULL: 77 | return null; 78 | default: 79 | throw new IllegalArgumentException("Unsupported or unknown type"); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/sqlbuilder/ISqlBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database.sqlbuilder; 17 | 18 | import com.strider.datadefender.DbConfig.Vendor; 19 | 20 | /** 21 | * Interface for all classes implementing SQL builder. 22 | * 23 | * @author Armenak Grigoryan 24 | */ 25 | public interface ISqlBuilder { 26 | 27 | /** 28 | * Add a limit to returned rows for the passed sqlString. 29 | * 30 | * @param sqlString 31 | * @param limit 32 | * @return 33 | */ 34 | String buildSelectWithLimit(String sqlString, int limit); 35 | 36 | /** 37 | * Prefix table name with schema if present. 38 | * 39 | * @param tableName 40 | * @return 41 | */ 42 | String prefixSchema(String tableName); 43 | 44 | /** 45 | * Method to get DB Vendor 46 | * @return 47 | */ 48 | Vendor getVendor(); 49 | } 50 | 51 | 52 | //~ Formatted by Jindent --- http://www.jindent.com 53 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/sqlbuilder/MsSqlBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database.sqlbuilder; 17 | 18 | import com.strider.datadefender.DbConfig; 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | import lombok.extern.log4j.Log4j2; 22 | 23 | /** 24 | * MS SQL Server implementation of the ISqlBuilder 25 | * 26 | * @author Armenak Grigoryan 27 | * @author Luis Marques 28 | */ 29 | @Log4j2 30 | public class MsSqlBuilder extends SqlBuilder { 31 | 32 | public MsSqlBuilder(DbConfig config) { 33 | super(config); 34 | } 35 | 36 | /** 37 | * Uses TOP to limit the passed query. 38 | * 39 | * @param sqlString 40 | * @param limit 41 | * @return 42 | */ 43 | @Override 44 | public String buildSelectWithLimit(final String sqlString, final int limit) { 45 | String query = sqlString.replaceFirst("(?i)(SELECT)(\\s+)", "$1 TOP " + limit + " $2"); 46 | log.debug("Query after adding limit: [{}]", query); 47 | return query; 48 | } 49 | 50 | /** 51 | * Prefixes the schema name, followed by a "." character if set. 52 | * Additionally surrounds schema name (if set) and tableName with square 53 | * brackets, so the end result is either "[tableName]" or 54 | * "[schemaName].[tableName]". 55 | * 56 | * @param tableName 57 | * @return 58 | */ 59 | @Override 60 | public String prefixSchema(final String tableName) { 61 | final String schema = config.getSchema(); 62 | if (StringUtils.isNotBlank(schema)) { 63 | return "[" + schema + "].[" + tableName + "]"; 64 | } 65 | return "[" + tableName + "]"; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/sqlbuilder/OracleSqlBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | */ 17 | package com.strider.datadefender.database.sqlbuilder; 18 | 19 | import com.strider.datadefender.DbConfig; 20 | import org.apache.commons.lang3.StringUtils; 21 | 22 | import lombok.extern.log4j.Log4j2; 23 | 24 | /** 25 | * Oracle implementation of the ISqlBuilder 26 | * 27 | * @author Armenak Grigoryan 28 | */ 29 | @Log4j2 30 | public class OracleSqlBuilder extends SqlBuilder { 31 | 32 | public OracleSqlBuilder(DbConfig config) { 33 | super(config); 34 | } 35 | 36 | /** 37 | * Uses ROWNUM to limit the passed query. 38 | * 39 | * @param sqlString 40 | * @param limit 41 | * @return 42 | */ 43 | @Override 44 | public String buildSelectWithLimit(final String sqlString, final int limit) { 45 | String sql = "SELECT q.* FROM (" + StringUtils.stripEnd(sqlString.trim(), ";") + ") q WHERE ROWNUM <= " + limit; 46 | log.debug("Query after adding limit: [{}]", sql); 47 | return sql; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/database/sqlbuilder/SqlBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.database.sqlbuilder; 17 | 18 | import com.strider.datadefender.DbConfig; 19 | import com.strider.datadefender.DbConfig.Vendor; 20 | 21 | import org.apache.commons.lang3.StringUtils; 22 | 23 | import lombok.extern.log4j.Log4j2; 24 | import lombok.RequiredArgsConstructor; 25 | 26 | /** 27 | * Provides 'default' implementation which can be overridden. 28 | * @author Akira Matsuo 29 | */ 30 | @Log4j2 31 | @RequiredArgsConstructor 32 | public class SqlBuilder implements ISqlBuilder { 33 | 34 | protected final DbConfig config; 35 | 36 | /** 37 | * Appends "LIMIT {n}" to the end of the query. 38 | * 39 | * @param sqlString 40 | * @param limit 41 | * @return 42 | */ 43 | @Override 44 | public String buildSelectWithLimit(final String sqlString, final int limit) { 45 | final StringBuilder sql = new StringBuilder(sqlString); 46 | if (limit != 0) { 47 | sql.append(" LIMIT ").append(limit); 48 | } 49 | log.debug("Query after adding limit: [{}]", sql); 50 | return sql.toString(); 51 | } 52 | 53 | /** 54 | * Prepends the schema name followed by a single "." to the passed tableName 55 | * if a schema name is configured. 56 | * 57 | * @param tableName 58 | * @return 59 | */ 60 | @Override 61 | public String prefixSchema(final String tableName) { 62 | final String schema = config.getSchema(); 63 | if (StringUtils.isNotBlank(schema)) { 64 | if(config.getVendor() == Vendor.POSTGRESQL) { 65 | return schema + "." + "\"" + tableName + "\""; 66 | }else { 67 | return schema + "." + tableName; 68 | } 69 | } 70 | return tableName; 71 | } 72 | 73 | /** 74 | * Method to get DB Vendor 75 | * @return 76 | */ 77 | @Override 78 | public Vendor getVendor() { 79 | return config.getVendor(); 80 | } 81 | } -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/discoverer/ColumnDiscoverer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014-1018, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | */ 17 | package com.strider.datadefender.discoverer; 18 | 19 | import com.strider.datadefender.DataDefenderException; 20 | import com.strider.datadefender.database.metadata.IMetaData; 21 | import com.strider.datadefender.database.metadata.TableMetaData; 22 | import com.strider.datadefender.report.ReportUtil; 23 | import com.strider.datadefender.utils.Score; 24 | import com.strider.datadefender.database.IDbFactory; 25 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 26 | 27 | import java.io.IOException; 28 | import java.sql.SQLException; 29 | import java.util.ArrayList; 30 | import java.util.LinkedHashSet; 31 | import java.util.List; 32 | import java.util.regex.Pattern; 33 | import java.util.stream.Collectors; 34 | 35 | import org.apache.commons.collections.CollectionUtils; 36 | import org.apache.commons.lang3.StringUtils; 37 | 38 | import lombok.extern.log4j.Log4j2; 39 | 40 | /** 41 | * @author Armenak Grigoryan 42 | */ 43 | @Log4j2 44 | public class ColumnDiscoverer { 45 | 46 | private IDbFactory factory; 47 | private List patterns; 48 | 49 | public ColumnDiscoverer(IDbFactory factory, List patterns) { 50 | this.factory = factory; 51 | this.patterns = patterns; 52 | } 53 | 54 | public List discover() throws DataDefenderException, IOException, SQLException { 55 | log.info("Column discovery in process"); 56 | 57 | final IMetaData metaData = factory.fetchMetaData(); 58 | final List list = metaData.getMetaData(); 59 | 60 | List columns = list.stream().flatMap((t) -> t.getColumns().stream()) 61 | .filter((c) -> patterns.stream().anyMatch((p) -> p.matcher(c.getColumnName().toLowerCase()).matches())) 62 | .collect(Collectors.toList()); 63 | 64 | log.info("Preparing report ..."); 65 | 66 | // Report column names 67 | List uniqueMatches = null; 68 | if (CollectionUtils.isNotEmpty(columns)) { 69 | uniqueMatches = new ArrayList<>(new LinkedHashSet<>(columns)); 70 | log.info("-----------------"); 71 | log.info("List of suspects:"); 72 | log.info("-----------------"); 73 | uniqueMatches.sort((a, b) -> a.compareTo(b)); 74 | 75 | final Score score = new Score(); 76 | 77 | for (final ColumnMetaData entry : uniqueMatches) { 78 | 79 | // Row count 80 | final int rowCount = ReportUtil.rowCount(factory, entry.getTable().getTableName()); 81 | 82 | // Getting 5 sample values 83 | final List sampleDataList = ReportUtil.sampleData(factory, entry); 84 | 85 | // Output 86 | log.info("Column : " + entry); 87 | log.info(StringUtils.repeat('=', entry.toString().length() + 30)); 88 | log.info("Number of rows in the table: " + rowCount); 89 | log.info("Score : " + score.columnScore(rowCount)); 90 | log.info("Sample data"); 91 | log.info(StringUtils.repeat('-', 11)); 92 | 93 | for (final String sampleData : sampleDataList) { 94 | log.info(sampleData); 95 | } 96 | 97 | log.info(""); 98 | } 99 | 100 | log.info("Overall score: " + score.dataStoreScore()); 101 | } else { 102 | log.info("No suspects have been found. Please refine your criteria."); 103 | } 104 | 105 | return uniqueMatches; 106 | } 107 | } -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/discoverer/Discoverer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | */ 17 | package com.strider.datadefender.discoverer; 18 | 19 | import com.strider.datadefender.DataDefenderException; 20 | import com.strider.datadefender.ModelDiscoveryConfig; 21 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 22 | import com.strider.datadefender.requirement.file.Generator; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | import java.io.InputStream; 27 | import java.util.List; 28 | import java.util.Map; 29 | import java.util.stream.Collectors; 30 | 31 | import javax.xml.bind.JAXBException; 32 | 33 | import opennlp.tools.namefind.NameFinderME; 34 | import opennlp.tools.namefind.TokenNameFinderModel; 35 | import opennlp.tools.tokenize.TokenizerME; 36 | import opennlp.tools.tokenize.TokenizerModel; 37 | 38 | import lombok.Data; 39 | import lombok.extern.log4j.Log4j2; 40 | 41 | /** 42 | * Holds common logic for Discoverers. 43 | * @author Akira Matsuo 44 | */ 45 | @Log4j2 46 | public abstract class Discoverer { 47 | 48 | public static final String DEFAULT_TOKEN_MODEL = "en-token.bin"; 49 | public static final Map BUILT_IN_MODELS = Map.of( 50 | "date", "en-ner-date.bin", 51 | "location", "en-ner-location.bin", 52 | "money", "en-ner-money.bin", 53 | "organization", "en-ner-organization.bin", 54 | "person", "en-ner-person.bin", 55 | "time", "en-ner-time.bin" 56 | ); 57 | 58 | @Data 59 | public static class ColumnMatch { 60 | final private ColumnMetaData column; 61 | final private double averageProbability; 62 | final private String model; 63 | final private List probabilityList; 64 | } 65 | 66 | protected List matches; 67 | 68 | protected final ModelDiscoveryConfig config; 69 | protected final TokenizerME tokenizer; 70 | 71 | public Discoverer(ModelDiscoveryConfig config) throws IOException { 72 | this.config = config; 73 | if (config.getTokenModel() != null) { 74 | tokenizer = new TokenizerME(new TokenizerModel(config.getTokenModel())); 75 | } else { 76 | try (InputStream stream = Discoverer.class.getResourceAsStream(DEFAULT_TOKEN_MODEL)) { 77 | tokenizer = new TokenizerME(new TokenizerModel(stream)); 78 | } 79 | } 80 | } 81 | 82 | public double calculateAverage(final List values) { 83 | Double sum = 0.0; 84 | 85 | if (!values.isEmpty()) { 86 | for (final Probability value : values) { 87 | sum += value.getProbabilityValue(); 88 | } 89 | 90 | return sum / values.size(); 91 | } 92 | 93 | return sum; 94 | } 95 | 96 | private Model createModelFrom(TokenNameFinderModel tnf, String modelName) { 97 | NameFinderME nameFinder = new NameFinderME(tnf); 98 | return new Model(tokenizer, nameFinder, modelName); 99 | } 100 | 101 | /** 102 | * Creates model POJO based on OpenNLP model file 103 | * 104 | * @param modelName 105 | * @return Model 106 | */ 107 | public Model createModel(final File modelFile) throws IOException { 108 | return createModelFrom(new TokenNameFinderModel(modelFile), modelFile.getName()); 109 | } 110 | 111 | /** 112 | * Creates model POJO based on a built-in OpenNLP model 113 | * 114 | * @param modelName 115 | * @return Model 116 | */ 117 | public Model createModel(final String modelName) throws IOException { 118 | try (InputStream stream = Discoverer.class.getResourceAsStream(BUILT_IN_MODELS.get(modelName))) { 119 | return createModelFrom(new TokenNameFinderModel(stream), modelName); 120 | } 121 | } 122 | } -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/discoverer/FileDiscoveryException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.discoverer; 22 | 23 | /** 24 | * Application-level exception 25 | * @author Armenak Grigoryan 26 | */ 27 | public class FileDiscoveryException extends Exception { 28 | private static final long serialVersionUID = 1L; 29 | 30 | public FileDiscoveryException(final String msg) { 31 | super(msg); 32 | } 33 | 34 | public FileDiscoveryException(final String msg, final Throwable t) { 35 | super(msg, t); 36 | } 37 | } 38 | 39 | 40 | //~ Formatted by Jindent --- http://www.jindent.com 41 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/discoverer/IDiscoverer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.discoverer; 22 | 23 | import com.strider.datadefender.anonymizer.DatabaseAnonymizerException; 24 | 25 | /** 26 | * Defines contract for all discoverers 27 | * @author Armenak Grigoryan 28 | */ 29 | public interface IDiscoverer { 30 | 31 | /** 32 | * Discovers data or data containers with data which can be be the subject 33 | * for data anonymization. Results found should be stored in matches member variable. 34 | * @param dbFactory 35 | * @param properties 36 | * @param tables Optional list of tables to anonymize - 37 | * if the collection is empty, all tables specified in requirements 38 | * are anonymized. 39 | * @throws com.strider.datadefender.AnonymizerException 40 | * @returns List of results introduced for testing purposes. 41 | */ 42 | 43 | // List discover(IDBFactory dbFactory, Properties dataDiscoveryProperties, Set tables) throws DatabaseAnonymizerException; 44 | // List discover(Properties dataDiscoveryProperties) throws DatabaseAnonymizerException; 45 | 46 | /** 47 | * Creates a sample requirement file from a list of sorted matches found by the discovery method. 48 | * The requirement file will most likely require customization (for example; anonymzation function) before you want to run the Anonymizer against it. 49 | * @param fileName 50 | * @throws DatabaseAnonymizerException 51 | */ 52 | void createRequirement(String fileName) throws DatabaseAnonymizerException; 53 | } 54 | 55 | 56 | //~ Formatted by Jindent --- http://www.jindent.com 57 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/discoverer/Model.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014-2015, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.discoverer; 22 | 23 | import opennlp.tools.namefind.NameFinderME; 24 | import opennlp.tools.tokenize.Tokenizer; 25 | 26 | public class Model { 27 | final private Tokenizer tokenizer; 28 | final private NameFinderME nameFinder; 29 | final private String name; 30 | 31 | public Model(final Tokenizer tokenizer, final NameFinderME nameFinder, final String name) { 32 | this.name = name; 33 | this.tokenizer = tokenizer; 34 | this.nameFinder = nameFinder; 35 | } 36 | 37 | public String getName() { 38 | return this.name; 39 | } 40 | 41 | public NameFinderME getNameFinder() { 42 | return this.nameFinder; 43 | } 44 | 45 | public Tokenizer getTokenizer() { 46 | return this.tokenizer; 47 | } 48 | } 49 | 50 | 51 | //~ Formatted by Jindent --- http://www.jindent.com 52 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/discoverer/Probability.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014-2020, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | package com.strider.datadefender.discoverer; 20 | 21 | import java.util.Comparator; 22 | 23 | /** 24 | * This class represents the probability of sentence in database column 25 | * 26 | * @author Armenak Grigoryan 27 | */ 28 | public class Probability { 29 | private final String sentence; 30 | private final Double probabilityValue; 31 | 32 | public Probability(final String sentence, final Double probabilityValue) { 33 | this.sentence = sentence; 34 | this.probabilityValue = probabilityValue; 35 | } 36 | 37 | public Double getProbabilityValue() { 38 | return this.probabilityValue; 39 | } 40 | 41 | public String getSentence() { 42 | return this.sentence; 43 | } 44 | 45 | /** 46 | * @return comparator used for sorting 47 | */ 48 | 49 | public static Comparator compare() { 50 | return Comparator.comparing(Probability::getSentence) 51 | .thenComparing(Probability::getProbabilityValue); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/extensions/BiographicFunctions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2016, Armenak Grigoryan, Matthew Eaton, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.extensions; 17 | 18 | import com.strider.datadefender.anonymizer.functions.Core; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import lombok.extern.log4j.Log4j2; 24 | import org.apache.commons.lang3.StringUtils; 25 | 26 | /** 27 | * @author Matthew Eaton 28 | */ 29 | @Log4j2 30 | public class BiographicFunctions extends Core { 31 | 32 | /** 33 | * Algorithm is taken from https://en.wikipedia.org/wiki/Social_Insurance_Number 34 | * @param sin 35 | * @return boolean true, if SIN is valid, otherwise false 36 | */ 37 | public static boolean isValidSIN(final String sin) { 38 | boolean valid = false; 39 | 40 | if (sin.length() < 9) { 41 | log.debug("SIN length is < 9"); 42 | 43 | return valid; 44 | } 45 | 46 | if (!StringUtils.isNumeric(sin)) { 47 | log.debug("SIN " + sin + " is not number"); 48 | 49 | return valid; 50 | } 51 | 52 | final int[] sinArray = new int[sin.length()]; 53 | final int[] checkArray = { 54 | 1, 2, 1, 2, 1, 2, 1, 2, 1 55 | }; 56 | final List sinList = new ArrayList(); 57 | 58 | for (int i = 0; i < 9; i++) { 59 | sinArray[i] = Integer.valueOf(sin.substring(i, i + 1)); 60 | sinArray[i] = sinArray[i] * checkArray[i]; 61 | } 62 | 63 | int sum = 0; 64 | 65 | for (int i = 0; i < 9; i++) { 66 | final String tmp = String.valueOf(sinArray[i]); 67 | 68 | if (tmp.length() == 1) { 69 | sinList.add(Integer.valueOf(tmp)); 70 | sum += Integer.valueOf(tmp); 71 | } else { 72 | sinList.add(Integer.valueOf(tmp.substring(0, 1))); 73 | sum += Integer.valueOf(tmp.substring(0, 1)); 74 | sinList.add(Integer.valueOf(tmp.substring(1, 2))); 75 | sum += Integer.valueOf(tmp.substring(1, 2)); 76 | } 77 | } 78 | 79 | if ((sum % 10) == 0) { 80 | valid = true; 81 | } 82 | 83 | return valid; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/extensions/ExtensionExample.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.extensions; 17 | 18 | import com.strider.datadefender.anonymizer.functions.Core; 19 | 20 | import static java.lang.Math.random; 21 | import static java.lang.Math.round; 22 | import static java.lang.String.valueOf; 23 | 24 | /** 25 | * @author Armenak Grigoryan 26 | */ 27 | public class ExtensionExample extends Core { 28 | 29 | /** 30 | * Generates random 9-digit student number 31 | * @return String 32 | */ 33 | public String randomStudentNumber() { 34 | return valueOf(round(random() * 100000000)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/extractor/IExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014, Armenak Grigoryan, Matthew Eaton, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | package com.strider.datadefender.extractor; 19 | 20 | import com.strider.datadefender.database.DatabaseException; 21 | 22 | /** 23 | * Defines contract for all generators 24 | * @author Matthew Eaton 25 | */ 26 | public interface IExtractor { 27 | 28 | /** 29 | * Extracts data from table columns and generates files 30 | * 31 | * @throws DatabaseException 32 | */ 33 | void extract() throws DatabaseException; 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/file/metadata/FileMatchMetaData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.file.metadata; 17 | 18 | /** 19 | * Data object to hold all metadata for matching data in Discovery applications. 20 | * Currently this object holds column and table metadata; ie, duplicating info. 21 | * But since the numbers of tables/columns will always be 22 | * limited in size, don't really see an immediate need to refactor this. 23 | * @author Armenak Grigoryan 24 | */ 25 | public class FileMatchMetaData { 26 | private String model = ""; 27 | private final String directory; 28 | private final String fileName; 29 | private double averageProbability; 30 | 31 | public FileMatchMetaData(final String directory, final String fileName) { 32 | this.directory = directory; 33 | this.fileName = fileName; 34 | } 35 | 36 | public double getAverageProbability() { 37 | return this.averageProbability; 38 | } 39 | 40 | public void setAverageProbability(final double averageProbability) { 41 | this.averageProbability = averageProbability; 42 | } 43 | 44 | public String getDirectory() { 45 | return this.directory; 46 | } 47 | 48 | public String getFileName() { 49 | return this.fileName; 50 | } 51 | 52 | public String getModel() { 53 | return this.model; 54 | } 55 | 56 | public void setModel(final String model) { 57 | this.model = model; 58 | } 59 | 60 | @Override 61 | public String toString() { 62 | return this.directory + "." + this.fileName; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/functions/NamedParameter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.functions; 17 | 18 | import java.lang.annotation.Documented; 19 | import java.lang.annotation.ElementType; 20 | import java.lang.annotation.Target; 21 | 22 | /** 23 | * Declares a parameter's name for reflection. 24 | * 25 | * Provides an alternative to compiling with -parameters so a method's 26 | * parameters can be named and referenced by name. 27 | * 28 | * @author Zaahid Bateson 29 | */ 30 | @Documented 31 | @Target({ ElementType.PARAMETER }) 32 | public @interface NamedParameter { 33 | /** 34 | * The name of the parameter that should be used to reference it in 35 | * external configuration. 36 | * @return 37 | */ 38 | public String value(); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/functions/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.functions; 17 | 18 | import lombok.extern.log4j.Log4j2; 19 | 20 | /** 21 | * 22 | * @author Armenak Grigoryan 23 | */ 24 | @Log4j2 25 | public class Utils { 26 | 27 | /** 28 | * The string used to separator package and class name 29 | */ 30 | public static final String SEPARATOR = "."; 31 | 32 | /** 33 | * Returns fully specified class name. 34 | * 35 | * @param fullMethodName Fully specified method name. 36 | * @return Method name. 37 | * 38 | * @throws RuntimeException Parameter is an empty string. 39 | */ 40 | public static String getClassName(final String fullMethodName) { 41 | if (fullMethodName.length() == 0) { 42 | log.error("Please specify fully specified methid name in Requirement document"); 43 | 44 | return ""; 45 | } 46 | 47 | final int index = fullMethodName.lastIndexOf(SEPARATOR); 48 | 49 | if (index != -1) { 50 | return fullMethodName.substring(0, index); 51 | } 52 | 53 | return ""; 54 | } 55 | 56 | /** 57 | * Returns the method name. 58 | * 59 | * @param fullMethodName Fully specified method name. 60 | * @return Method name. 61 | * @throws com.strider.datadefender.DatabaseAnonymizerException 62 | */ 63 | public static String getMethodName(final String fullMethodName) { 64 | if (fullMethodName.length() == 0) { 65 | log.error("Please specify fully specified methid name in Requirement document"); 66 | 67 | return ""; 68 | } 69 | 70 | final int index = fullMethodName.lastIndexOf(SEPARATOR); 71 | 72 | if (index != -1) { 73 | return fullMethodName.substring(index + 1); 74 | } 75 | 76 | return ""; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/ClassAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement; 17 | 18 | import com.strider.datadefender.requirement.registry.ClassAndFunctionRegistry; 19 | import javax.xml.bind.annotation.adapters.XmlAdapter; 20 | 21 | import org.apache.commons.lang3.ClassUtils; 22 | import org.apache.commons.lang3.StringUtils; 23 | 24 | /** 25 | * XmlAdapter to get a Class type from a String, and vice-versa. 26 | * 27 | * @author Zaahid Bateson 28 | */ 29 | public class ClassAdapter extends XmlAdapter> { 30 | 31 | private ClassAndFunctionRegistry registry; 32 | 33 | public ClassAdapter() { 34 | this(ClassAndFunctionRegistry.singleton()); 35 | } 36 | 37 | public ClassAdapter(ClassAndFunctionRegistry registry) { 38 | this.registry = registry; 39 | } 40 | 41 | @Override 42 | public Class unmarshal(String value) throws Exception { 43 | String t = StringUtils.trimToEmpty(value); 44 | if (StringUtils.isBlank(t)) { 45 | return String.class; 46 | } 47 | return registry.getClassForName(value); 48 | } 49 | 50 | @Override 51 | public String marshal(Class bt) throws Exception { 52 | return registry.getNameForClass(bt); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/Column.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement; 17 | 18 | import com.strider.datadefender.requirement.plan.Plan; 19 | import com.strider.datadefender.requirement.plan.PlanRef; 20 | import java.lang.reflect.InvocationTargetException; 21 | import java.sql.ResultSet; 22 | import java.sql.SQLException; 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | import javax.xml.bind.annotation.XmlAccessType; 27 | import javax.xml.bind.annotation.XmlAccessorType; 28 | import javax.xml.bind.annotation.XmlAttribute; 29 | import javax.xml.bind.annotation.XmlElement; 30 | import javax.xml.bind.annotation.XmlElementWrapper; 31 | import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; 32 | 33 | import org.apache.commons.collections4.CollectionUtils; 34 | import org.apache.commons.lang3.ClassUtils; 35 | 36 | import lombok.Data; 37 | import lombok.extern.log4j.Log4j2; 38 | 39 | import static java.util.Collections.unmodifiableList; 40 | import org.apache.commons.beanutils.ConvertUtils; 41 | 42 | /** 43 | * JAXB class that defines column elements in Requirement.xml file 44 | * 45 | * @author Armenak Grigoryan 46 | */ 47 | @Log4j2 48 | @Data 49 | @XmlAccessorType(XmlAccessType.NONE) 50 | public class Column { 51 | 52 | @XmlAttribute 53 | private String name; 54 | 55 | @XmlAttribute(name = "skip-empty") 56 | private boolean ignoreEmpty = true; 57 | 58 | @XmlJavaTypeAdapter(ClassAdapter.class) 59 | @XmlAttribute 60 | private Class type = String.class; 61 | 62 | @XmlElement 63 | private Plan plan; 64 | 65 | @XmlElement(name = "plan-ref") 66 | private PlanRef planRef; 67 | 68 | @XmlElementWrapper(name = "exclusions") 69 | @XmlElement(name = "exclude") 70 | private List exclusions; 71 | 72 | public Column() { 73 | } 74 | 75 | public Column(String name) { 76 | this.name = name; 77 | } 78 | 79 | /** 80 | * Returns a list of exclusions 81 | * 82 | * @return List 83 | */ 84 | public List getExclusions() { 85 | final List lst = new ArrayList<>(); 86 | if (!CollectionUtils.isEmpty(exclusions)) { 87 | lst.addAll(exclusions); 88 | } 89 | if (ignoreEmpty) { 90 | lst.add(new Exclude(true)); 91 | } 92 | return unmodifiableList(lst); 93 | } 94 | 95 | /** 96 | * Returns the plan used by this column, either the one defined under it, 97 | * or the one referenced by planRef. 98 | * 99 | * @return 100 | */ 101 | public Plan getResolvedPlan() { 102 | if (planRef != null) { 103 | return planRef.getRef(); 104 | } 105 | return plan; 106 | } 107 | 108 | /** 109 | * Calls all functions defined under Functions in order. 110 | * 111 | * @param rs ResultSet 112 | * @return 113 | * @throws SQLException 114 | * @throws IllegalAccessException 115 | * @throws InvocationTargetException 116 | * @throws java.lang.InstantiationException 117 | */ 118 | public Object invokeFunctionChain(ResultSet rs) 119 | throws SQLException, 120 | IllegalAccessException, 121 | InvocationTargetException, 122 | InstantiationException { 123 | 124 | Object startingValue = null; 125 | Class argType = getResolvedPlan().getDynamicArgumentType(); 126 | if (argType != null && ClassUtils.isAssignable(ResultSet.class, argType)) { 127 | startingValue = rs; 128 | } else { 129 | startingValue = rs.getObject(name, type); 130 | } 131 | return ConvertUtils.convert(getResolvedPlan().invoke(startingValue), type); 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/Requirement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | */ 17 | package com.strider.datadefender.requirement; 18 | 19 | import com.strider.datadefender.requirement.plan.GlobalPlan; 20 | import com.strider.datadefender.requirement.registry.ClassAndFunctionRegistry; 21 | 22 | import java.util.List; 23 | import java.util.stream.Collectors; 24 | import javax.xml.bind.Unmarshaller; 25 | 26 | import javax.xml.bind.annotation.XmlAccessType; 27 | import javax.xml.bind.annotation.XmlAccessorType; 28 | import javax.xml.bind.annotation.XmlAttribute; 29 | import javax.xml.bind.annotation.XmlElement; 30 | import javax.xml.bind.annotation.XmlElementWrapper; 31 | import javax.xml.bind.annotation.XmlRootElement; 32 | 33 | import org.apache.commons.collections4.CollectionUtils; 34 | import org.apache.commons.lang3.StringUtils; 35 | 36 | import lombok.AccessLevel; 37 | import lombok.Data; 38 | import lombok.Getter; 39 | import lombok.Setter; 40 | import lombok.extern.log4j.Log4j2; 41 | 42 | /** 43 | * JAXB class that defines elements in Requirement.xml file 44 | * 45 | * @author Armenak Grigoryan 46 | */ 47 | @Data 48 | @Log4j2 49 | @XmlAccessorType(XmlAccessType.NONE) 50 | @XmlRootElement(name = "anonymizer") 51 | public class Requirement { 52 | 53 | @XmlAccessorType(XmlAccessType.NONE) 54 | @Data 55 | public static class AutoresolvePackage { 56 | @XmlAttribute 57 | private String name; 58 | 59 | @Getter(AccessLevel.NONE) 60 | @Setter(AccessLevel.NONE) 61 | private ClassAndFunctionRegistry registry; 62 | 63 | public AutoresolvePackage() { 64 | this(ClassAndFunctionRegistry.singleton()); 65 | } 66 | 67 | public AutoresolvePackage(String name) { 68 | this(); 69 | this.name = name; 70 | } 71 | 72 | public AutoresolvePackage(String name, boolean register) { 73 | this(name); 74 | if (register) { 75 | registry.registerAutoResolvePackage(name); 76 | } 77 | } 78 | 79 | public AutoresolvePackage(ClassAndFunctionRegistry registry) { 80 | this.registry = registry; 81 | } 82 | 83 | public void afterUnmarshal(Unmarshaller unmarshaller, Object parent) { 84 | log.debug("Found autoresolve package: {}", name); 85 | registry.registerAutoResolvePackage(name); 86 | } 87 | } 88 | 89 | @XmlElement(name = "anonymizer-version") 90 | private double anonymizerVersion = 2.0d; 91 | 92 | @XmlElement 93 | private String project; 94 | 95 | @XmlElement(name = "project-version") 96 | private String version; 97 | 98 | @XmlElementWrapper(name = "autoresolve-classes") 99 | @XmlElement(name = "package") 100 | private List autoresolve; 101 | 102 | @XmlElementWrapper(name = "column-plans") 103 | @XmlElement(name = "plan") 104 | private List plans; 105 | 106 | @XmlElementWrapper(name = "tables") 107 | @XmlElement(name = "table") 108 | private List tables; 109 | 110 | /** 111 | * Returns a list of Table elements that match entries in the passed filter. 112 | * Attempts to filter out differences with schema, so 'schema.tablename' matches 'tablename'. 113 | * 114 | * @param filter 115 | * @return 116 | */ 117 | public List
getFilteredTables(List filter) { 118 | if (CollectionUtils.isEmpty(filter)) { 119 | return tables; 120 | } 121 | return tables.stream().filter((req) -> 122 | filter.stream().anyMatch((s) -> { 123 | String r = req.getName(); 124 | if (s.equalsIgnoreCase(r)) { 125 | return true; 126 | } else if (s.contains(".") && !r.contains(".")) { 127 | return StringUtils.endsWithIgnoreCase(s, "." + r); 128 | } else if (r.contains(".") && !s.contains(".")) { 129 | return StringUtils.endsWithIgnoreCase(r, "." + s); 130 | } 131 | return false; 132 | }) 133 | ).collect(Collectors.toList()); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/RequirementException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | * 16 | */ 17 | package com.strider.datadefender.requirement; 18 | 19 | import com.strider.datadefender.DataDefenderException; 20 | 21 | /** 22 | * Package level exception. Extends application level exception. 23 | * @author Zaahid Bateson 24 | */ 25 | public class RequirementException extends DataDefenderException { 26 | public RequirementException(final String msg) { 27 | super(msg); 28 | } 29 | public RequirementException(final String msg, final Throwable t) { 30 | super(msg, t); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/Table.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement; 17 | 18 | import java.util.List; 19 | 20 | import javax.xml.bind.annotation.XmlAccessType; 21 | import javax.xml.bind.annotation.XmlAccessorType; 22 | import javax.xml.bind.annotation.XmlAttribute; 23 | import javax.xml.bind.annotation.XmlElement; 24 | import javax.xml.bind.annotation.XmlElementWrapper; 25 | 26 | import lombok.Data; 27 | import org.apache.commons.collections4.CollectionUtils; 28 | 29 | /** 30 | * JAXB class that defines parameter table in Requirement.xml file 31 | * 32 | * @author Armenak Grigoryan 33 | */ 34 | @XmlAccessorType(XmlAccessType.NONE) 35 | @Data 36 | public class Table { 37 | 38 | @XmlAttribute 39 | private String name; 40 | 41 | @XmlAttribute(name = "primary-key") 42 | private String primaryKey; 43 | 44 | @XmlElement 45 | private String where; 46 | 47 | @XmlElementWrapper(name = "primary-key") 48 | @XmlElement(name = "key") 49 | private List primaryKeys; 50 | 51 | @XmlElementWrapper(name = "columns") 52 | @XmlElement(name = "column") 53 | private List columns; 54 | 55 | @XmlElementWrapper(name = "exclusions") 56 | @XmlElement(name = "exclude") 57 | private List exclusions; 58 | 59 | public Table() { 60 | } 61 | 62 | public Table(String name) { 63 | this.name = name; 64 | } 65 | 66 | public List getPrimaryKeyColumnNames() { 67 | if (CollectionUtils.isNotEmpty(primaryKeys)) { 68 | return primaryKeys; 69 | } 70 | return List.of(primaryKey); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/file/JaxbValidationEventHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Zaahid Bateson. 3 | * 4 | * This library is free software; you can redistribute it and/or 5 | * modify it under the terms of the GNU Lesser General Public 6 | * License as published by the Free Software Foundation; either 7 | * version 2.1 of the License, or (at your option) any later version. 8 | * 9 | * This library is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * Lesser General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Lesser General Public 15 | * License along with this library; if not, write to the Free Software 16 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 17 | * MA 02110-1301 USA 18 | */ 19 | package com.strider.datadefender.requirement.file; 20 | 21 | 22 | import org.w3c.dom.Node; 23 | 24 | import javax.xml.bind.ValidationEvent; 25 | import javax.xml.bind.ValidationEventHandler; 26 | import javax.xml.bind.ValidationEventLocator; 27 | import java.net.URL; 28 | import java.util.Map; 29 | 30 | import org.apache.logging.log4j.Level; 31 | import lombok.extern.log4j.Log4j2; 32 | import org.apache.commons.lang3.exception.ExceptionUtils; 33 | 34 | /** 35 | * 36 | * @author Zaahid Bateson 37 | */ 38 | @Log4j2 39 | public class JaxbValidationEventHandler implements ValidationEventHandler { 40 | 41 | @Override 42 | public boolean handleEvent(ValidationEvent event) { 43 | 44 | if (event == null) { 45 | throw new IllegalArgumentException(); 46 | } 47 | 48 | Map severityToLevel = Map.of( 49 | ValidationEvent.WARNING, Level.WARN, 50 | ValidationEvent.ERROR, Level.ERROR, 51 | ValidationEvent.FATAL_ERROR, Level.FATAL 52 | ); 53 | 54 | Level level = severityToLevel.getOrDefault(event.getSeverity(), Level.ERROR); 55 | String location = getFormattedLocation(event.getLocator()); 56 | String message = event.getMessage(); 57 | if (message == null && event.getLinkedException() != null) { 58 | message = ExceptionUtils.getRootCauseMessage(event.getLinkedException()); 59 | } 60 | log.log(level, "{}\n\tLocation: {},{}", message, location, event.getLinkedException()); 61 | return event.getSeverity() == ValidationEvent.WARNING; 62 | } 63 | 64 | private String getFormattedLocation(ValidationEventLocator locator) { 65 | 66 | String ret = "Location unavailable"; 67 | if (locator != null) { 68 | 69 | ret = ""; 70 | 71 | int line = locator.getLineNumber(); 72 | URL url = locator.getURL(); 73 | Object obj = locator.getObject(); 74 | Node node = locator.getNode(); 75 | 76 | if (line > -1) { 77 | ret += "Line: " + line; 78 | if (url != null) { 79 | ret += " In: " + url; 80 | } 81 | } 82 | if (obj != null) { 83 | ret += " Object: " + obj; 84 | } 85 | if (node != null) { 86 | ret += " Node: " + node; 87 | } 88 | } 89 | 90 | return ret; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/file/Loader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014, Armenak Grigoryan, Matthew Eaton, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | package com.strider.datadefender.requirement.file; 19 | 20 | import java.io.File; 21 | import java.io.FileInputStream; 22 | import java.io.FileNotFoundException; 23 | 24 | import javax.xml.bind.JAXBContext; 25 | import javax.xml.bind.JAXBException; 26 | import javax.xml.bind.Unmarshaller; 27 | import static javax.xml.bind.JAXBContext.newInstance; 28 | 29 | import com.strider.datadefender.requirement.Requirement; 30 | import com.strider.datadefender.requirement.registry.ClassAndFunctionRegistry; 31 | import java.lang.reflect.InvocationTargetException; 32 | import javax.xml.XMLConstants; 33 | import javax.xml.validation.Schema; 34 | import javax.xml.validation.SchemaFactory; 35 | 36 | import lombok.RequiredArgsConstructor; 37 | import lombok.extern.log4j.Log4j2; 38 | import org.xml.sax.SAXException; 39 | 40 | /** 41 | * Utility class to help handling requirement objects 42 | * @author Matthew Eaton 43 | */ 44 | @Log4j2 45 | @RequiredArgsConstructor 46 | public class Loader { 47 | 48 | private final JAXBContext jaxbContext; 49 | private final Unmarshaller unmarshaller; 50 | 51 | public Loader() throws JAXBException { 52 | 53 | jaxbContext = newInstance(Requirement.class); 54 | unmarshaller = jaxbContext.createUnmarshaller(); 55 | unmarshaller.setEventHandler(new JaxbValidationEventHandler()); 56 | } 57 | 58 | /** 59 | * Load requirement XML file and return a Requirement object representing 60 | * it. 61 | * 62 | * @param requirementFile Requirement filename and path 63 | * @param version required version 64 | * @return Requirement object loaded based on file 65 | * @throws javax.xml.bind.JAXBException 66 | * @throws java.io.FileNotFoundException 67 | * @throws java.lang.NoSuchMethodException 68 | * @throws java.lang.InstantiationException 69 | * @throws java.lang.IllegalAccessException 70 | * @throws java.lang.reflect.InvocationTargetException 71 | * @throws org.xml.sax.SAXException 72 | */ 73 | public Requirement load(final String requirementFile) throws 74 | FileNotFoundException, 75 | JAXBException, 76 | NoSuchMethodException, 77 | InstantiationException, 78 | IllegalAccessException, 79 | IllegalArgumentException, 80 | InvocationTargetException, 81 | SAXException { 82 | 83 | Requirement requirement = null; 84 | log.info("Loading requirement file: {}", requirementFile); 85 | 86 | SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); 87 | Schema schema = schemaFactory.newSchema(Loader.class.getResource("requirement.xsd")); 88 | 89 | unmarshaller.setSchema(schema); 90 | requirement = (Requirement) unmarshaller.unmarshal( 91 | new FileInputStream(new File(requirementFile)) 92 | ); 93 | ClassAndFunctionRegistry.singleton().registerFunctions(requirement); 94 | return requirement; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/package-info.java: -------------------------------------------------------------------------------- 1 | @XmlSchema( 2 | namespace = "https://armenak.github.io/DataDefender/anonymizer", 3 | elementFormDefault = XmlNsForm.QUALIFIED, 4 | xmlns = { @XmlNs(namespaceURI = "https://armenak.github.io/DataDefender/anonymizer", prefix = "") } 5 | ) 6 | package com.strider.datadefender.requirement; 7 | 8 | import javax.xml.bind.annotation.XmlNs; 9 | import javax.xml.bind.annotation.XmlSchema; 10 | import javax.xml.bind.annotation.XmlNsForm; 11 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/plan/ArrayElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement.plan; 17 | 18 | import javax.xml.bind.annotation.XmlAccessType; 19 | import javax.xml.bind.annotation.XmlAccessorType; 20 | import javax.xml.bind.annotation.XmlAttribute; 21 | 22 | import lombok.Data; 23 | 24 | /** 25 | * JAXB class defining values for array parameters. 26 | * 27 | * Represents the value of a single array element for a given Parameter. 28 | * 29 | * @see Parameter.getArrayElements 30 | * @author Zaahid Bateson 31 | */ 32 | @XmlAccessorType(XmlAccessType.NONE) 33 | @Data 34 | public class ArrayElement { 35 | /** 36 | * The array element's value 37 | */ 38 | @XmlAttribute 39 | private String value; 40 | 41 | @XmlAttribute(name = "pass-current-value") 42 | private boolean isDynamicValue = false; 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/plan/FunctionAttributeAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement.plan; 17 | 18 | import javax.xml.bind.annotation.adapters.XmlAdapter; 19 | 20 | /** 21 | * XmlAdapter to get a Function type from a String attribute. 22 | * 23 | * @author Zaahid Bateson 24 | */ 25 | public class FunctionAttributeAdapter extends XmlAdapter { 26 | 27 | @Override 28 | public Function unmarshal(String value) throws Exception { 29 | return new Function(value, true); 30 | } 31 | 32 | @Override 33 | public String marshal(Function f) throws Exception { 34 | return (f != null) ? f.getFunctionName() : null; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/plan/GlobalPlan.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement.plan; 17 | 18 | import java.lang.reflect.InvocationTargetException; 19 | 20 | import javax.xml.bind.Unmarshaller; 21 | import javax.xml.bind.annotation.XmlAccessType; 22 | import javax.xml.bind.annotation.XmlAccessorType; 23 | import javax.xml.bind.annotation.XmlAttribute; 24 | import javax.xml.bind.annotation.XmlID; 25 | 26 | import lombok.extern.log4j.Log4j2; 27 | import lombok.Getter; 28 | import lombok.NoArgsConstructor; 29 | import lombok.Setter; 30 | import org.apache.commons.collections4.CollectionUtils; 31 | 32 | /** 33 | * 34 | * @author Zaahid Bateson 35 | */ 36 | @Log4j2 37 | @Getter 38 | @Setter 39 | @NoArgsConstructor 40 | @XmlAccessorType(XmlAccessType.NONE) 41 | public class GlobalPlan extends Plan { 42 | 43 | @XmlID 44 | @XmlAttribute 45 | private String id; 46 | 47 | public GlobalPlan(String id) { 48 | this.id = id; 49 | } 50 | 51 | /** 52 | * Uses functionName and parameters to find the method to associate with 53 | * 'Function'. 54 | * 55 | * @param unmarshaller 56 | * @param parent 57 | * @throws ClassNotFoundException 58 | * @throws InstantiationException 59 | * @throws IllegalAccessException 60 | * @throws InvocationTargetException 61 | */ 62 | @Override 63 | public void afterUnmarshal(Unmarshaller unmarshaller, Object parent) 64 | throws ClassNotFoundException, 65 | InstantiationException, 66 | IllegalAccessException, 67 | InvocationTargetException { 68 | 69 | log.debug("Unmarshalling GlobalPlan with id {}", id); 70 | log.debug("Number of functions: {}", CollectionUtils.size(getFunctions())); 71 | // do nothing 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/plan/Invokable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement.plan; 17 | 18 | /** 19 | * Defines a single "invoke" method with the hopes to make Plan/Function 20 | * not just chainable but also nestable. 21 | * 22 | * @author Zaahid Bateson 23 | */ 24 | public interface Invokable { 25 | /** 26 | * Invokes the underlying function with the passed runningValue for dynamic 27 | * arguments. 28 | * 29 | * @param runningValue 30 | * @return 31 | * @throws Exception 32 | */ 33 | Object invoke(Object runningValue) throws Exception; 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/plan/PlanRef.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement.plan; 17 | 18 | import com.strider.datadefender.requirement.Column; 19 | import java.lang.reflect.InvocationTargetException; 20 | 21 | import javax.xml.bind.Unmarshaller; 22 | import javax.xml.bind.annotation.XmlAccessType; 23 | import javax.xml.bind.annotation.XmlAccessorType; 24 | import javax.xml.bind.annotation.XmlAttribute; 25 | import javax.xml.bind.annotation.XmlIDREF; 26 | 27 | import lombok.extern.log4j.Log4j2; 28 | import lombok.Data; 29 | import lombok.NoArgsConstructor; 30 | 31 | /** 32 | * 33 | * @author Zaahid Bateson 34 | */ 35 | @Log4j2 36 | @Data 37 | @NoArgsConstructor 38 | @XmlAccessorType(XmlAccessType.NONE) 39 | public class PlanRef { 40 | 41 | @XmlIDREF 42 | @XmlAttribute(name = "ref-id") 43 | private GlobalPlan ref; 44 | 45 | public PlanRef(GlobalPlan ref) { 46 | this.ref = ref; 47 | } 48 | 49 | /** 50 | * Uses functionName and parameters to find the method to associate with 51 | * 'Function'. 52 | * 53 | * @param unmarshaller 54 | * @param parent 55 | * @throws ClassNotFoundException 56 | * @throws InstantiationException 57 | * @throws IllegalAccessException 58 | * @throws InvocationTargetException 59 | */ 60 | public void afterUnmarshal(Unmarshaller unmarshaller, Object parent) 61 | throws ClassNotFoundException, 62 | InstantiationException, 63 | IllegalAccessException, 64 | InvocationTargetException { 65 | 66 | log.debug("Unmarshalling plan-ref"); 67 | log.debug("ref-id: {}", () -> ref.getId()); 68 | 69 | if (!(parent instanceof Column)) { 70 | return; 71 | } 72 | final Class columnType = ((Column) parent).getType(); 73 | ref.initialize(columnType); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/plan/package-info.java: -------------------------------------------------------------------------------- 1 | @XmlSchema( 2 | namespace = "https://armenak.github.io/DataDefender/anonymizer", 3 | elementFormDefault = XmlNsForm.QUALIFIED, 4 | xmlns = { @XmlNs(namespaceURI = "https://armenak.github.io/DataDefender/anonymizer", prefix = "") } 5 | ) 6 | package com.strider.datadefender.requirement.plan; 7 | 8 | import javax.xml.bind.annotation.XmlNs; 9 | import javax.xml.bind.annotation.XmlSchema; 10 | import javax.xml.bind.annotation.XmlNsForm; 11 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/registry/DatabaseAwareRequirementFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement.registry; 17 | 18 | import com.strider.datadefender.database.IDbFactory; 19 | 20 | /** 21 | * Configures a member variable IDbFactory in a concrete 'initialize' method. 22 | * 23 | * @author Zaahid Bateson 24 | */ 25 | public abstract class DatabaseAwareRequirementFunction extends RequirementFunction { 26 | 27 | protected IDbFactory dbFactory; 28 | 29 | public final void initialize(IDbFactory dbFactory) { 30 | this.dbFactory = dbFactory; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/requirement/registry/RequirementFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.requirement.registry; 17 | 18 | /** 19 | * Specifies that the class contains functions for use by anonymizer functions. 20 | * 21 | * This identifies classes that the anonymizer should try to construct, versus 22 | * classes that identify the return type of a column (and methods that operate 23 | * on them, for example a function java.lang.String#substring) that should not 24 | * be constructed. 25 | * 26 | * @author Zaahid Bateson 27 | */ 28 | public abstract class RequirementFunction { 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/specialcase/EmailDetector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014-2020, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.specialcase; 17 | 18 | import com.strider.datadefender.discoverer.Discoverer.ColumnMatch; 19 | import org.apache.commons.validator.routines.EmailValidator; 20 | 21 | import com.strider.datadefender.database.metadata.TableMetaData; 22 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 23 | 24 | import lombok.extern.log4j.Log4j2; 25 | import org.apache.commons.lang3.StringUtils; 26 | 27 | /** 28 | * @author Armenak Grigoryan 29 | */ 30 | @Log4j2 31 | public class EmailDetector implements SpecialCase { 32 | 33 | public static ColumnMatch detectEmail(final ColumnMetaData metaData, final String text) { 34 | if (StringUtils.isNotBlank(text) && isValidEmail(text)) { 35 | log.debug("Email detected: " + text); 36 | return new ColumnMatch(metaData, 1.0, "email", null); 37 | } else { 38 | log.debug("Email " + text + " is not a valid email"); 39 | } 40 | 41 | return null; 42 | } 43 | 44 | /** 45 | * @param email Email address 46 | * @return true if email is valid, otherwise false 47 | */ 48 | private static boolean isValidEmail(final String email) { 49 | 50 | EmailValidator eValidator = EmailValidator.getInstance(); 51 | if (eValidator.isValid(email)) { 52 | log.debug("*************** Email " + email + " is valid"); 53 | return true; 54 | } else { 55 | return false; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/specialcase/PhiDetector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2020, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.specialcase; 17 | 18 | import com.strider.datadefender.discoverer.Discoverer.ColumnMatch; 19 | import java.io.BufferedReader; 20 | import java.io.FileReader; 21 | import java.io.IOException; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | import java.util.Locale; 26 | 27 | import com.strider.datadefender.discoverer.Probability; 28 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 29 | import java.util.Objects; 30 | 31 | import lombok.extern.log4j.Log4j2; 32 | import org.apache.commons.lang3.StringUtils; 33 | 34 | /** 35 | * @author Armenak Grigoryan 36 | */ 37 | @Log4j2 38 | public class PhiDetector implements SpecialCase { 39 | 40 | private static final String PHI_FILE = "phi.txt"; 41 | private static List phiList = new ArrayList(); 42 | 43 | static { 44 | try { 45 | log.info("*** reading from " + PHI_FILE); 46 | 47 | try (BufferedReader br = new BufferedReader(new FileReader(PHI_FILE))) { 48 | for (String line; (line = br.readLine()) != null; ) { 49 | phiList.add(line); 50 | } 51 | } 52 | } catch (IOException ioe) { 53 | log.error(ioe.toString()); 54 | } 55 | } 56 | 57 | /** 58 | * Generates random 9-digit student number 59 | * @param data 60 | * @param text 61 | * @return String 62 | */ 63 | public static ColumnMatch isPhiTerm(final ColumnMetaData data, final String text) { 64 | if (StringUtils.isNotBlank(text) 65 | && Objects.equals(String.class, data.getColumnType()) 66 | && phiList.contains(text.trim().toLowerCase(Locale.ENGLISH))) { 67 | 68 | log.debug("PHI detected: " + text); 69 | return new ColumnMatch( 70 | data, 71 | 100.0, 72 | "phi", 73 | List.of(new Probability(text, 1.00)) 74 | ); 75 | } 76 | return null; 77 | } 78 | } -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/specialcase/SpecialCase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014-2017, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.specialcase; 17 | 18 | /** 19 | * Marker interface for all special case classes 20 | * 21 | * @author Armenak Grigoryan 22 | */ 23 | public interface SpecialCase {} 24 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/ApplicationLock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This is free software; you can redistribute it and/or modify it 3 | * under the terms of the GNU Lesser General Public License as 4 | * published by the Free Software Foundation; either version 2.1 of 5 | * the License, or (at your option) any later version. 6 | * 7 | * This software is distributed in the hope that it will be useful, 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 | * Lesser General Public License for more details. 11 | */ 12 | package com.strider.datadefender.utils; 13 | 14 | import java.io.File; 15 | import java.io.FileNotFoundException; 16 | import java.io.IOException; 17 | import java.io.RandomAccessFile; 18 | 19 | import java.nio.channels.FileChannel; 20 | import java.nio.channels.FileLock; 21 | import java.nio.channels.OverlappingFileLockException; 22 | 23 | import com.strider.datadefender.DataDefenderException; 24 | import lombok.extern.log4j.Log4j2; 25 | 26 | /** 27 | * Most of the code is copied from this page: http://www.rgagnon.com/javadetails/java-0288.html 28 | * 29 | * Modified by Armenak Grigoryan 30 | */ 31 | @Log4j2 32 | public class ApplicationLock { 33 | 34 | private final String appName; 35 | private File file; 36 | private FileChannel channel; 37 | private FileLock lock; 38 | 39 | /** 40 | * Constructor 41 | * 42 | * @param appName application name 43 | */ 44 | public ApplicationLock(final String appName) { 45 | this.appName = appName; 46 | } 47 | 48 | private void closeLock() throws DataDefenderException { 49 | try { 50 | lock.release(); 51 | } catch (IOException e) { 52 | throw new DataDefenderException("Problem releasing file lock", e); 53 | } 54 | 55 | try { 56 | channel.close(); 57 | } catch (IOException e) { 58 | throw new DataDefenderException("Problem closing channel", e); 59 | } 60 | } 61 | 62 | private void deleteFile() { 63 | file.delete(); 64 | } 65 | 66 | /** 67 | * Returns true if there is another instance of the application is running. 68 | * Otherwise returns false. 69 | * 70 | * @return boolean 71 | * @throws com.strider.datadefender.DataDefenderException 72 | */ 73 | public boolean isAppActive() throws DataDefenderException { 74 | try { 75 | file = new File(System.getProperty("user.home"), appName + ".tmp"); 76 | channel = new RandomAccessFile(file, "rw").getChannel(); 77 | log.debug("Creating lock file " + file.getName()); 78 | 79 | try { 80 | lock = channel.tryLock(); 81 | log.debug("Locking file ..."); 82 | } catch (OverlappingFileLockException | IOException e) { 83 | 84 | // already locked 85 | log.error("File " + file.getName() + " already locket"); 86 | closeLock(); 87 | 88 | return true; 89 | } 90 | 91 | if (lock == null) { 92 | closeLock(); 93 | 94 | return true; 95 | } 96 | 97 | Runtime.getRuntime().addShutdownHook(new Thread() { 98 | 99 | // destroy the lock when the JVM is closing 100 | @Override 101 | public void run() { 102 | try { 103 | log.debug("Closing lock file"); 104 | closeLock(); 105 | deleteFile(); 106 | } catch (DataDefenderException ae) { 107 | log.error("Problem closing file lock"); 108 | } 109 | } 110 | }); 111 | 112 | return false; 113 | } catch (FileNotFoundException fnfe) { 114 | closeLock(); 115 | 116 | return true; 117 | } 118 | } 119 | } 120 | 121 | 122 | //~ Formatted by Jindent --- http://www.jindent.com 123 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/Encoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2021, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.utils; 17 | 18 | import java.io.UnsupportedEncodingException; 19 | 20 | import java.security.InvalidKeyException; 21 | import java.security.MessageDigest; 22 | import java.security.NoSuchAlgorithmException; 23 | import java.util.Arrays; 24 | import java.util.Base64; 25 | 26 | import javax.crypto.BadPaddingException; 27 | import javax.crypto.Cipher; 28 | import javax.crypto.IllegalBlockSizeException; 29 | import javax.crypto.NoSuchPaddingException; 30 | import javax.crypto.spec.SecretKeySpec; 31 | 32 | import lombok.extern.log4j.Log4j2; 33 | 34 | /** 35 | * This class implements determenistic data anonymization 36 | * by encoding the data using "salt" and decoding it. 37 | * 38 | * @author Armenak Grigoryan 39 | */ 40 | @Log4j2 41 | public final class Encoder { 42 | 43 | private static SecretKeySpec secretKey; 44 | private static byte[] key; 45 | 46 | /** 47 | * Empty constructor 48 | */ 49 | public Encoder() { 50 | } 51 | 52 | public static void setKey(String myKey) { 53 | MessageDigest sha = null; 54 | 55 | try { 56 | key = myKey.getBytes("UTF-8"); 57 | sha = MessageDigest.getInstance("SHA-1"); 58 | key = sha.digest(key); 59 | key = Arrays.copyOf(key, 16); 60 | secretKey = new SecretKeySpec(key, "AES"); 61 | } 62 | catch (NoSuchAlgorithmException | UnsupportedEncodingException e) { 63 | log.debug(e); 64 | } 65 | } 66 | 67 | 68 | public String encrypt(String strToEncrypt, String secret) { 69 | 70 | try { 71 | setKey(secret); 72 | Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding"); 73 | cipher.init(Cipher.ENCRYPT_MODE, secretKey); 74 | return Base64.getEncoder().encodeToString(cipher.doFinal(strToEncrypt.getBytes("UTF-8"))); 75 | } catch (UnsupportedEncodingException | InvalidKeyException | NoSuchAlgorithmException | 76 | BadPaddingException | IllegalBlockSizeException | NoSuchPaddingException e) { 77 | log.error("Error while encrypting: " + e.toString()); 78 | } 79 | 80 | return null; 81 | 82 | } 83 | 84 | 85 | public String decrypt(String strToDecrypt, String secret) { 86 | try { 87 | setKey(secret); 88 | Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING"); 89 | cipher.init(Cipher.DECRYPT_MODE, secretKey); 90 | return new String(cipher.doFinal(Base64.getDecoder().decode(strToDecrypt))); 91 | } catch (InvalidKeyException | NoSuchAlgorithmException | BadPaddingException | 92 | IllegalBlockSizeException | NoSuchPaddingException e) { 93 | log.error("Error while decrypting: " + e.toString()); 94 | } 95 | return null; 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/ICloseableNoException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2015, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.utils; 22 | 23 | /** 24 | * Just hide the Exception thrown by close() as we don't need to throw one. 25 | * @author Akira Matsuo 26 | */ 27 | public interface ICloseableNoException extends AutoCloseable { 28 | @Override 29 | void close(); 30 | } 31 | 32 | 33 | //~ Formatted by Jindent --- http://www.jindent.com 34 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/IConsumerWithException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2015, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.utils; 22 | 23 | /** 24 | * @author Akira Matsuo 25 | * 26 | * @param 27 | * Much like a java.util.Consumer but throws an exception. 28 | */ 29 | @FunctionalInterface 30 | public interface IConsumerWithException { 31 | void accept(T obj) throws E; 32 | } 33 | 34 | 35 | //~ Formatted by Jindent --- http://www.jindent.com 36 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/ISupplierWithException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2015, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | package com.strider.datadefender.utils; 19 | 20 | /** 21 | * @author Akira Matsuo 22 | * 23 | * @param 24 | * Much like a java.util.Supplier but throws an exception. 25 | * @param 26 | */ 27 | @FunctionalInterface 28 | public interface ISupplierWithException { 29 | T get() throws E; 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/LikeMatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.utils; 22 | 23 | import java.util.Locale; 24 | import java.util.regex.Pattern; 25 | 26 | /** 27 | * Simple matcher to mimic SQL LIKE queries. 28 | * 29 | * At the moment only "%", "_" and "?" are supported. 30 | * 31 | * @author Zaahid Bateson 32 | */ 33 | public class LikeMatcher { 34 | final private String regex; 35 | 36 | /** 37 | * Initializes a LikeMatcher with the given pattern. 38 | * 39 | * @param pattern the LIKE query pattern 40 | */ 41 | public LikeMatcher(final String pattern) { 42 | 43 | // splitting on '?', '_', and '%' with look-behind and look-ahead so they're included in the split array 44 | final String[] parts = pattern.split("((?<=[\\?\\_\\%])|(?=[\\?\\_\\%]))"); 45 | final StringBuilder reg = new StringBuilder("^"); 46 | 47 | for (final String part : parts) { 48 | if ("%".equals(part)) { 49 | reg.append(".*?"); 50 | } else if ("?".equals(part) || "_".equals(part)) { 51 | reg.append('.'); 52 | } else { 53 | reg.append(Pattern.quote(part.toLowerCase(Locale.ENGLISH))); 54 | } 55 | } 56 | 57 | reg.append('$'); 58 | this.regex = reg.toString(); 59 | } 60 | 61 | /** 62 | * Returns true if the given string matches the Like pattern. 63 | * 64 | * @param str the string to test against 65 | * @return 66 | */ 67 | public boolean matches(final String str) { 68 | return str.toLowerCase(Locale.ENGLISH).matches(regex); 69 | } 70 | } 71 | 72 | 73 | //~ Formatted by Jindent --- http://www.jindent.com 74 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/Score.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 4 | * by the @authors tag. See the copyright.txt in the distribution for a 5 | * full listing of individual contributors. 6 | * 7 | * This is free software; you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as 9 | * published by the Free Software Foundation; either version 2.1 of 10 | * the License, or (at your option) any later version. 11 | * 12 | * This software is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | * Lesser General Public License for more details. 16 | * 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.utils; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | /** 27 | * This class calculates the risk score for an individual column and for entire data store. 28 | * 29 | * @author Armenak Grigoryan 30 | */ 31 | public class Score { 32 | private static final int SCORE_LOW = 1; 33 | private static final int SCORE_MEDIUM = 2; 34 | private static final int SCORE_HIGH = 3; 35 | private static final List averageScore = new ArrayList<>(); 36 | 37 | public String columnScore(final int rowCount) { 38 | int score = 0; 39 | 40 | if (rowCount == 0) { 41 | score = 0; 42 | } else if ((rowCount > 0) && (rowCount <= 100)) { 43 | score = SCORE_LOW; 44 | } else if ((rowCount > 100) && (rowCount <= 1000)) { 45 | score = SCORE_MEDIUM; 46 | } else { 47 | score = SCORE_HIGH; 48 | } 49 | 50 | averageScore.add(score); 51 | 52 | return getScoreDefinition(score); 53 | } 54 | 55 | public String dataStoreScore() { 56 | float averageDataStoreScore = 0; 57 | int tmp = 0; 58 | 59 | for (final int score : averageScore) { 60 | tmp += score; 61 | } 62 | 63 | if ((averageScore != null) && (averageScore.size() > 0)) { 64 | averageDataStoreScore = tmp / averageScore.size(); 65 | } 66 | 67 | return getScoreDefinition(Math.round(averageDataStoreScore)); 68 | } 69 | 70 | public String getScoreDefinition(final int score) { 71 | String scoreDefinition = ""; 72 | 73 | switch (score) { 74 | case 1: 75 | scoreDefinition = "Low"; 76 | break; 77 | case 2: 78 | scoreDefinition = "Medium"; 79 | break; 80 | case 3: 81 | scoreDefinition = "High"; 82 | break; 83 | default: 84 | scoreDefinition = "Undefined"; 85 | break; 86 | } 87 | 88 | return scoreDefinition; 89 | } 90 | } 91 | 92 | 93 | //~ Formatted by Jindent --- http://www.jindent.com 94 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/Xeger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 Wilfred Springer 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Modified by Armenak Grigoryan 17 | */ 18 | package com.strider.datadefender.utils; 19 | 20 | import java.util.List; 21 | import java.util.Random; 22 | 23 | import dk.brics.automaton.Automaton; 24 | import dk.brics.automaton.RegExp; 25 | import dk.brics.automaton.State; 26 | import dk.brics.automaton.Transition; 27 | 28 | /** 29 | * An object that will generate text from a regular expression. In a way, it's 30 | * the opposite of a regular expression matcher: an instance of this class will 31 | * produce text that is guaranteed to match the regular expression passed in. 32 | */ 33 | public class Xeger { 34 | private final Automaton automaton; 35 | private final Random random; 36 | 37 | /** 38 | * As {@link nl.flotsam.xeger.Xeger#Xeger(String, java.util.Random)}, creating a {@link java.util.Random} instance 39 | * implicityly. 40 | * @param regex 41 | */ 42 | public Xeger(final String regex) { 43 | this(regex, new Random()); 44 | } 45 | 46 | /** 47 | * Constructs a new instance, accepting the regular expression and the randomizer. 48 | * 49 | * @param regex The regular expression. (Not null.) 50 | * @param random The object that will randomize the way the String is generated. (Not null.) 51 | * @throws IllegalArgumentException If the regular expression is invalid. 52 | */ 53 | public Xeger(final String regex, final Random random) { 54 | assert regex != null; 55 | assert random != null; 56 | this.automaton = new RegExp(regex).toAutomaton(); 57 | this.random = random; 58 | } 59 | 60 | private void appendChoice(final StringBuilder builder, final Transition transition) { 61 | final char c = (char) XegerUtils.getRandomInt(transition.getMin(), transition.getMax(), random); 62 | 63 | builder.append(c); 64 | } 65 | 66 | /** 67 | * Generates a random String that is guaranteed to match the regular expression passed to the constructor. 68 | * @return String 69 | */ 70 | public String generate() { 71 | final StringBuilder builder = new StringBuilder(); 72 | 73 | generate(builder, automaton.getInitialState()); 74 | 75 | return builder.toString(); 76 | } 77 | 78 | private void generate(final StringBuilder builder, final State state) { 79 | final List transitions = state.getSortedTransitions(true); 80 | 81 | if (transitions.isEmpty()) { 82 | assert state.isAccept(); 83 | 84 | return; 85 | } 86 | 87 | final int nroptions = state.isAccept() 88 | ? transitions.size() 89 | : transitions.size() - 1; 90 | final int option = XegerUtils.getRandomInt(0, nroptions, random); 91 | 92 | if (state.isAccept() && (option == 0)) { // 0 is considered stop 93 | return; 94 | } 95 | 96 | // Moving on to next transition 97 | final Transition transition = transitions.get(option - (state.isAccept() 98 | ? 1 99 | : 0)); 100 | 101 | appendChoice(builder, transition); 102 | generate(builder, transition.getDest()); 103 | } 104 | } 105 | 106 | 107 | //~ Formatted by Jindent --- http://www.jindent.com 108 | -------------------------------------------------------------------------------- /src/main/java/com/strider/datadefender/utils/XegerUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009 Wilfred Springer 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Modified by Armenak Grigoryan 17 | */ 18 | 19 | 20 | 21 | package com.strider.datadefender.utils; 22 | 23 | import java.util.Random; 24 | 25 | public class XegerUtils { 26 | 27 | /** 28 | * Generates a random number within the given bounds. 29 | * 30 | * @param min The minimum number (inclusive). 31 | * @param max The maximum number (inclusive). 32 | * @param random The object used as the randomizer. 33 | * @return A random number in the given range. 34 | */ 35 | public final static int getRandomInt(final int min, final int max, final Random random) { 36 | final int dif = max - min; 37 | final float number = random.nextFloat(); // 0 <= number < 1 38 | 39 | return min + Math.round(number * dif); 40 | } 41 | } 42 | 43 | 44 | //~ Formatted by Jindent --- http://www.jindent.com 45 | -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/anonymizer/functions/cities.txt: -------------------------------------------------------------------------------- 1 | Tokyo 2 | Jakarta 3 | Chongqing 4 | Manila 5 | Delhi 6 | Seoul 7 | Mumbai 8 | Shanghai 9 | São Paulo 10 | Beijing 11 | New York City 12 | Lagos 13 | Mexico City 14 | Guangzhou 15 | Dhaka 16 | Osaka 17 | Cairo 18 | Karachi 19 | Moscow 20 | Bangkok 21 | Chengdu 22 | Los Angeles 23 | Kolkata 24 | Buenos Aires 25 | Istanbul 26 | Tehran 27 | London 28 | Shenzhen 29 | Tianjin 30 | Kinshasa 31 | Rio de Janeiro 32 | Paris 33 | Baoding 34 | Lahore 35 | Lima 36 | Bangalore 37 | Ho Chi Minh City 38 | Harbin 39 | Wuhan 40 | Shijiazhuang 41 | Bogotá 42 | Suzhou 43 | Linyi 44 | Chennai 45 | Nagoya 46 | Nanyang 47 | Zhengzhou 48 | Hyderabad 49 | Surabaya 50 | Hangzhou 51 | Johannesburg 52 | Chicago 53 | Xi'an 54 | Quanzhou 55 | Taipei 56 | Dongguan 57 | Bandung 58 | Hanoi 59 | Shenyang 60 | Baghdad 61 | Onitsha 62 | Kuala Lumpur 63 | Ahmedabad 64 | Luanda 65 | Washington, D.C. 66 | Dallas 67 | Hong Kong 68 | Pune 69 | Nanjing 70 | Boston 71 | Santiago 72 | Houston 73 | Riyadh 74 | Düsseldorf 75 | Madrid 76 | Toronto 77 | Surat 78 | San Francisco 79 | -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/anonymizer/functions/countries.txt: -------------------------------------------------------------------------------- 1 | Afghanistan 2 | Albania 3 | Algeria 4 | Andorra 5 | Angola 6 | Antigua and Barbuda 7 | Argentina 8 | Armenia 9 | Australia 10 | Austria 11 | Azerbaijan 12 | Bahamas 13 | Bahrain 14 | Bangladesh 15 | Barbados 16 | Belarus 17 | Belgium 18 | Belize 19 | Benin 20 | Bhutan 21 | Bolivia 22 | Bosnia and Herzegovina 23 | Botswana 24 | Brazil 25 | Brunei 26 | Bulgaria 27 | Burkina Faso 28 | Burundi 29 | Côte d'Ivoire 30 | Cabo Verde 31 | Cambodia 32 | Cameroon 33 | Canada 34 | Central African Republic 35 | Chad 36 | Chile 37 | China 38 | Colombia 39 | Comoros 40 | Congo (Congo-Brazzaville) 41 | Costa Rica 42 | Croatia 43 | Cuba 44 | Cyprus 45 | Czechia (Czech Republic) 46 | Democratic Republic of the Congo 47 | Denmark 48 | Djibouti 49 | Dominica 50 | Dominican Republic 51 | Ecuador 52 | Egypt 53 | El Salvador 54 | Equatorial Guinea 55 | Eritrea 56 | Estonia 57 | Eswatini (fmr. "Swaziland") 58 | Ethiopia 59 | Fiji 60 | Finland 61 | France 62 | Gabon 63 | Gambia 64 | Georgia 65 | Germany 66 | Ghana 67 | Greece 68 | Grenada 69 | Guatemala 70 | Guinea 71 | Guinea-Bissau 72 | Guyana 73 | Haiti 74 | Holy See 75 | Honduras 76 | Hungary 77 | Iceland 78 | India 79 | Indonesia 80 | Iran 81 | Iraq 82 | Ireland 83 | Israel 84 | Italy 85 | Jamaica 86 | Japan 87 | Jordan 88 | Kazakhstan 89 | Kenya 90 | Kiribati 91 | Kuwait 92 | Kyrgyzstan 93 | Laos 94 | Latvia 95 | Lebanon 96 | Lesotho 97 | Liberia 98 | Libya 99 | Liechtenstein 100 | Lithuania 101 | Luxembourg 102 | Madagascar 103 | Malawi 104 | Malaysia 105 | Maldives 106 | Mali 107 | Malta 108 | Marshall Islands 109 | Mauritania 110 | Mauritius 111 | Mexico 112 | Micronesia 113 | Moldova 114 | Monaco 115 | Mongolia 116 | Montenegro 117 | Morocco 118 | Mozambique 119 | Myanmar (formerly Burma) 120 | Namibia 121 | Nauru 122 | Nepal 123 | Netherlands 124 | New Zealand 125 | Nicaragua 126 | Niger 127 | Nigeria 128 | North Korea 129 | North Macedonia 130 | Norway 131 | Oman 132 | Pakistan 133 | Palau 134 | Palestine State 135 | Panama 136 | Papua New Guinea 137 | Paraguay 138 | Peru 139 | Philippines 140 | Poland 141 | Portugal 142 | Qatar 143 | Romania 144 | Russia 145 | Rwanda 146 | Saint Kitts and Nevis 147 | Saint Lucia 148 | Saint Vincent and the Grenadines 149 | Samoa 150 | San Marino 151 | Sao Tome and Principe 152 | Saudi Arabia 153 | Senegal 154 | Serbia 155 | Seychelles 156 | Sierra Leone 157 | Singapore 158 | Slovakia 159 | Slovenia 160 | Solomon Islands 161 | Somalia 162 | South Africa 163 | South Korea 164 | South Sudan 165 | Spain 166 | Sri Lanka 167 | Sudan 168 | Suriname 169 | Sweden 170 | Switzerland 171 | Syria 172 | Tajikistan 173 | Tanzania 174 | Thailand 175 | Timor-Leste 176 | Togo 177 | Tonga 178 | Trinidad and Tobago 179 | Tunisia 180 | Turkey 181 | Turkmenistan 182 | Tuvalu 183 | Uganda 184 | Ukraine 185 | United Arab Emirates 186 | United Kingdom 187 | United States of America 188 | Uruguay 189 | Uzbekistan 190 | Vanuatu 191 | Venezuela 192 | Vietnam 193 | Yemen 194 | Zambia 195 | Zimbabwe 196 | -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/anonymizer/functions/provinces_states.txt: -------------------------------------------------------------------------------- 1 | Alabama 2 | Alaska 3 | Arizona 4 | Arkansas 5 | California 6 | Colorado 7 | Connecticut 8 | Delaware 9 | District of Columbia 10 | Florida 11 | Georgia 12 | Guam 13 | Hawaii 14 | Idaho 15 | Illinois 16 | Indiana 17 | Iowa 18 | Kansas 19 | Kentucky 20 | Louisiana 21 | Maine 22 | Maryland 23 | Massachusetts 24 | Michigan 25 | Minnesota 26 | Mississippi 27 | Missouri 28 | Montana 29 | Nebraska 30 | Nevada 31 | New Hampshire 32 | New Jersey 33 | New Mexico 34 | New York 35 | North Carolina 36 | North Dakota 37 | Ohio 38 | Oklahoma 39 | Oregon 40 | Pennsylvania 41 | Puerto Rico 42 | Rhode Island 43 | South Carolina 44 | South Dakota 45 | Tennessee 46 | Texas 47 | Utah 48 | Vermont 49 | Virgin Islands 50 | Virginia 51 | Washington 52 | West Virginia 53 | Wisconsin 54 | Wyoming 55 | Alberta 56 | British Columbia 57 | Manitoba 58 | New Brunswick 59 | Newfoundland 60 | Northwest Territories 61 | Nova Scotia 62 | Ontario 63 | Prince Edward Island 64 | Quebec 65 | Saskatchewan 66 | Yukon Territory 67 | -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/anonymizer/functions/provinces_states_codes.txt: -------------------------------------------------------------------------------- 1 | AL 2 | AK 3 | AZ 4 | AR 5 | CA 6 | CO 7 | CT 8 | DE 9 | DC 10 | FL 11 | GA 12 | GU 13 | HI 14 | ID 15 | IL 16 | IN 17 | IA 18 | KS 19 | KY 20 | LA 21 | ME 22 | MD 23 | MA 24 | MI 25 | MN 26 | MS 27 | MO 28 | MT 29 | NE 30 | NV 31 | NH 32 | NJ 33 | NM 34 | NY 35 | NC 36 | ND 37 | OH 38 | OK 39 | OR 40 | PA 41 | PR 42 | RI 43 | SC 44 | SD 45 | TN 46 | TX 47 | UT 48 | VT 49 | VI 50 | VA 51 | WA 52 | WV 53 | WI 54 | WY 55 | AB 56 | BC 57 | MB 58 | NB 59 | NF 60 | NT 61 | NS 62 | ON 63 | PE 64 | PQ 65 | SK 66 | YT 67 | -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-ner-date.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-ner-date.bin -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-ner-location.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-ner-location.bin -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-ner-money.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-ner-money.bin -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-ner-organization.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-ner-organization.bin -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-ner-person.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-ner-person.bin -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-ner-time.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-ner-time.bin -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-sent.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-sent.bin -------------------------------------------------------------------------------- /src/main/resources/com/strider/datadefender/discoverer/en-token.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/armenak/DataDefender/40a43c621d88b326fee206e45fd1b7e8760a6a0b/src/main/resources/com/strider/datadefender/discoverer/en-token.bin -------------------------------------------------------------------------------- /src/main/resources/filediscovery.properties: -------------------------------------------------------------------------------- 1 | probability_threshold=0.5 2 | english_tokens=/Users/strider/work/strider/file_discovery/en-token.bin 3 | english_sentenses=/Users/strider/work/strider/file_discovery/en-sent.bin 4 | generic=/Users/strider/work/strider/file_discovery/en-ner-person.bin 5 | location=/Users/strider/work/strider/file_discovery/en-ner-location.bin 6 | date=/Users/strider/work/strider/file_discovery/en-ner-date.bin 7 | money=/Users/strider/work/strider/file_discovery/en-ner-money.bin 8 | organization=/Users/strider/work/strider/file_discovery/en-ner-organization.bin 9 | time=/Users/strider/work/strider/file_discovery/en-ner-time.bin 10 | names=/Users/strider/work/strider/file_discovery/names.dict 11 | limit=100 12 | models=generic,money,time,location,organization 13 | directories=/Users/strider/work/strider/file_discovery/files 14 | extensions=com.strider.datadefender.specialcase.SinDetector.detectSin 15 | threshold_count=6 16 | threshold_highrisk=3 17 | score_calculation=yes -------------------------------------------------------------------------------- /src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | status=error 2 | dest=err 3 | 4 | appender.console.type=Console 5 | appender.console.name=STDOUT 6 | appender.console.layout.type=PatternLayout 7 | appender.console.layout.pattern=[%highlight{%p}{FATAL=red, ERROR=red, WARN=yellow bright, INFO=blue bright, DEBUG=green bright}] %m%n 8 | appender.console.filter.threshold.type=DynamicThresholdFilter 9 | appender.console.filter.threshold.defaultThreshold=warn 10 | appender.console.filter.threshold.key=console-level 11 | appender.console.filter.threshold.keyValuePair.type=KeyValuePair 12 | appender.console.filter.threshold.keyValuePair.key=info 13 | appender.console.filter.threshold.keyValuePair.value=info 14 | appender.console.filter.threshold.keyValuePair2.type=KeyValuePair 15 | appender.console.filter.threshold.keyValuePair2.key=debug 16 | appender.console.filter.threshold.keyValuePair2.value=debug 17 | appender.console.filter.threshold.onMatch=ACCEPT 18 | appender.console.filter.threshold.onMismatch=DENY 19 | 20 | appender.rolling.type=RollingFile 21 | appender.rolling.name=FILE 22 | appender.rolling.fileName=logs/datadefender.log 23 | appender.rolling.filePattern=logs/datadefender-%d{yy-MM-dd-hh-mm-ss}.log 24 | appender.rolling.layout.type=PatternLayout 25 | appender.rolling.layout.pattern=%d{HH:mm:ss} %c{2.} [%p] - %m%n 26 | appender.rolling.policies.type=OnStartupTriggeringPolicy 27 | appender.rolling.filter.threshold.type=DynamicThresholdFilter 28 | appender.rolling.filter.threshold.defaultThreshold=info 29 | appender.rolling.filter.threshold.key=file-level 30 | appender.rolling.filter.threshold.keyValuePair.type=KeyValuePair 31 | appender.rolling.filter.threshold.keyValuePair.key=debug 32 | appender.rolling.filter.threshold.keyValuePair.value=debug 33 | appender.rolling.filter.threshold.onMatch=ACCEPT 34 | appender.rolling.filter.threshold.onMismatch=DENY 35 | 36 | rootLogger.level=all 37 | rootLogger.appenderRef.file.ref=FILE 38 | rootLogger.appenderRef.console.ref=STDOUT 39 | -------------------------------------------------------------------------------- /src/main/resources/phi.txt: -------------------------------------------------------------------------------- 1 | Schizophrenia 2 | Schizophrenic 3 | Depression 4 | Depressed 5 | Police 6 | RCMP 7 | Suicide 8 | Suicidal 9 | Psycho 10 | -------------------------------------------------------------------------------- /src/run-scripts/datadefender: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -cp ".:extensions/*:extensions:lib:DataDefender.jar" com.strider.datadefender.DataDefender "$@" 3 | -------------------------------------------------------------------------------- /src/run-scripts/datadefender.bat: -------------------------------------------------------------------------------- 1 | java -cp ".;extensions/*;extensions;lib;DataDefender.jar" com.strider.datadefender.DataDefender %* 2 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/anonymizer/functions/LipsumTest.java: -------------------------------------------------------------------------------- 1 | package com.strider.datadefender.anonymizer.functions; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import static org.junit.jupiter.api.Assertions.*; 5 | 6 | /** 7 | * 8 | * @author Zaahid Bateson 9 | */ 10 | public class LipsumTest { 11 | 12 | @Test 13 | public void testSentences() throws Exception { 14 | Lipsum test = new Lipsum(); 15 | String sentences = test.sentences(3, 3); 16 | assertTrue(sentences.matches("([^\\.]+\\.){3}")); 17 | } 18 | 19 | @Test 20 | public void testParagraphs() throws Exception { 21 | Lipsum test = new Lipsum(); 22 | String paragraphs = test.paragraphs(3); 23 | assertTrue(paragraphs.matches("^[^\r]+\r\n\r\n[^\r]+\r\n\r\n[^\r]+$")); 24 | } 25 | 26 | @Test 27 | public void testSimilar() throws Exception { 28 | Lipsum test = new Lipsum(); 29 | 30 | String sentences = test.similar("This is a test. It is excellent. Should have a minimum of two sentences."); 31 | System.out.println("Testing for 3 sentences generated by similar text"); 32 | assertTrue(sentences.matches("([^\\.]+\\.){2,}")); 33 | 34 | String paras = test.similar("This is also a test.\n\nIt is better.\n\nShould have three paragraphs."); 35 | System.out.println("Testing for 3 paragraphs generated by text with 3 paragraphs"); 36 | assertTrue(paras.matches("^[^\r]+\r\n\r\n[^\r]+\r\n\r\n[^\r]+$")); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/database/DbConnectionTest.java: -------------------------------------------------------------------------------- 1 | package com.strider.datadefender.database; 2 | 3 | import com.strider.datadefender.DbConfig; 4 | 5 | import java.sql.Connection; 6 | 7 | import org.junit.jupiter.api.Test; 8 | import org.junit.jupiter.api.extension.ExtendWith; 9 | import org.mockito.junit.jupiter.MockitoExtension; 10 | import org.mockito.Mock; 11 | 12 | import static org.junit.jupiter.api.Assertions.*; 13 | import static org.mockito.Mockito.*; 14 | 15 | /** 16 | * Using mock to test Connection. 17 | * @author Akira Matsuo 18 | */ 19 | @ExtendWith(MockitoExtension.class) 20 | public class DbConnectionTest { 21 | 22 | @Mock 23 | DbConfig config; 24 | @Mock 25 | Connection mockConnection; 26 | 27 | @Test 28 | public void testConnect() throws Exception { 29 | final DbConnection test = new DbConnection(config) { 30 | @Override 31 | public Connection connect() throws DatabaseException { 32 | return doConnect(() -> mockConnection); 33 | } 34 | }; 35 | assertEquals(mockConnection, test.connect()); 36 | verify(mockConnection).setAutoCommit(false); 37 | } 38 | 39 | @Test 40 | public void testConnectSupplier() throws Exception { 41 | when(config.getUrl()).thenReturn("jdbc:h2:mem:utest;MODE=MySQL;DB_CLOSE_DELAY=-1"); 42 | when(config.getUsername()).thenReturn("username"); 43 | when(config.getUsername()).thenReturn("password"); 44 | 45 | final DbConnection test = new DbConnection(config); 46 | final Connection con = test.connect(); 47 | assertNotNull(con); 48 | assertFalse(con.isClosed()); 49 | assertFalse(con.getAutoCommit()); 50 | 51 | verify(config, atLeast(1)).getUrl(); 52 | verify(config, atLeast(1)).getUsername(); 53 | verify(config, atLeast(1)).getPassword(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/database/IDbFactoryTest.java: -------------------------------------------------------------------------------- 1 | package com.strider.datadefender.database; 2 | 3 | import com.strider.datadefender.DataDefenderException; 4 | import com.strider.datadefender.DbConfig; 5 | import com.strider.datadefender.DbConfig.Vendor; 6 | 7 | import org.junit.jupiter.api.Test; 8 | import org.junit.jupiter.api.extension.ExtendWith; 9 | import org.mockito.Mock; 10 | import org.mockito.junit.jupiter.MockitoExtension; 11 | 12 | import static org.junit.jupiter.api.Assertions.*; 13 | import static org.mockito.Mockito.*; 14 | 15 | /** 16 | * @author Akira Matsuo 17 | */ 18 | @ExtendWith(MockitoExtension.class) 19 | public class IDbFactoryTest { 20 | 21 | @Mock 22 | DbConfig config; 23 | 24 | @Test 25 | public void testInvalidNoProps() { 26 | when(config.getVendor()).thenReturn(Vendor.H2); 27 | assertThrows(DatabaseException.class, () -> IDbFactory.get(config).getConnection()); 28 | } 29 | 30 | @Test 31 | public void testInvalidProps() throws DatabaseException, DataDefenderException { 32 | when(config.getVendor()).thenReturn(null); 33 | assertThrows(IllegalArgumentException.class, () -> 34 | IDbFactory.get(config).getConnection() 35 | ); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/database/metadata/MetaDataTest.java: -------------------------------------------------------------------------------- 1 | package com.strider.datadefender.database.metadata; 2 | 3 | import com.strider.datadefender.DbConfig; 4 | import com.strider.datadefender.database.metadata.TableMetaData.ColumnMetaData; 5 | 6 | import java.sql.Connection; 7 | import java.sql.DatabaseMetaData; 8 | import java.sql.ResultSet; 9 | import java.sql.Types; 10 | import java.util.List; 11 | 12 | import org.junit.jupiter.api.BeforeEach; 13 | import org.junit.jupiter.api.Test; 14 | import org.junit.jupiter.api.extension.ExtendWith; 15 | import org.mockito.junit.jupiter.MockitoExtension; 16 | import org.mockito.Mock; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | import static org.mockito.Mockito.*; 20 | 21 | /** 22 | * 23 | * @author Akira Matsuo 24 | * 25 | * Tests the default implementation of MetaData that's used for Oracle and MSSQL. 26 | * 27 | */ 28 | @ExtendWith(MockitoExtension.class) 29 | public class MetaDataTest { 30 | 31 | @Mock 32 | Connection mockConnection; 33 | @Mock 34 | DbConfig mockConfig; 35 | @Mock 36 | SqlTypeToClass mockSqlTypeMap; 37 | @Mock 38 | ResultSet mockTableRs; 39 | @Mock 40 | ResultSet mockColumnRs; 41 | @Mock 42 | DatabaseMetaData mockDbMetaData; 43 | @Mock 44 | ResultSet emptyRs; 45 | 46 | @BeforeEach 47 | public void setUp() throws Exception { 48 | 49 | System.out.println("Setting up..."); 50 | 51 | when(mockConfig.getSchema()).thenReturn("such-schema"); 52 | when(mockConnection.getMetaData()).thenReturn(mockDbMetaData); 53 | 54 | when(mockDbMetaData.getTables(any(), any(), any(), any())).thenReturn(mockTableRs); 55 | when(mockTableRs.next()).thenReturn(Boolean.TRUE).thenReturn(false); 56 | when(mockTableRs.getString("TABLE_NAME")).thenReturn("such-table"); 57 | 58 | when(mockDbMetaData.getPrimaryKeys(any(), any(), any())).thenReturn(emptyRs); 59 | when(mockDbMetaData.getImportedKeys(any(), any(), any())).thenReturn(emptyRs); 60 | when(emptyRs.next()).thenReturn(false); 61 | 62 | when(mockDbMetaData.getColumns(any(), any(), any(), any())).thenReturn(mockColumnRs); 63 | when(mockColumnRs.next()).thenReturn(Boolean.TRUE).thenReturn(Boolean.FALSE); 64 | when(mockColumnRs.getString("COLUMN_NAME")).thenReturn("such-column"); 65 | when(mockColumnRs.getInt("COLUMN_SIZE")).thenReturn(1337); 66 | when(mockColumnRs.getInt("DATA_TYPE")).thenReturn(Types.CHAR); 67 | when(mockSqlTypeMap.getTypeFrom(Types.CHAR)).thenReturn(String.class); 68 | } 69 | 70 | @Test 71 | public void testGetMetaData() throws Throwable { 72 | 73 | MetaData test = new MetaData(mockConfig, mockConnection, mockSqlTypeMap); 74 | List ret = test.getMetaData(); 75 | 76 | verify(mockDbMetaData, times(1)).getTables(isNull(), eq("such-schema"), isNull(), eq(new String[] { "TABLE" })); 77 | verify(mockTableRs, times(2)).next(); 78 | verify(mockDbMetaData, times(1)).getColumns(isNull(), eq("such-schema"), eq("such-table"), isNull()); 79 | verify(mockColumnRs, times(2)).next(); 80 | verify(mockColumnRs).getString(eq("COLUMN_NAME")); 81 | verify(mockColumnRs).getInt(eq("COLUMN_SIZE")); 82 | verify(mockColumnRs).getInt(eq("DATA_TYPE")); 83 | 84 | assertNotNull(ret); 85 | assertEquals(1, ret.size()); 86 | assertNotNull(ret.get(0)); 87 | assertEquals("such-schema", ret.get(0).getSchemaName()); 88 | assertEquals("such-table", ret.get(0).getTableName()); 89 | 90 | assertNotNull(ret.get(0).getColumns()); 91 | assertEquals(1, ret.get(0).getColumns().size()); 92 | 93 | ColumnMetaData col = ret.get(0).getColumn(1); 94 | assertNotNull(col); 95 | assertEquals("such-column", col.getColumnName()); 96 | assertSame(col, ret.get(0).getColumn("such-column")); 97 | assertEquals(1337, col.getColumnSize()); 98 | assertEquals(String.class, col.getColumnType()); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/database/sqlbuilder/SqlBuilderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this license header, choose License Headers in Project Properties. 3 | * To change this template file, choose Tools | Templates 4 | * and open the template in the editor. 5 | */ 6 | package com.strider.datadefender.database.sqlbuilder; 7 | 8 | import com.strider.datadefender.DbConfig; 9 | import org.junit.jupiter.api.Test; 10 | import org.junit.jupiter.api.extension.ExtendWith; 11 | import org.mockito.junit.jupiter.MockitoExtension; 12 | import org.mockito.Mock; 13 | 14 | import static org.junit.jupiter.api.Assertions.*; 15 | import static org.mockito.Mockito.*; 16 | 17 | /** 18 | * 19 | * @author Zaahid Bateson 20 | */ 21 | @ExtendWith(MockitoExtension.class) 22 | public class SqlBuilderTest { 23 | 24 | @Mock 25 | DbConfig mockConfig; 26 | 27 | @Test 28 | public void testBuildSelectWithLimit() { 29 | SqlBuilder test = new SqlBuilder(mockConfig); 30 | String sql = test.buildSelectWithLimit("toost", 1); 31 | assertEquals("toost LIMIT 1", sql); 32 | } 33 | 34 | @Test 35 | public void testBuildSelectWithZeroLimit() { 36 | SqlBuilder test = new SqlBuilder(mockConfig); 37 | String sql = test.buildSelectWithLimit("toost", 0); 38 | assertEquals("toost", sql); 39 | } 40 | 41 | @Test 42 | public void testPrefixSchema() { 43 | when(mockConfig.getSchema()).thenReturn("scheems"); 44 | SqlBuilder test = new SqlBuilder(mockConfig); 45 | String tableName = test.prefixSchema("tablas"); 46 | assertEquals("scheems.tablas", tableName); 47 | } 48 | 49 | @Test 50 | public void testPrefixBlankSchema() { 51 | when(mockConfig.getSchema()).thenReturn(null).thenReturn("").thenReturn(" "); 52 | SqlBuilder test = new SqlBuilder(mockConfig); 53 | assertEquals("tablas", test.prefixSchema("tablas")); 54 | assertEquals("tablas", test.prefixSchema("tablas")); 55 | assertEquals("tablas", test.prefixSchema("tablas")); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/functions/UtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.functions; 17 | 18 | import com.strider.datadefender.anonymizer.DatabaseAnonymizerException; 19 | 20 | import org.junit.jupiter.api.Test; 21 | import static org.junit.jupiter.api.Assertions.*; 22 | 23 | /** 24 | * @author Armenak Grigoryan 25 | */ 26 | public class UtilsTest { 27 | 28 | /** 29 | * Initializes logger 30 | */ 31 | private final String FULL_CLASS_NAME = "com.strider.datadefender.anonymizer.functions.Core"; 32 | private final String EXPECTED_RESULTS_1 = "com.strider.datadefender.anonymizer.functions"; 33 | private final String EXPECTED_RESULTS_2 = "Core"; 34 | 35 | @Test 36 | public void testGetClassName() throws Exception { 37 | System.out.println("Executing testGetClassName..."); 38 | final String result = Utils.getClassName(FULL_CLASS_NAME); 39 | assertEquals(EXPECTED_RESULTS_1, result); 40 | } 41 | 42 | @Test 43 | public void testGetMethodName() throws DatabaseAnonymizerException { 44 | System.out.println("Executing testGetMethodName..."); 45 | final String result = Utils.getMethodName(FULL_CLASS_NAME); 46 | assertEquals(EXPECTED_RESULTS_2, result); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/utils/EncoderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014-2021, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | 17 | package com.strider.datadefender.utils; 18 | 19 | 20 | import java.io.IOException; 21 | 22 | import lombok.extern.log4j.Log4j2; 23 | 24 | import org.junit.jupiter.api.AfterEach; 25 | import org.junit.jupiter.api.AfterAll; 26 | import org.junit.jupiter.api.BeforeEach; 27 | import org.junit.jupiter.api.BeforeAll; 28 | import org.junit.jupiter.api.Test; 29 | import static org.junit.jupiter.api.Assertions.*; 30 | 31 | import com.strider.datadefender.anonymizer.functions.Bio; 32 | import java.io.BufferedReader; 33 | import java.io.File; 34 | import java.io.FileReader; 35 | 36 | /** 37 | * 38 | * @author strider 39 | */ 40 | @Log4j2 41 | public class EncoderTest { 42 | 43 | private static String hash = null; 44 | 45 | /** 46 | * Static block to load hash string for "salting" the encryption 47 | */ 48 | static { 49 | String file = System.getProperty("user.dir") + "/hash.txt"; 50 | File f = new File(file); 51 | try { 52 | BufferedReader reader = new BufferedReader(new FileReader(f)); 53 | hash = reader.readLine(); 54 | reader.close(); 55 | } catch (IOException ex) { 56 | log.error("Problem finding file hash.txt", ex); 57 | } 58 | } 59 | 60 | 61 | public EncoderTest() { 62 | } 63 | 64 | @BeforeAll 65 | public static void setUpClass() { 66 | } 67 | 68 | @AfterAll 69 | public static void tearDownClass() { 70 | } 71 | 72 | @BeforeEach 73 | public void setUp() { 74 | } 75 | 76 | @AfterEach 77 | public void tearDown() { 78 | } 79 | 80 | /** 81 | * Test of encode method, of class Encoder. 82 | */ 83 | @Test 84 | public void testRandomFirstNameWithParameter() { 85 | log.debug("Testing enocode() method"); 86 | 87 | String originalValue = "Armenak Grigoryan"; 88 | 89 | String encryptedValue = new Bio().randomFirstName(originalValue); 90 | log.debug("Encrypted value=" + encryptedValue); 91 | 92 | String decryptedValue = new Encoder().decrypt(encryptedValue, hash); 93 | log.debug("Decrypted value=" + decryptedValue); 94 | 95 | assertEquals(originalValue, decryptedValue); 96 | } 97 | 98 | @Test 99 | public void testRandomFirstNameWithoutParameter() { 100 | log.debug("Testing randomFirstName() method"); 101 | 102 | String randomFirstName = ""; 103 | try { 104 | randomFirstName = new Bio().randomFirstName(); 105 | } catch (IOException ex) { 106 | log.error("Problem generating random first name", ex); 107 | } 108 | 109 | assertTrue(randomFirstName != null); 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/utils/LikeMatcherTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014, Armenak Grigoryan, and individual contributors as indicated 3 | * by the @authors tag. See the copyright.txt in the distribution for a 4 | * full listing of individual contributors. 5 | * 6 | * This is free software; you can redistribute it and/or modify it 7 | * under the terms of the GNU Lesser General Public License as 8 | * published by the Free Software Foundation; either version 2.1 of 9 | * the License, or (at your option) any later version. 10 | * 11 | * This software is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | * Lesser General Public License for more details. 15 | */ 16 | package com.strider.datadefender.utils; 17 | 18 | import org.junit.jupiter.api.Test; 19 | import static org.junit.jupiter.api.Assertions.*; 20 | 21 | /** 22 | * 23 | * @author Zaahid Bateson 24 | */ 25 | public class LikeMatcherTest { 26 | private static final String TEST_NAME_1 = "ZaahiD"; 27 | 28 | @Test 29 | public void testMatchingAtBeginning() { 30 | System.out.println("Testing match begginning of string"); 31 | 32 | final LikeMatcher matcher = new LikeMatcher("%hid"); 33 | 34 | assertTrue(matcher.matches(TEST_NAME_1)); 35 | assertTrue(matcher.matches("aaHID")); 36 | assertTrue(matcher.matches("HID")); 37 | assertFalse(matcher.matches("Zaahid is not here")); 38 | assertFalse(matcher.matches("hiding away")); 39 | } 40 | 41 | @Test 42 | public void testMatchingAtEnd() { 43 | System.out.println("Testing match at end of string"); 44 | 45 | final LikeMatcher matcher = new LikeMatcher("za%"); 46 | 47 | assertTrue(matcher.matches(TEST_NAME_1)); 48 | assertTrue(matcher.matches("ZAA")); 49 | assertTrue(matcher.matches("za")); 50 | assertFalse(matcher.matches("Not Zaahid")); 51 | assertFalse(matcher.matches("Some Za")); 52 | } 53 | 54 | @Test 55 | public void testMixedMatcher() { 56 | System.out.println("Testing match with a mix of '%', '?', and '_'"); 57 | 58 | final LikeMatcher matcher = new LikeMatcher("%Z_?h?d%"); 59 | 60 | assertTrue(matcher.matches(TEST_NAME_1)); 61 | assertFalse(matcher.matches("ZaHID")); 62 | assertFalse(matcher.matches("Zhd")); 63 | assertTrue(matcher.matches("Zaphod")); 64 | assertTrue(matcher.matches("Zaahid really is Zaphod")); 65 | assertTrue(matcher.matches("Yes, Zaahid is here")); 66 | } 67 | 68 | @Test 69 | public void testMultiMatcher() { 70 | System.out.println("Testing match with multiple %'s"); 71 | 72 | final LikeMatcher matcher = new LikeMatcher("Z%h%d"); 73 | 74 | assertTrue(matcher.matches(TEST_NAME_1)); 75 | assertTrue(matcher.matches("ZaHID")); 76 | assertTrue(matcher.matches("Zhd")); 77 | assertFalse(matcher.matches("Zaahid is not here")); 78 | assertFalse(matcher.matches("Someone is hiding Zaahid")); 79 | } 80 | 81 | @Test 82 | public void testSingleCharMatcher() { 83 | System.out.println("Testing match with '?' and '_'"); 84 | 85 | final LikeMatcher matcher = new LikeMatcher("Z_?h?d"); 86 | 87 | assertTrue(matcher.matches(TEST_NAME_1)); 88 | assertFalse(matcher.matches("ZaHID")); 89 | assertFalse(matcher.matches("Zhd")); 90 | assertTrue(matcher.matches("Zaphod")); 91 | } 92 | } 93 | 94 | 95 | //~ Formatted by Jindent --- http://www.jindent.com 96 | -------------------------------------------------------------------------------- /src/test/java/com/strider/datadefender/utils/XegerTest.java: -------------------------------------------------------------------------------- 1 | package com.strider.datadefender.utils; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import static org.junit.jupiter.api.Assertions.*; 5 | 6 | /** 7 | * 8 | * @author strider 9 | */ 10 | public class XegerTest { 11 | 12 | private static final String REGEXP_1 = "[ab]{4,6}c"; 13 | private static final String REGEXP_2 = "[0-9]{3}-[0-9]{3}-[0-9]{3}"; 14 | 15 | /** 16 | * Test of generate method, of class Xeger. 17 | */ 18 | @Test 19 | public void testGenerate() { 20 | System.out.println("Generate string"); 21 | 22 | final Xeger instance = new Xeger(REGEXP_1); 23 | 24 | for (int i = 0; i < 100; i++) { 25 | final String text = instance.generate(); 26 | 27 | assertTrue(text.matches(REGEXP_1)); 28 | } 29 | } 30 | 31 | /** 32 | * Test of generate method, of class Xeger. 33 | */ 34 | @Test 35 | public void testGenerateSIN() { 36 | System.out.println("Generate SIN"); 37 | 38 | final Xeger instance = new Xeger(REGEXP_2); 39 | final String text = instance.generate(); 40 | 41 | System.out.println(text); 42 | assertTrue(text.matches(REGEXP_2)); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/resources/AppPropertiesTest.properties: -------------------------------------------------------------------------------- 1 | # sample test properties file 2 | xxx=yyy -------------------------------------------------------------------------------- /src/test/resources/Requirement-H2DB-DG.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Test H2DB Client 4 | 1.0 5 | 6 |
7 | 8 | 9 | com.strider.datadefender.functions.CoreFunctions.randomFirstName 10 | 11 | 12 | 13 | 14 | 15 | com.strider.datadefender.functions.CoreFunctions.randomFirstName 16 | 17 | 18 | 19 | 20 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /src/test/resources/Requirement-H2DB.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Test H2DB Client 4 | 1.0 5 | 6 | 7 | 8 | 9 | com.strider.datadefender.functions.CoreFunctions.randomFirstName 10 | 11 | 12 | 13 | 14 | 15 | com.strider.datadefender.functions.CoreFunctions.randomLastName 16 | 17 | 18 | 19 | 20 | 21 |
22 | 23 | 24 | -------------------------------------------------------------------------------- /src/test/resources/core-test.txt: -------------------------------------------------------------------------------- 1 | toost 2 | bloost 3 | -------------------------------------------------------------------------------- /src/test/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | status=error 2 | dest=err 3 | 4 | appender.console.type=Console 5 | appender.console.name=STDOUT 6 | appender.console.layout.type=PatternLayout 7 | appender.console.layout.pattern=%d{HH:mm:ss} %c{2.} [%p] - %m%n 8 | 9 | rootLogger.level=debug 10 | rootLogger.appenderRef.console.ref=STDOUT 11 | -------------------------------------------------------------------------------- /src/test/resources/names.txt: -------------------------------------------------------------------------------- 1 | mysql 2 | mssql 3 | oracle 4 | h2db --------------------------------------------------------------------------------