├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── config ├── config.properties └── credentials ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── amazonaws │ │ └── services │ │ └── dynamodbv2 │ │ └── online │ │ └── index │ │ ├── AWSConnection.java │ │ ├── AttributeValueConverter.java │ │ ├── Correction.java │ │ ├── CorrectionReader.java │ │ ├── OptionChecker.java │ │ ├── OptionLoader.java │ │ ├── Options.java │ │ ├── PrintHelper.java │ │ ├── TableHelper.java │ │ ├── TableRWRateLimiter.java │ │ ├── TableReader.java │ │ ├── TableWriter.java │ │ ├── ViolationChecker.java │ │ ├── ViolationDetector.java │ │ ├── ViolationRecord.java │ │ └── ViolationWriter.java └── resources │ └── log4j.properties └── test └── java └── com └── amazonaws └── services └── dynamodbv2 └── online └── index ├── AWSConnectionTest.java ├── CorrectionReaderTest.java ├── CorrectionTest.java ├── OptionCheckerTest.java ├── OptionLoaderTest.java ├── OptionsTest.java ├── RandomDataGenerator.java ├── TableHelperTest.java ├── TableReaderTest.java ├── TableWriterTest.java ├── ViolationCheckerTest.java ├── ViolationRecordTest.java └── integration └── tests ├── TableManager.java ├── TestUtils.java ├── ViolationCorrectionTest.java └── ViolationDetectionTest.java /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | 4 | Version 2.0, January 2004 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | “License” shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | “Licensor” shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 13 | 14 | “Legal Entity” shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 15 | 16 | “You” (or “Your”) shall mean an individual or Legal Entity exercising permissions granted by this License. 17 | 18 | “Source” form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 19 | 20 | “Object” form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 21 | 22 | “Work” shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 23 | 24 | “Derivative Works” shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 25 | 26 | “Contribution” shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as “Not a Contribution.” 27 | 28 | “Contributor” shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 29 | 30 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 31 | 32 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 33 | 34 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 35 | 36 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 37 | You must cause any modified files to carry prominent notices stating that You changed the files; and 38 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 39 | If the Work includes a “NOTICE” text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 40 | 41 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 42 | 43 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 44 | 45 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 46 | 47 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 48 | 49 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 50 | 51 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 52 | 53 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | DynamoDB Online Index Violation Detector 2 | Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | For an overview of the DynamoDB Online Index Violation Detector Tool, please refer to the [AWS DynamoDB Documentation](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.OnlineOps.ViolationDetection.html). 3 | 4 | ## Minimum Requirements 5 | - Java 1.7+ 6 | - Maven 7 | 8 | ## Getting Started 9 | To download the code and to build and create a runnable jar, use the following commands: 10 | ``` 11 | git clone https://github.com/awslabs/dynamodb-online-index-violation-detector.git 12 | cd dynamodb-online-index-violation-detector 13 | mvn package 14 | ``` 15 | This will generate a jar named ViolationDetector.jar in the dynamodb-online-index-violation-detector directory. 16 | 17 | ## Usage 18 | You can run the generated jar named ViolationDetector.jar to start detecting and correcting violations on any DynamoDB table. 19 | ``` 20 | java -jar ViolationDetector.jar [options] 21 | ``` 22 | ### Available options: 23 | - -p,--configFilePath \ 24 | - Path of the config file. This option is required for both detection and correction. Refer to the sample [config.properties](https://github.com/awslabs/dynamodb-online-index-violation-detector/tree/master/config/config.properties) file. 25 | - -t,--detect \ 26 | - Detect violations on given table. With 'keep', violations will be kept and recorded. With 'delete', violations will be deleted and recorded. 27 | - -c,--correct \ 28 | - Correct violations based on records on correction input file. With 'delete', records on input file will be deleted from the table. With 'update', records on input file will be updated to the table. 29 | - -h,--help 30 | - Help and usage information 31 | 32 | For detailed instructions, refer to the [AWS DynamoDB Documentation](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.OnlineOps.ViolationDetection.html). 33 | 34 | ## Running Integration Tests 35 | In order to run integration tests, you will have to start DynamoDB Local server on port 8000. For downloading and starting DynamoDB Local, refer to the instructions given in the [AWS DynamoDB Local Documentation](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html). 36 | Once you have the server running, run the following command to start integration tests: 37 | ``` 38 | mvn integration-test 39 | ``` 40 | 41 | ## Limitations 42 | - The 'recordDetails' option with value set as 'true' does not work for binary attribute values that cannot be encoded using UTF-8 character set. 43 | -------------------------------------------------------------------------------- /config/config.properties: -------------------------------------------------------------------------------- 1 | # Properties file for violation detection tool configuration. 2 | 3 | # Path of AWS credentials file. For security reason, we do not ask for 4 | # access key or range key directly. 5 | awsCredentialsFile = ./config/credentials 6 | 7 | # AWS region for the table. 8 | dynamoDBRegion = us-west-2 9 | 10 | # Table name. 11 | tableName = table-to-test 12 | 13 | # GSI hash key name. 14 | # Use this option only if GSI hash key will be checked. 15 | # Remove trailing spaces if they are not part of the name 16 | gsiHashKeyName = gsiHashKeyName 17 | 18 | # GSI hash key type. 19 | # Use this option only if GSI hash key will be checked. 20 | # Valid values: N, S, B 21 | gsiHashKeyType = gsiHashKeyType 22 | 23 | # GSI range key name. 24 | # Use this option only if GSI range key will be checked. 25 | # Remove trailing spaces if they are not part of the name 26 | gsiRangeKeyName = gsiRangeKeyName 27 | 28 | # GSI range key type. 29 | # Use this option only if GSI range key will be checked. 30 | # Valid values: N, S, B 31 | gsiRangeKeyType = gsiRangeKeyType 32 | 33 | # 'true' to record violation details to output file. 'false' to only provide 34 | # the number of violations. 35 | # Valid: 'true' or 'false'. 36 | # Default value: 'true'. 37 | recordDetails = true 38 | 39 | # Set it to 'true' if violated values are required in the output file. 40 | # You will need this if you wish to run Violation correction afterwards, with conditional update as 'true' 41 | # Valid: 'true' or 'false' 42 | # Default value: 'false' 43 | recordGsiValueInViolationRecord = false 44 | 45 | # Output path of detection output file. Supports both local directory 46 | # and S3 path with filename ending with '.csv' 47 | # Example: 48 | # Local file: //local/path/myoutput.csv 49 | # S3 path: s3://bucket/myoutput.csv 50 | # Default value: ./violation_detection.csv 51 | detectionOutputPath = ./gsi_violation_check.csv 52 | 53 | # Number of segments indicates the number of threads that will be created for parallel scan. 54 | # If = 1, sequential scan will be used; If > 1, parallel scan will be used. 55 | # Valid: 1 ~ 4096. 56 | # Default value: 1. 57 | numOfSegments = 1 58 | 59 | # Number of violations to be scanned. Scan will stop when given number of 60 | # violations are found. 61 | # This is optional. Default value will be used when commented. 62 | # Default value: -1, will scan the entire table. 63 | numOfViolations = -1 64 | 65 | # Number of records to be scanned. Scan will stop when given number of items 66 | # are scanned. 67 | # This is optional. Default value will be used when commented. 68 | # Default value: -1, will scan the entire table. 69 | numOfRecords = -1 70 | 71 | # Percentage of provisioned read/write IOPS of the table that scan/update 72 | # (delete) operations will use during detection/correction. 73 | # This is optional. Default value will be used when commented. 74 | # Valid: 1 ~ 100, integer. 75 | # Default value: 25. 76 | readWriteIOPSPercent = 25 77 | 78 | # Input file path for violation correction 79 | correctionInputPath = ./gsi_violation_check.csv 80 | 81 | # Output file path for violation correction in update mode. 82 | # It is generated only if there are update errors. 83 | # Supports both local directory and S3 path with filename ending with '.csv'. 84 | # Example: 85 | # Local file: //local/path/myoutput.csv 86 | # S3 path: s3://bucket/myoutput.csv 87 | # Default value: ./violation_update_errors.csv 88 | correctionOutputPath = ./gsi_violation_check_result.csv 89 | -------------------------------------------------------------------------------- /config/credentials: -------------------------------------------------------------------------------- 1 | accessKey:foo 2 | secretKey:bar 3 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | com.amazonaws.services.dynamodbv2 6 | dynamodb-online-index-violation-detector 7 | jar 8 | DynamoDB Online Index Violation Detector 9 | 0.1.0 10 | A tool for Amazon DynamoDB to find violations on an online GSI's hash key and range key. 11 | https://aws.amazon.com/dynamodb 12 | 13 | 14 | https://github.com/awslabs/dynamodb-online-index-violation-detector.git 15 | 16 | 17 | 18 | 19 | Apache License, Version 2.0 20 | https://aws.amazon.com/apache2.0 21 | repo 22 | 23 | 24 | 25 | 26 | UTF-8 27 | 28 | 29 | 30 | 31 | junit 32 | junit 33 | 4.1 34 | test 35 | 36 | 37 | 38 | org.mockito 39 | mockito-all 40 | 1.9.5 41 | test 42 | 43 | 44 | 45 | log4j 46 | log4j 47 | 1.2.15 48 | 49 | 50 | com.sun.jmx 51 | jmxri 52 | 53 | 54 | com.sun.jdmk 55 | jmxtools 56 | 57 | 58 | javax.jms 59 | jms 60 | 61 | 62 | 63 | 64 | 65 | commons-cli 66 | commons-cli 67 | 1.2 68 | 69 | 70 | 71 | org.apache.commons 72 | commons-csv 73 | 1.1 74 | 75 | 76 | 77 | com.google.guava 78 | guava 79 | 18.0 80 | 81 | 82 | 83 | com.amazonaws 84 | aws-java-sdk 85 | 1.9.13 86 | 87 | 88 | 89 | 90 | 91 | 92 | org.apache.maven.plugins 93 | maven-assembly-plugin 94 | 2.4 95 | 96 | ViolationDetector 97 | ../${project.artifactId} 98 | 99 | 100 | com.amazonaws.services.dynamodbv2.online.index.ViolationDetector 101 | 102 | 103 | 104 | jar-with-dependencies 105 | 106 | false 107 | false 108 | 109 | 110 | 111 | make-assembly 112 | package 113 | 114 | single 115 | 116 | 117 | 118 | 119 | 120 | 121 | org.apache.maven.plugins 122 | maven-surefire-plugin 123 | 2.18 124 | 125 | 126 | **/integration/tests/*Test.java 127 | 128 | 129 | 130 | 131 | integration-tests 132 | integration-test 133 | 134 | test 135 | 136 | 137 | false 138 | 139 | None 140 | 141 | 142 | **/integration/tests/*Test.java 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/AWSConnection.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.io.FileInputStream; 18 | import java.io.FileNotFoundException; 19 | import java.io.IOException; 20 | 21 | import com.amazonaws.auth.AWSCredentials; 22 | import com.amazonaws.auth.PropertiesCredentials; 23 | import com.amazonaws.regions.Region; 24 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 25 | import com.amazonaws.services.s3.AmazonS3Client; 26 | 27 | /** 28 | * Providing AWS Clients, including DynamoDB and S3. 29 | * 30 | */ 31 | public class AWSConnection { 32 | 33 | private AWSCredentials awsCredentials; 34 | public static final String DDB_LOCAL_ENDPOINT = "http://localhost:8000"; 35 | 36 | /** 37 | * Constructor for unit tests 38 | */ 39 | protected AWSConnection(AWSCredentials awsCredentials) { 40 | this.awsCredentials = awsCredentials; 41 | } 42 | 43 | public AWSConnection(String credentialFilePath) 44 | throws FileNotFoundException, IOException { 45 | this.awsCredentials = loadCredentialFile(credentialFilePath); 46 | } 47 | 48 | private AWSCredentials loadCredentialFile(String credentialsFilePath) throws IllegalArgumentException { 49 | try { 50 | return new PropertiesCredentials(getInputStream(credentialsFilePath)); 51 | } catch (IOException ioe) { 52 | throw new IllegalArgumentException("Error: Failed to read credential file " + credentialsFilePath); 53 | } 54 | } 55 | 56 | private FileInputStream getInputStream(String credentialsFilePath) throws FileNotFoundException { 57 | try { 58 | return new FileInputStream(credentialsFilePath); 59 | } catch (FileNotFoundException ffe) { 60 | throw new IllegalArgumentException("Error: Credential file " + credentialsFilePath + " not Found."); 61 | } 62 | } 63 | 64 | public AmazonDynamoDBClient getDynamoDBClient(Region dynamoDBTableRegion, boolean runOnDDBLocal) { 65 | AmazonDynamoDBClient dynamoDBClient = new AmazonDynamoDBClient(awsCredentials); 66 | dynamoDBClient.setRegion(dynamoDBTableRegion); 67 | if(runOnDDBLocal) { 68 | dynamoDBClient.setEndpoint(DDB_LOCAL_ENDPOINT); 69 | } 70 | return dynamoDBClient; 71 | } 72 | 73 | public AmazonS3Client getS3Client() { 74 | return new AmazonS3Client(awsCredentials); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/AttributeValueConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.io.IOException; 18 | import java.nio.ByteBuffer; 19 | import java.nio.CharBuffer; 20 | import java.nio.charset.CharacterCodingException; 21 | import java.nio.charset.Charset; 22 | import java.nio.charset.CharsetDecoder; 23 | import java.nio.charset.CharsetEncoder; 24 | import java.util.ArrayList; 25 | import java.util.HashMap; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 30 | import com.fasterxml.jackson.core.JsonProcessingException; 31 | import com.fasterxml.jackson.core.type.TypeReference; 32 | import com.fasterxml.jackson.databind.ObjectMapper; 33 | 34 | /** 35 | * Parse attribute values of DynamoDB. 36 | */ 37 | public class AttributeValueConverter { 38 | private static final Charset charset = Charset.forName("UTF-8"); 39 | private static CharsetDecoder decoder = charset.newDecoder(); 40 | private static CharsetEncoder encoder = charset.newEncoder(); 41 | private static final ObjectMapper mapper = new ObjectMapper(); 42 | 43 | public static AttributeValue parseFromWithAttributeTypeString(String value) throws IllegalArgumentException { 44 | try { 45 | Map valueMap = mapper.readValue(value, new TypeReference>(){}); 46 | AttributeValue attributeValue = new AttributeValue(); 47 | if (null != valueMap.get("N")) { 48 | attributeValue.withN(valueMap.get("N")); 49 | } else if (null != valueMap.get("S")) { 50 | attributeValue.withS(valueMap.get("S")); 51 | } else if (null != valueMap.get("B")) { 52 | attributeValue.withB(encoder.encode(CharBuffer.wrap(valueMap.get("B")))); 53 | } else if (null != valueMap.get("NS")) { 54 | List numberSet = mapper.readValue(valueMap.get("NS"), new TypeReference>(){}); 55 | attributeValue.withNS(numberSet); 56 | } else if (null != valueMap.get("SS")) { 57 | List stringSet = mapper.readValue(valueMap.get("SS"), new TypeReference>(){}); 58 | attributeValue.withSS(stringSet); 59 | } else if (null != valueMap.get("BS")) { 60 | List binaryStringSet = mapper.readValue(valueMap.get("BS"), new TypeReference>(){}); 61 | List bianrySet = new ArrayList(); 62 | for (String bss : binaryStringSet) { 63 | bianrySet.add(encoder.encode(CharBuffer.wrap(bss))); 64 | } 65 | attributeValue.withBS(bianrySet); 66 | } else { 67 | throw new IllegalArgumentException("Error: Invalid Attribute Value Type. "); 68 | } 69 | return attributeValue; 70 | } catch (CharacterCodingException cce) { 71 | throw new IllegalArgumentException("Error: Failed to encode binary into string."); 72 | } catch (IOException e) { 73 | throw new IllegalArgumentException("Error: Failed to parse set from string."); 74 | } 75 | } 76 | 77 | public static AttributeValue parseFromBlankString(String attribtueType, String value) throws IllegalArgumentException { 78 | try { 79 | AttributeValue attributeValue = new AttributeValue(); 80 | if (attribtueType.equals("N")) { 81 | attributeValue.withN(value); 82 | } else if (attribtueType.equals("S")) { 83 | attributeValue.withS(value); 84 | } else if (attribtueType.equals("B")) { 85 | attributeValue.withB(encoder.encode(CharBuffer.wrap(value))); 86 | } else if (attribtueType.equals("NS")) { 87 | List numberSet = mapper.readValue(value, new TypeReference>(){}); 88 | attributeValue.withNS(numberSet); 89 | } else if (attribtueType.equals("SS")) { 90 | List stringSet = mapper.readValue(value, new TypeReference>(){}); 91 | attributeValue.withSS(stringSet); 92 | } else if (attribtueType.equals("BS")) { 93 | List binaryStringSet = mapper.readValue(value, new TypeReference>(){}); 94 | List bianrySet = new ArrayList(); 95 | for (String bss : binaryStringSet) { 96 | bianrySet.add(encoder.encode(CharBuffer.wrap(bss))); 97 | } 98 | attributeValue.withBS(bianrySet); 99 | } else { 100 | throw new IllegalArgumentException("Error: Invalid Attribute Value Type. "); 101 | } 102 | return attributeValue; 103 | } catch (CharacterCodingException cce) { 104 | throw new IllegalArgumentException("Error: Failed to encode binary into string."); 105 | } catch (IOException e) { 106 | throw new IllegalArgumentException("Error: Failed to parse set from string."); 107 | } 108 | } 109 | public static String toStringWithAttributeType(AttributeValue attributeValue) throws IllegalArgumentException { 110 | try { 111 | Map valueMap = new HashMap(); 112 | if (null != attributeValue.getS()) { 113 | valueMap.put("S",attributeValue.getS()); 114 | } 115 | if (null != attributeValue.getN()) { 116 | valueMap.put("N", attributeValue.getN()); 117 | } 118 | if (null != attributeValue.getB()) { 119 | valueMap.put("B", decoder.decode(attributeValue.getB()).toString()); 120 | } 121 | if (null != attributeValue.getNS()) { 122 | valueMap.put("NS", mapper.writeValueAsString(attributeValue.getNS())); 123 | } 124 | if (null != attributeValue.getSS()) { 125 | valueMap.put("SS", mapper.writeValueAsString(attributeValue.getSS())); 126 | } 127 | if (null != attributeValue.getBS()) { 128 | List binaryList = new ArrayList(); 129 | for (ByteBuffer bb : attributeValue.getBS()) { 130 | binaryList.add(decoder.decode(bb).toString()); 131 | } 132 | valueMap.put("BS", mapper.writeValueAsString(binaryList)); 133 | } 134 | return mapper.writeValueAsString(valueMap); 135 | } catch (CharacterCodingException cce) { 136 | throw new IllegalArgumentException("Error: Failed to decode binary from string."); 137 | } catch (JsonProcessingException e) { 138 | throw new IllegalArgumentException("Error: Failed to convert set into string."); 139 | } 140 | } 141 | 142 | public static String toBlankString(AttributeValue attributeValue) throws IllegalArgumentException { 143 | try { 144 | String value = null; 145 | if (null != attributeValue.getS()) { 146 | value = attributeValue.getS(); 147 | } 148 | if (null != attributeValue.getN()) { 149 | value = attributeValue.getN(); 150 | } 151 | if (null != attributeValue.getB()) { 152 | value = decoder.decode(attributeValue.getB()).toString(); 153 | } 154 | if (null != attributeValue.getNS()) { 155 | value = mapper.writeValueAsString(attributeValue.getNS()); 156 | } 157 | if (null != attributeValue.getSS()) { 158 | value = mapper.writeValueAsString(attributeValue.getSS()); 159 | } 160 | if (null != attributeValue.getBS()) { 161 | List binaryList = new ArrayList(); 162 | for (ByteBuffer bb : attributeValue.getBS()) { 163 | binaryList.add(decoder.decode(bb).toString()); 164 | } 165 | value = mapper.writeValueAsString(binaryList); 166 | } 167 | return value; 168 | } catch (CharacterCodingException cce) { 169 | throw new IllegalArgumentException("Error: Failed to decode binary from string."); 170 | } catch (JsonProcessingException e) { 171 | throw new IllegalArgumentException("Error: Failed to convert set into string."); 172 | } 173 | } 174 | 175 | } 176 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/CorrectionReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.io.FileReader; 18 | import java.io.IOException; 19 | import java.io.Reader; 20 | import java.util.ArrayList; 21 | import java.util.Iterator; 22 | import java.util.List; 23 | 24 | import org.apache.commons.csv.CSVFormat; 25 | import org.apache.commons.csv.CSVParser; 26 | import org.apache.commons.csv.CSVRecord; 27 | 28 | /** 29 | * Read input correction file. 30 | * 31 | */ 32 | public class CorrectionReader { 33 | private Reader reader = null; 34 | private CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(',').withIgnoreEmptyLines(true); 35 | private CSVParser parser = null; 36 | private Iterator recordIterator; 37 | private CSVRecord currentRecord; 38 | 39 | /** 40 | * Constructor for unit test 41 | */ 42 | protected CorrectionReader(Reader reader, Iterator recordIterator) { 43 | this.reader = reader; 44 | this.recordIterator = recordIterator; 45 | } 46 | 47 | public CorrectionReader() {} 48 | 49 | public void loadCSVFile(String csvFilePath) throws IOException { 50 | reader = new FileReader(csvFilePath); 51 | parser = new CSVParser(reader, format); 52 | recordIterator = parser.iterator(); 53 | } 54 | 55 | public boolean ifContainsColumn(String columnName) { 56 | return parser.getHeaderMap().containsKey(columnName); 57 | } 58 | 59 | public boolean moveToNextRecordIfHas() { 60 | if (recordIterator.hasNext()) { 61 | currentRecord = recordIterator.next(); 62 | return true; 63 | } 64 | return false; 65 | } 66 | 67 | public String getValueInRecordByName(String recordColumnName) { 68 | try { 69 | String value = currentRecord.get(recordColumnName); 70 | if(value.equals("")) { 71 | return null; 72 | } 73 | return value; 74 | } catch (IllegalArgumentException iae) { 75 | return null; 76 | } 77 | } 78 | 79 | public List getHeader() { 80 | return new ArrayList(parser.getHeaderMap().keySet()); 81 | } 82 | 83 | public List getCurrentRecord() { 84 | List record = new ArrayList(); 85 | for(int i = 0 ; i < currentRecord.size() ; i++) { 86 | record.add(i, currentRecord.get(i)); 87 | } 88 | return record; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/OptionChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; 18 | 19 | /** 20 | * Check if given option value is valid. 21 | * 22 | */ 23 | public class OptionChecker { 24 | private final String S3_PATH_PREFIX = "s3://"; 25 | 26 | public boolean isValidKeyType(String keyType) { 27 | if (!ScalarAttributeType.B.name().equals(keyType) && !ScalarAttributeType.S.name().equals(keyType) && !ScalarAttributeType.N.name().equals(keyType)) { 28 | return false; 29 | } 30 | return true; 31 | } 32 | 33 | public boolean isS3Path(String path) { 34 | return path.startsWith(S3_PATH_PREFIX) ? true : false; 35 | } 36 | 37 | public boolean isNumberInRange(int number, int lowerBound, int upperBound) { 38 | if (number > upperBound || number < lowerBound) { 39 | return false; 40 | } 41 | return true; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/OptionLoader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.io.FileInputStream; 18 | import java.io.FileNotFoundException; 19 | import java.io.IOException; 20 | import java.util.Properties; 21 | 22 | import com.amazonaws.regions.Region; 23 | import com.amazonaws.regions.Regions; 24 | 25 | /** 26 | * Load options from property file and check options. 27 | * 28 | */ 29 | public class OptionLoader { 30 | private Properties properties = new Properties(); 31 | private OptionChecker optionChecker; 32 | private Options options = Options.getInstance(); 33 | 34 | /** 35 | * Constructor for unit test purpose only. 36 | */ 37 | protected OptionLoader(Properties properties, OptionChecker optionChecker, Options options) { 38 | this.properties = properties; 39 | this.optionChecker = optionChecker; 40 | this.options = options; 41 | } 42 | 43 | public OptionLoader(String propertyFilePath) { 44 | optionChecker = new OptionChecker(); 45 | loadPropertyFile(propertyFilePath); 46 | } 47 | 48 | protected void loadPropertyFile(String propertyFilePath) { 49 | try { 50 | properties.load(new FileInputStream(propertyFilePath)); 51 | } catch (FileNotFoundException fnfe) { 52 | throw new IllegalArgumentException("Error: Property file" + propertyFilePath + " does not exist."); 53 | } catch (IOException e) { 54 | throw new IllegalArgumentException("Error: Failed to load properties from " + propertyFilePath + " ."); 55 | } 56 | } 57 | 58 | public Options getOptions() { 59 | return this.options; 60 | } 61 | 62 | /** 63 | * Load common properties for both detection and correction. 64 | */ 65 | private void loadCommonProperties() throws IllegalArgumentException { 66 | String credentialFilePath = loadCredentialFilePath(); 67 | options.setCredentialFilePath(credentialFilePath); 68 | 69 | Region region = loadDynamoDBRegion(); 70 | options.setDynamoDBRegion(region); 71 | 72 | String tableName = loadTableName(); 73 | options.setTableName(tableName); 74 | 75 | String gsiHashKeyName = loadGsiHashKeyName(); 76 | String gsiHashKeyType = loadGsiHashKeyType(); 77 | checkGsiHashKey(gsiHashKeyName, gsiHashKeyType); 78 | options.setGsiHashKeyName(gsiHashKeyName); 79 | options.setGsiHashKeyType(gsiHashKeyType); 80 | 81 | String gsiRangeKeyName = loadGsiRangeKeyName(); 82 | String gsiRangeKeyType = loadGsiRangeKeyType(); 83 | checkGsiRangeKey(gsiRangeKeyName, gsiRangeKeyType); 84 | checkGsiHashKeyAndRangeKey(gsiHashKeyName, gsiRangeKeyName); 85 | options.setGsiRangeKeyName(gsiRangeKeyName); 86 | options.setGsiRangeKeyType(gsiRangeKeyType); 87 | 88 | int scanIOPSPercent = loadScanIOPSPercent(); 89 | options.setReadWriteIOPSPercentage(scanIOPSPercent); 90 | } 91 | 92 | protected String loadCredentialFilePath() throws IllegalArgumentException { 93 | String credentialFilePath = properties.getProperty(Options.AWS_CREDENTIAL_FILE); 94 | if (null == credentialFilePath) { 95 | throw new IllegalArgumentException("Error: " + Options.AWS_CREDENTIAL_FILE + " missing."); 96 | } 97 | return credentialFilePath.trim(); 98 | } 99 | 100 | protected Region loadDynamoDBRegion() throws IllegalArgumentException { 101 | String regionName = properties.getProperty(Options.DYNAMODB_REGION); 102 | if (null == regionName) { 103 | throw new IllegalArgumentException("Error: " + Options.DYNAMODB_REGION + " dynamoDBRegion missing."); 104 | } 105 | Region region = null; 106 | try { 107 | region = Region.getRegion(Regions.fromName(regionName.trim())); 108 | } catch (IllegalArgumentException e) { 109 | throw new IllegalArgumentException("Error: Given " + Options.DYNAMODB_REGION + " " + regionName + " is invalid."); 110 | } 111 | return region; 112 | } 113 | 114 | protected String loadTableName() throws IllegalArgumentException { 115 | String tableName = properties.getProperty(Options.TABLE_NAME); 116 | if (null == tableName) { 117 | throw new IllegalArgumentException("Error: " + Options.TABLE_NAME + " missing."); 118 | } 119 | return tableName.trim(); 120 | } 121 | 122 | protected String loadGsiHashKeyName() { 123 | return properties.getProperty(Options.GSI_HASH_KEY_NAME, null); 124 | } 125 | 126 | protected String loadGsiHashKeyType() throws IllegalArgumentException { 127 | String gsiHashKeyType = properties.getProperty(Options.GSI_HASH_KEY_TYPE, null); 128 | if (null != gsiHashKeyType) { 129 | gsiHashKeyType = gsiHashKeyType.trim(); 130 | if(!optionChecker.isValidKeyType(gsiHashKeyType)) { 131 | throw new IllegalArgumentException("Error: Given " + Options.GSI_HASH_KEY_TYPE + " " + gsiHashKeyType + " not valid key type."); 132 | } 133 | } 134 | return gsiHashKeyType; 135 | } 136 | 137 | protected void checkGsiHashKey(String gsiHashKeyName, String gsiHashKeyType) throws IllegalArgumentException { 138 | if (null == gsiHashKeyName && null != gsiHashKeyType) { 139 | throw new IllegalArgumentException("Error: " + Options.GSI_HASH_KEY_TYPE + " set while " + Options.GSI_HASH_KEY_NAME + " missing."); 140 | } 141 | if (null != gsiHashKeyName && null == gsiHashKeyType) { 142 | throw new IllegalArgumentException("Error: " + Options.GSI_HASH_KEY_NAME + " set while " + Options.GSI_HASH_KEY_TYPE + " missing."); 143 | } 144 | } 145 | 146 | protected String loadGsiRangeKeyName() throws IllegalArgumentException { 147 | return properties.getProperty(Options.GSI_RANGE_KEY_NAME, null); 148 | } 149 | 150 | protected String loadGsiRangeKeyType() throws IllegalArgumentException { 151 | String gsiRangeKeyType = properties.getProperty(Options.GSI_RANGE_KEY_TYPE, null); 152 | if(null != gsiRangeKeyType) { 153 | gsiRangeKeyType = gsiRangeKeyType.trim(); 154 | if(!optionChecker.isValidKeyType(gsiRangeKeyType)) { 155 | throw new IllegalArgumentException("Error: Given " + Options.GSI_RANGE_KEY_TYPE + " " + gsiRangeKeyType + " not a valid key type."); 156 | } 157 | } 158 | return gsiRangeKeyType; 159 | } 160 | 161 | protected void checkGsiRangeKey(String gsiRangeKeyName, String gsiRangeKeyType) throws IllegalArgumentException { 162 | if (null == gsiRangeKeyName && null != gsiRangeKeyType) { 163 | throw new IllegalArgumentException("Error: " + Options.GSI_RANGE_KEY_TYPE + "set while " + Options.GSI_RANGE_KEY_NAME + " missing."); 164 | } 165 | if (null != gsiRangeKeyName && null == gsiRangeKeyType) { 166 | throw new IllegalArgumentException("Error: " + Options.GSI_RANGE_KEY_TYPE + "set while " + Options.GSI_RANGE_KEY_NAME + " missing."); 167 | } 168 | } 169 | 170 | protected void checkGsiHashKeyAndRangeKey(String gsiHashKeyName, String gsiRangeKeyName) throws IllegalArgumentException { 171 | if (null != gsiRangeKeyName && null != gsiHashKeyName && gsiHashKeyName.equals(gsiRangeKeyName)) { 172 | throw new IllegalArgumentException("Error: " + Options.GSI_HASH_KEY_NAME + " and " + Options.GSI_RANGE_KEY_NAME + " should be different."); 173 | } 174 | if (null == gsiHashKeyName && null == gsiRangeKeyName) { 175 | throw new IllegalArgumentException("Error: '" + Options.GSI_HASH_KEY_NAME + "' and '" + Options.GSI_RANGE_KEY_NAME + "' can not be null together."); 176 | } 177 | } 178 | 179 | protected int loadScanIOPSPercent() throws IllegalArgumentException { 180 | String scanIOPSPercentStr = properties.getProperty(Options.READ_WRITE_IOPS_PERCENT, Options.READ_WRITE_IOPS_PERCENT_DEFAULT).trim(); 181 | try { 182 | int scanIOPSPercent = Integer.parseInt(scanIOPSPercentStr); 183 | if (!optionChecker.isNumberInRange(scanIOPSPercent, Options.MIN_READ_WRITE_IOPS_PERCENT, Options.MAX_READ_WRITE_IOPS_PERCENT)) { 184 | throw new IllegalArgumentException("Error: Given " + Options.READ_WRITE_IOPS_PERCENT + " " + scanIOPSPercentStr + " exceeds range " 185 | + Options.MIN_READ_WRITE_IOPS_PERCENT + " ~ " + Options.MAX_READ_WRITE_IOPS_PERCENT + " ."); 186 | } 187 | return scanIOPSPercent; 188 | } catch (NumberFormatException nfe) { 189 | throw new IllegalArgumentException("Error: Given " + Options.READ_WRITE_IOPS_PERCENT + " " + scanIOPSPercentStr + " not valid integer format."); 190 | } 191 | } 192 | 193 | /** 194 | * Load options for violation detection 195 | */ 196 | public void loadDetectionOptions() throws IllegalArgumentException { 197 | loadCommonProperties(); 198 | 199 | boolean recordDetail = loadRecordDetails(); 200 | options.setRecordDetails(recordDetail); 201 | 202 | boolean recordGsiValueInViolationRecord = loadRecordGsiValueInViolationRecord(); 203 | options.setRecordGsiValueInViolationRecord(recordGsiValueInViolationRecord); 204 | checkRecordDetailsAndRecordGsiValueConflict(recordDetail, recordGsiValueInViolationRecord); 205 | 206 | String outputPath = loadDetectionOutputPath(); 207 | options.setDetectionOutputPath(outputPath); 208 | boolean isOutputS3Path = optionChecker.isS3Path(outputPath); 209 | if (isOutputS3Path) { 210 | options.setTmpDetectionOutputPath(Options.TEMP_DETECTION_OUTPUT_PATH); 211 | } 212 | options.setIsDetectionOutputS3Path(isOutputS3Path); 213 | 214 | int numOfSegments = loadNumOfSegments(); 215 | options.setNumOfSegments(numOfSegments); 216 | 217 | int numOfViolations = loadNumOfViolations(); 218 | options.setNumOfViolations(numOfViolations); 219 | 220 | int numOfRecords = loadNumOfRecords(); 221 | options.setNumOfRecords(numOfRecords); 222 | } 223 | 224 | protected boolean loadRecordDetails() throws IllegalArgumentException { 225 | String recordDetail = properties.getProperty(Options.RECORD_DETAILS, Options.RECORD_DETAILS_DEFAULT).trim(); 226 | if (!recordDetail.equalsIgnoreCase("true") && !recordDetail.equalsIgnoreCase("false")) { 227 | throw new IllegalArgumentException("Error: Given " + Options.RECORD_DETAILS + " invalid, should be 'true' or 'false' if set."); 228 | } 229 | return Boolean.parseBoolean(recordDetail); 230 | } 231 | 232 | protected boolean loadRecordGsiValueInViolationRecord() throws IllegalArgumentException { 233 | String recordGsiValueInViolationRecord = properties.getProperty(Options.RECORD_GSI_VALUE_IN_VIOLATION_RECORD, Options.READ_WRITE_IOPS_PERCENT_DEFAULT).trim(); 234 | if (!recordGsiValueInViolationRecord.equalsIgnoreCase("true") && !recordGsiValueInViolationRecord.equalsIgnoreCase("false")) { 235 | throw new IllegalArgumentException("Error: Given " + Options.RECORD_GSI_VALUE_IN_VIOLATION_RECORD + " value '" + recordGsiValueInViolationRecord 236 | + "' invalid, should be 'true' or 'false' if set."); 237 | } 238 | return Boolean.parseBoolean(recordGsiValueInViolationRecord); 239 | } 240 | 241 | protected void checkRecordDetailsAndRecordGsiValueConflict(boolean recordDetail, boolean recordGsiValueInViolationRecord) { 242 | if (!recordDetail && recordGsiValueInViolationRecord) { 243 | throw new IllegalArgumentException("Error: Conflcct! " + Options.RECORD_GSI_VALUE_IN_VIOLATION_RECORD + " set true while " + Options.RECORD_DETAILS 244 | + " set as false."); 245 | } 246 | } 247 | 248 | protected String loadDetectionOutputPath() { 249 | return properties.getProperty(Options.DETECTION_OUTPUT_PATH, Options.DETECTION_OUTPUT_PATH_DEFAULT).trim(); 250 | } 251 | 252 | protected int loadNumOfSegments() throws IllegalArgumentException { 253 | String numOfSegmentsStr = properties.getProperty(Options.NUM_OF_SEGMENTS, Options.NUM_OF_SEGMENTS_DEFAULT).trim(); 254 | try { 255 | int numOfSegments = Integer.parseInt(numOfSegmentsStr); 256 | if (!optionChecker.isNumberInRange(numOfSegments, Options.MIN_NUM_OF_SEGMENTS, Options.MAX_NUM_OF_SEGMENTS)) { 257 | throw new IllegalArgumentException("Error: Given " + Options.NUM_OF_SEGMENTS + " " + numOfSegmentsStr + " exceeds range " 258 | + Options.MIN_NUM_OF_SEGMENTS + " ~ " + Options.MAX_NUM_OF_SEGMENTS + "."); 259 | } 260 | return numOfSegments; 261 | } catch (NumberFormatException nfe) { 262 | throw new IllegalArgumentException("Error: Given " + Options.NUM_OF_SEGMENTS + " " + numOfSegmentsStr + " is not valid integer format."); 263 | } 264 | } 265 | 266 | protected int loadNumOfViolations() throws IllegalArgumentException { 267 | String numOfViolationsStr = properties.getProperty(Options.NUM_OF_VIOLATIONS); 268 | try { 269 | /** If not set, use default. Else check if negative */ 270 | int numOfViolations = Options.NUM_OF_VIOLATIONS_DEFAULT; 271 | if (null != numOfViolationsStr) { 272 | numOfViolations = Integer.parseInt(numOfViolationsStr.trim()); 273 | if (numOfViolations != Options.NUM_OF_VIOLATIONS_DEFAULT && numOfViolations <= 0) { 274 | throw new IllegalArgumentException("Error: Given " + Options.NUM_OF_VIOLATIONS + " " + numOfViolationsStr + " invalid, must be positive."); 275 | } 276 | } 277 | return numOfViolations; 278 | } catch (NumberFormatException nfe) { 279 | throw new IllegalArgumentException("Error: Given " + Options.NUM_OF_VIOLATIONS + numOfViolationsStr + " is not a valid integer format."); 280 | } 281 | } 282 | 283 | protected int loadNumOfRecords() throws IllegalArgumentException { 284 | String numOfRecordsStr = properties.getProperty(Options.NUM_OF_RECORDS); 285 | try { 286 | /** If not set, use default. Else check if negative */ 287 | int numOfRecords = Options.NUM_OF_RECORDS_DEFAULT; 288 | if (null != numOfRecordsStr) { 289 | numOfRecords = Integer.parseInt(numOfRecordsStr.trim()); 290 | if (numOfRecords != Options.NUM_OF_RECORDS_DEFAULT && numOfRecords <= 0) { 291 | throw new IllegalArgumentException("Error: Given " + Options.NUM_OF_RECORDS + " " + numOfRecords + " invalid, must be positive."); 292 | } 293 | } 294 | return numOfRecords; 295 | } catch (NumberFormatException nfe) { 296 | throw new IllegalArgumentException("Error: Given " + Options.NUM_OF_RECORDS + " " + numOfRecordsStr + " is not a valid integer format."); 297 | } 298 | } 299 | 300 | /** 301 | * Load options for correction. 302 | */ 303 | public void loadCorrectionOptions() throws IllegalArgumentException { 304 | loadCommonProperties(); 305 | 306 | String correctionInputPath = loadCorrectionInputPath(); 307 | options.setCorrectionInputPath(correctionInputPath); 308 | boolean isInputS3Path = optionChecker.isS3Path(correctionInputPath); 309 | if (isInputS3Path) { 310 | options.setTmpCorrectionInputPath(Options.TEMP_CORRECTION_INPUT_PATH); 311 | } 312 | options.setIsCorrectionInputS3Path(isInputS3Path); 313 | 314 | String correctionOutputPath = loadCorrectionOutputPath(); 315 | options.setCorrectionOutputPath(correctionOutputPath); 316 | boolean isCorrectionOutputS3Path = optionChecker.isS3Path(correctionOutputPath); 317 | if (isCorrectionOutputS3Path) { 318 | options.setTmpCorrectionOutputPath(Options.TEMP_CORRECTION_OUTPUT_PATH); 319 | } 320 | options.setIsCorrectionOutputS3Path(isCorrectionOutputS3Path); 321 | 322 | // validate that input and output paths are not the same 323 | if(correctionInputPath.equals(correctionOutputPath)) { 324 | throw new IllegalArgumentException("Error: " + Options.CORRECTION_INPUT_PATH + " and " + 325 | Options.CORRECTION_OUTPUT_PATH + " cannot be the same."); 326 | } 327 | } 328 | 329 | protected String loadCorrectionInputPath() throws IllegalArgumentException { 330 | String correctionInputPath = properties.getProperty(Options.CORRECTION_INPUT_PATH); 331 | if (null == correctionInputPath) { 332 | throw new IllegalArgumentException("Error: " + Options.CORRECTION_INPUT_PATH + " missing."); 333 | } 334 | return correctionInputPath.trim(); 335 | } 336 | 337 | protected String loadCorrectionOutputPath() throws IllegalArgumentException { 338 | return properties.getProperty(Options.CORRECTION_OUTPUT_PATH, Options.CORRECTION_OUTPUT_PATH_DEFAULT).trim(); 339 | } 340 | } 341 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/Options.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import com.amazonaws.regions.Region; 18 | 19 | /** 20 | * A Singleton class for options loading. 21 | * 22 | */ 23 | public class Options { 24 | /** Instance */ 25 | private static Options instance = new Options(); 26 | 27 | /** Option names on property file */ 28 | public static final String AWS_CREDENTIAL_FILE = "awsCredentialsFile"; 29 | public static final String DYNAMODB_REGION = "dynamoDBRegion"; 30 | public static final String TABLE_NAME = "tableName"; 31 | public static final String GSI_HASH_KEY_NAME = "gsiHashKeyName"; 32 | public static final String GSI_HASH_KEY_TYPE = "gsiHashKeyType"; 33 | public static final String GSI_RANGE_KEY_NAME = "gsiRangeKeyName"; 34 | public static final String GSI_RANGE_KEY_TYPE = "gsiRangeKeyType"; 35 | public static final String READ_WRITE_IOPS_PERCENT = "readWriteIOPSPercent"; 36 | public static final String RECORD_DETAILS = "recordDetails"; 37 | public static final String RECORD_GSI_VALUE_IN_VIOLATION_RECORD = "recordGsiValueInViolationRecord"; 38 | public static final String EXISTING_GSI = "existingGSI"; 39 | public static final String DETECTION_OUTPUT_PATH = "detectionOutputPath"; 40 | public static final String NUM_OF_SEGMENTS = "numOfSegments"; 41 | public static final String NUM_OF_VIOLATIONS = "numOfViolations"; 42 | public static final String NUM_OF_RECORDS = "numOfRecords"; 43 | public static final String CORRECTION_INPUT_PATH = "correctionInputPath"; 44 | public static final String CORRECTION_OUTPUT_PATH = "correctionOutputPath"; 45 | 46 | /** Default value and limits */ 47 | public static final String READ_WRITE_IOPS_PERCENT_DEFAULT = "25"; 48 | public static int MIN_READ_WRITE_IOPS_PERCENT = 1; 49 | public static int MAX_READ_WRITE_IOPS_PERCENT = 100; 50 | public static final String RECORD_DETAILS_DEFAULT = "true"; 51 | public static final String RECORD_GSI_VALUE_IN_VIOLATION_RECORD_DEFAULT = "false"; 52 | public static final String EXISTING_GSI_DEFAULT = "false"; 53 | public static final String DETECTION_OUTPUT_PATH_DEFAULT = "./violation_detection.csv"; 54 | public static final String CORRECTION_OUTPUT_PATH_DEFAULT = "./violation_update_errors.csv"; 55 | public static final String TEMP_DETECTION_OUTPUT_PATH = "./detection.tmp"; 56 | public static final String NUM_OF_SEGMENTS_DEFAULT = "1"; 57 | public static int MIN_NUM_OF_SEGMENTS = 1; 58 | public static int MAX_NUM_OF_SEGMENTS = 4096; 59 | public static final int NUM_OF_VIOLATIONS_DEFAULT = -1; 60 | public static final int NUM_OF_RECORDS_DEFAULT = -1; 61 | public static final String TEMP_CORRECTION_INPUT_PATH = "./correction_input.tmp"; 62 | public static final String TEMP_CORRECTION_OUTPUT_PATH = "./correction_output.tmp"; 63 | 64 | /** Options provided by the users */ 65 | private String credentialFilePath = null; 66 | private Region dynamoDBRegion = null; 67 | private String tableName = null; 68 | private String gsiHashKeyName = null; 69 | private String gsiHashKeyType = null; 70 | private String gsiRangeKeyName = null; 71 | private String gsiRangeKeyType = null; 72 | private boolean recordDetails = true; 73 | private boolean recordGsiValueInViolationRecord = false; 74 | private String detectionOutputPath = null; 75 | private String tmpDetectionOutputPath = null; 76 | private String correctionOutputPath = null; 77 | private String tmpCorrectionOutputPath = null; 78 | private int numOfSegments = 1; 79 | private long numOfViolations = -1; 80 | private long numOfRecords = -1; 81 | private int readWriteIOPSPercent = 25; 82 | private String correctionInputPath = null; 83 | private String tmpCorrectionInputPath = null; 84 | private boolean isDetectionOutputS3Path = false; 85 | private boolean isInputS3path = false; 86 | private boolean isCorrectionOutputS3Path = false; 87 | 88 | private Options() { 89 | }; 90 | 91 | public static Options getInstance() { 92 | return instance; 93 | } 94 | 95 | public String getCredentialsFilePath() { 96 | return credentialFilePath; 97 | } 98 | 99 | public void setCredentialFilePath(String credentialFilePath) { 100 | this.credentialFilePath = credentialFilePath; 101 | } 102 | 103 | public Region getDynamoDBRegion() { 104 | return dynamoDBRegion; 105 | } 106 | 107 | public void setDynamoDBRegion(Region region) { 108 | this.dynamoDBRegion = region; 109 | } 110 | 111 | public String getTableName() { 112 | return tableName; 113 | } 114 | 115 | public void setTableName(String tableName) { 116 | this.tableName = tableName; 117 | } 118 | 119 | public String getGsiHashKeyName() { 120 | return gsiHashKeyName; 121 | } 122 | 123 | public void setGsiHashKeyName(String gsiHashKeyName) { 124 | this.gsiHashKeyName = gsiHashKeyName; 125 | } 126 | 127 | public String getGsiHashKeyType() { 128 | return gsiHashKeyType; 129 | } 130 | 131 | public void setGsiHashKeyType(String gsiHashKeyType) { 132 | this.gsiHashKeyType = gsiHashKeyType; 133 | } 134 | 135 | public String getGsiRangeKeyName() { 136 | return gsiRangeKeyName; 137 | } 138 | 139 | public void setGsiRangeKeyName(String gsiRangeKeyName) { 140 | this.gsiRangeKeyName = gsiRangeKeyName; 141 | } 142 | 143 | public String getGsiRangeKeyType() { 144 | return gsiRangeKeyType; 145 | } 146 | 147 | public void setGsiRangeKeyType(String gsiRangeKeyType) { 148 | this.gsiRangeKeyType = gsiRangeKeyType; 149 | } 150 | 151 | public boolean recordDetails() { 152 | return recordDetails; 153 | } 154 | 155 | public void setRecordDetails(boolean recordDetails) { 156 | this.recordDetails = recordDetails; 157 | } 158 | 159 | public boolean recordGsiValueInViolationRecord() { 160 | return recordGsiValueInViolationRecord; 161 | } 162 | 163 | public void setRecordGsiValueInViolationRecord(boolean recordGsiValueInViolationRecord) { 164 | this.recordGsiValueInViolationRecord = recordGsiValueInViolationRecord; 165 | } 166 | 167 | public String getDetectionOutputPath() { 168 | return detectionOutputPath; 169 | } 170 | 171 | public void setDetectionOutputPath(String detectionOutputPath) { 172 | this.detectionOutputPath = detectionOutputPath; 173 | } 174 | 175 | public String getTmpDetectionOutputPath() { 176 | return this.tmpDetectionOutputPath; 177 | } 178 | 179 | public void setTmpDetectionOutputPath(String tmpDetectionOutputPath) { 180 | this.tmpDetectionOutputPath = tmpDetectionOutputPath; 181 | } 182 | 183 | public boolean isDetectionOutputS3Path() { 184 | return this.isDetectionOutputS3Path; 185 | } 186 | 187 | public void setIsDetectionOutputS3Path(boolean isDetectionOutputS3Path) { 188 | this.isDetectionOutputS3Path = isDetectionOutputS3Path; 189 | } 190 | 191 | public String getS3PathBucketName(String path) { 192 | return path.split("/")[2]; 193 | } 194 | 195 | public String getS3PathKey(String path) { 196 | int start = path.indexOf("/", 5); 197 | if (start <= 0) 198 | throw new IllegalArgumentException("Error: Invalid S3 path: " + path); 199 | return path.substring(start + 1); 200 | } 201 | 202 | public int getNumOfSegments() { 203 | return numOfSegments; 204 | } 205 | 206 | public void setNumOfSegments(int numOfSegments) { 207 | this.numOfSegments = numOfSegments; 208 | } 209 | 210 | public long getNumOfViolations() { 211 | return numOfViolations; 212 | } 213 | 214 | public void setNumOfViolations(long numOfViolations) { 215 | this.numOfViolations = numOfViolations; 216 | } 217 | 218 | public long getNumOfRecords() { 219 | return numOfRecords; 220 | } 221 | 222 | public void setNumOfRecords(long numOfRecords) { 223 | this.numOfRecords = numOfRecords; 224 | } 225 | 226 | public int getReadWriteIOPSPercent() { 227 | return readWriteIOPSPercent; 228 | } 229 | 230 | public void setReadWriteIOPSPercentage(int readWriteIOPSPercent) { 231 | this.readWriteIOPSPercent = readWriteIOPSPercent; 232 | } 233 | 234 | public String getCorrectionInputPath() { 235 | return this.correctionInputPath; 236 | } 237 | 238 | public void setCorrectionInputPath(String correctionInputPath) { 239 | this.correctionInputPath = correctionInputPath; 240 | } 241 | 242 | public boolean isCorrectionInputS3Path() { 243 | return this.isInputS3path; 244 | } 245 | 246 | public void setIsCorrectionInputS3Path(boolean isInputS3Path) { 247 | this.isInputS3path = isInputS3Path; 248 | } 249 | 250 | public String getTmpCorrectionInputPath() { 251 | return this.tmpCorrectionInputPath; 252 | } 253 | 254 | public void setTmpCorrectionInputPath(String tmpCorrectionInputPath) { 255 | this.tmpCorrectionInputPath = tmpCorrectionInputPath; 256 | } 257 | 258 | public String getCorrectionOutputPath() { 259 | return correctionOutputPath; 260 | } 261 | 262 | public void setCorrectionOutputPath(String correctionOutputPath) { 263 | this.correctionOutputPath = correctionOutputPath; 264 | } 265 | 266 | public String getTmpCorrectionOutputPath() { 267 | return tmpCorrectionOutputPath; 268 | } 269 | 270 | public void setTmpCorrectionOutputPath(String tmpCorrectionOutputPath) { 271 | this.tmpCorrectionOutputPath = tmpCorrectionOutputPath; 272 | } 273 | 274 | public boolean isCorrectionOutputS3Path() { 275 | return isCorrectionOutputS3Path; 276 | } 277 | 278 | public void setIsCorrectionOutputS3Path(boolean isCorrectionOutputS3Path) { 279 | this.isCorrectionOutputS3Path = isCorrectionOutputS3Path; 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/PrintHelper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.nio.ByteBuffer; 18 | import java.util.Iterator; 19 | import java.util.Map; 20 | import java.util.Map.Entry; 21 | 22 | import org.apache.log4j.Logger; 23 | 24 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 25 | 26 | /** 27 | * Print progress information. 28 | * 29 | */ 30 | public class PrintHelper { 31 | 32 | private static final Logger logger = Logger.getLogger(PrintHelper.class); 33 | 34 | public static void printItem(int segment, long itemCount, Map attributeList) { 35 | logger.info("Segment " + segment + ", itemCount " + itemCount + ", "); 36 | for (Entry item : attributeList.entrySet()) { 37 | String attributeName = item.getKey(); 38 | AttributeValue value = item.getValue(); 39 | logger.info(attributeName + " " + (value.getS() == null ? "" : "S = [" + value.getS() + "]") 40 | + (value.getN() == null ? "" : "N = [" + value.getN() + "]") 41 | + (value.getB() == null ? "" : "B = [" + new String(value.getB().array()) + "]")); 42 | 43 | /** String Set */ 44 | if (value.getSS() != null) { 45 | logger.info("SS = ["); 46 | Iterator iterator = value.getSS().iterator(); 47 | while (iterator.hasNext()) { 48 | logger.info(iterator.next() + (iterator.hasNext() ? ", " : "")); 49 | } 50 | logger.info("]"); 51 | } 52 | 53 | /** Number Set */ 54 | if (value.getNS() != null) { 55 | logger.info("NS = ["); 56 | Iterator iterator = value.getNS().iterator(); 57 | while (iterator.hasNext()) { 58 | logger.info(iterator.next() + (iterator.hasNext() ? ", " : "")); 59 | } 60 | logger.info("]"); 61 | } 62 | 63 | /** Binary Set */ 64 | if (value.getBS() != null) { 65 | logger.info("SS = ["); 66 | Iterator iterator = value.getBS().iterator(); 67 | while (iterator.hasNext()) { 68 | logger.info(new String(iterator.next().array()) + (iterator.hasNext() ? ", " : "")); 69 | } 70 | logger.info("]"); 71 | } 72 | logger.info(", "); 73 | 74 | } 75 | } 76 | 77 | public static void printScanStartInfo(boolean parallelScan, String tableName, String gsiHashKeyName, String gsiRangeKeyName) { 78 | String message = "Violation detection started: " + (parallelScan == true ? "paralallel scan" : "sequential scan") + ", Table name: " + tableName 79 | + (gsiHashKeyName != null ? ", GSI hash Key: " + gsiHashKeyName : "") 80 | + (gsiRangeKeyName != null ? ", GSI range key: " + gsiRangeKeyName : ""); 81 | logger.info(message); 82 | } 83 | 84 | public static void printDeleteWarning() { 85 | String message = "WARNING: delete has been chosen, violation will be deleted from table!!"; 86 | logger.info(message); 87 | } 88 | 89 | public static void printScanProgress(long recordScanned, long recordScannedbyThread, long violationFound, long violationDelete) { 90 | String message = "Progress: " + "Items scanned in total: " + recordScanned + "," + "\tItems scanned by this thread: " + recordScannedbyThread 91 | + "," + "\tViolations found by this thread: " + violationFound + "," + "\tViolations deleted by this thread: " + violationDelete + "\t..."; 92 | logger.info(message); 93 | } 94 | 95 | public static void printNumOfViolationReachedExitInfo() { 96 | String message = "Given number of violations has been found, will stop scanning now."; 97 | logger.info(message); 98 | } 99 | 100 | public static void printNumOfItemReachedExitInfo() { 101 | String message = "Given number of items has been scanned, will stop scanning now"; 102 | logger.info(message); 103 | } 104 | 105 | public static void printScanSummary(long recordsScanned, long violationsFound, long violationDelete, String outputPath, boolean recordViolations) { 106 | String message = "Violation detection finished: " + "Records scanned: " + recordsScanned + ", Violations found: " + violationsFound 107 | + ", Violations deleted: " + violationDelete; 108 | if(recordViolations) { 109 | message += ", see results at: " + outputPath; 110 | } 111 | logger.info(message); 112 | } 113 | 114 | public static void printDeleteStartInfo(String inputFilePath) { 115 | String message = "Violation correction from file started: " + "Reading records from file: " + inputFilePath + ", will delete these records from table."; 116 | logger.info(message); 117 | } 118 | 119 | public static void printDeleteProgressInfo(int deleteItems, long totalNumOfItemsDeleted) { 120 | String message = "Violation delete progress: " + deleteItems + " items deleted, \t" + totalNumOfItemsDeleted + " total number of items delted..."; 121 | logger.info(message); 122 | } 123 | 124 | public static void printUpdateStartInfo(String inputFilePath) { 125 | String message = "Violation correction from file started: " + "Reading records from file: " + inputFilePath + ", will update these records from table."; 126 | logger.info(message); 127 | } 128 | 129 | public static void printCorrectionSummary(long violationUpdateRequests, long success, 130 | long failedConditionalUpdates, long unexpectedErrors, String outputFile) { 131 | String message = "Violation correction from file finished: " + "Total Requests With Updates: " + violationUpdateRequests 132 | + ", Successful Requests: " + success + ", Conditional Update Failures: " 133 | + failedConditionalUpdates + ", Errors: " + unexpectedErrors; 134 | 135 | if(failedConditionalUpdates > 0 || unexpectedErrors > 0) { 136 | message += "\nSee violation update error details at: " + outputFile; 137 | } 138 | logger.info(message); 139 | } 140 | 141 | public static void printCorrectionSummary(long violationUpdateRequests, long success, 142 | long unexpectedErrors, String outputFile) { 143 | String message = "Violation correction from file finished: " + "Total Requests With Updates: " + violationUpdateRequests 144 | + ", Successful Requests: " + success + ", Errors: " + unexpectedErrors; 145 | 146 | if(unexpectedErrors > 0) { 147 | message += "\nSee violation update error details at: " + outputFile; 148 | } 149 | logger.info(message); 150 | } 151 | 152 | public static void printCorrectionDeleteSummary(long violationDelete) { 153 | String message = "Violation correction from file finished: " + "Total Violations deleted: " + violationDelete; 154 | logger.info(message); 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/TableHelper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | 20 | import com.amazonaws.AmazonServiceException; 21 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 22 | import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; 23 | import com.amazonaws.services.dynamodbv2.model.DescribeTableResult; 24 | import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndexDescription; 25 | import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; 26 | import com.amazonaws.services.dynamodbv2.model.KeyType; 27 | import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; 28 | import com.amazonaws.services.dynamodbv2.model.TableDescription; 29 | 30 | /** 31 | * Get table information. 32 | * 33 | */ 34 | public class TableHelper { 35 | 36 | private AmazonDynamoDBClient dynamoDBClient; 37 | private TableDescription tableDescription; 38 | 39 | /** 40 | * Constructor for unit test. 41 | */ 42 | protected TableHelper(AmazonDynamoDBClient dynamoDBClient, TableDescription tableDescription) { 43 | this.dynamoDBClient = dynamoDBClient; 44 | this.tableDescription = tableDescription; 45 | } 46 | 47 | public TableHelper(AmazonDynamoDBClient dynamoDBClient, String tableName) 48 | throws IllegalArgumentException { 49 | this.dynamoDBClient = dynamoDBClient; 50 | describeTable(tableName); 51 | } 52 | 53 | private void describeTable(String tableName) throws IllegalArgumentException { 54 | try { 55 | DescribeTableResult describeResult = dynamoDBClient.describeTable(tableName); 56 | tableDescription = describeResult.getTable(); 57 | } catch (ResourceNotFoundException rnfe) { 58 | throw new IllegalArgumentException("Error: given table " + tableName + " does not exist in given region."); 59 | } catch (AmazonServiceException ase) { 60 | if (ase.getErrorCode().equals("UnrecognizedClientException")) 61 | throw new IllegalArgumentException("Error: Security token in credential file invalid."); 62 | else 63 | throw new IllegalArgumentException(ase.getMessage()); 64 | } 65 | } 66 | 67 | private KeySchemaElement getTableHashKey() { 68 | List keySchemaList = tableDescription.getKeySchema(); 69 | for (KeySchemaElement keyElement : keySchemaList) { 70 | if (keyElement.getKeyType().equals(KeyType.HASH.toString())) { 71 | return keyElement; 72 | } 73 | } 74 | return null; 75 | } 76 | 77 | public String getTableHashKeyName() { 78 | return getTableHashKey().getAttributeName(); 79 | } 80 | 81 | public String getTableHashKeyType() { 82 | String hashKeyName = getTableHashKeyName(); 83 | List definitions = tableDescription.getAttributeDefinitions(); 84 | for (AttributeDefinition attributeDef : definitions) { 85 | if (attributeDef.getAttributeName().endsWith(hashKeyName)) { 86 | return attributeDef.getAttributeType(); 87 | } 88 | } 89 | return null; 90 | } 91 | 92 | private KeySchemaElement getTableRangeKey() { 93 | List keySchemaList = tableDescription.getKeySchema(); 94 | for (KeySchemaElement keyElement : keySchemaList) { 95 | if (keyElement.getKeyType().equals(KeyType.RANGE.toString())) { 96 | return keyElement; 97 | } 98 | } 99 | return null; 100 | } 101 | 102 | public String getTableRangeKeyName() { 103 | return getTableRangeKey() == null ? null : getTableRangeKey().getAttributeName(); 104 | } 105 | 106 | public String getTableRangeKeyType() { 107 | if (null == getTableRangeKey()) 108 | return null; 109 | String rangeKeyName = getTableRangeKeyName(); 110 | List definitions = tableDescription.getAttributeDefinitions(); 111 | for (AttributeDefinition attributeDef : definitions) { 112 | if (attributeDef.getAttributeName().equals(rangeKeyName)) { 113 | return attributeDef.getAttributeType(); 114 | } 115 | } 116 | return null; 117 | } 118 | 119 | public List getListOfAttributesToFetch(String gsiHashKeyName, String gsiRangeKeyName) { 120 | List attributesToGet = new ArrayList(); 121 | attributesToGet.add(getTableHashKeyName()); 122 | if (getTableRangeKey() != null) { 123 | attributesToGet.add(getTableRangeKeyName()); 124 | } 125 | /** Both GSI hash key and range key are optional */ 126 | if (gsiHashKeyName != null) { 127 | attributesToGet.add(gsiHashKeyName); 128 | } 129 | if (gsiRangeKeyName != null) { 130 | attributesToGet.add(gsiRangeKeyName); 131 | } 132 | return attributesToGet; 133 | } 134 | 135 | public boolean isGsiExists(String gsiName) { 136 | List descriptionList = null; 137 | descriptionList = tableDescription.getGlobalSecondaryIndexes(); 138 | if (null == descriptionList) { 139 | return false; 140 | } 141 | 142 | for (GlobalSecondaryIndexDescription desc : descriptionList) { 143 | if (desc.getIndexName().equals(gsiName)) { 144 | return true; 145 | } 146 | } 147 | return false; 148 | } 149 | 150 | public long getReadCapacityUnits() { 151 | return tableDescription.getProvisionedThroughput().getReadCapacityUnits(); 152 | } 153 | 154 | public long getWriteCapacityUnits() { 155 | return tableDescription.getProvisionedThroughput().getWriteCapacityUnits(); 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/TableRWRateLimiter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.util.List; 18 | 19 | import com.amazonaws.services.dynamodbv2.model.ConsumedCapacity; 20 | import com.google.common.util.concurrent.RateLimiter; 21 | 22 | /** 23 | * Rate Limiter for read/write DynamoDB table. 24 | * 25 | */ 26 | public class TableRWRateLimiter { 27 | private double readWriteIOPSCapacityUnits; 28 | private double readWriteIOPSPercent; 29 | private double accumulatedReadWritePermits; 30 | private int numOfTasks; 31 | /** 32 | * Rate limiter is created with a permits per second rate. Each time the 33 | * limiter acquire the permitsToConsume, it will adjust the time to block 34 | * until the next permits is ready to release, so that the overall 35 | * rate it maintained. For more details, see: 36 | * http://docs.guava-libraries.googlecode.com/git/javadoc/com/google/common/util/concurrent/RateLimiter.html 37 | */ 38 | private RateLimiter rateLimiter; 39 | 40 | public TableRWRateLimiter(double readWriteIOPSCapacityUnits, double readWriteIOPSPercent, int numOfTasks) { 41 | this.readWriteIOPSCapacityUnits = readWriteIOPSCapacityUnits; 42 | this.readWriteIOPSPercent = readWriteIOPSPercent; 43 | this.numOfTasks = numOfTasks; 44 | this.accumulatedReadWritePermits = 0; 45 | double rateLimit = getRateLimit(); 46 | rateLimiter = RateLimiter.create(rateLimit); 47 | } 48 | 49 | private double getRateLimit() { 50 | double rateLimit = readWriteIOPSCapacityUnits * readWriteIOPSPercent / 100; 51 | double rateLimitPerTask = rateLimit / numOfTasks; 52 | if (rateLimitPerTask <= 0) { 53 | throw new IllegalArgumentException("Error: readWriteIOPSCapacityUnits: " + readWriteIOPSCapacityUnits + " or readWriteIOPSPercent: " 54 | + readWriteIOPSPercent + "too low, can not start read or write table."); 55 | } 56 | return rateLimitPerTask; 57 | } 58 | 59 | public void adjustRateWithConsumedCapacity(List bathWriteConsumedCapacity) { 60 | for (ConsumedCapacity consumedCapacity : bathWriteConsumedCapacity) { 61 | adjustRateWithConsumedCapacity(consumedCapacity); 62 | } 63 | } 64 | 65 | public void adjustRateWithConsumedCapacity(ConsumedCapacity consumedCapacity) { 66 | accumulatedReadWritePermits += consumedCapacity.getCapacityUnits(); 67 | if (accumulatedReadWritePermits > 1.0) { 68 | int intValueOfPermits = Double.valueOf(accumulatedReadWritePermits).intValue(); 69 | rateLimiter.acquire(intValueOfPermits); 70 | accumulatedReadWritePermits -= (double) intValueOfPermits; 71 | } 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/TableReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.io.IOException; 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.concurrent.ExecutorService; 21 | import java.util.concurrent.Executors; 22 | import java.util.concurrent.TimeUnit; 23 | import java.util.concurrent.atomic.AtomicLong; 24 | 25 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 26 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 27 | import com.amazonaws.services.dynamodbv2.model.ReturnConsumedCapacity; 28 | import com.amazonaws.services.dynamodbv2.model.ScanRequest; 29 | import com.amazonaws.services.dynamodbv2.model.ScanResult; 30 | 31 | /** 32 | * Scan table while checking violations. 33 | * 34 | */ 35 | public class TableReader { 36 | 37 | private static Options options; 38 | private static TableHelper tableHelper; 39 | private static AmazonDynamoDBClient dynamoDBClient; 40 | private static List attributesToGet; 41 | private static long itemsScanLimit; 42 | private static AtomicLong itemsScanned; 43 | private static long violationsFindLimit; 44 | private static AtomicLong violationsFound; 45 | private static AtomicLong violationsDeleted; 46 | 47 | // Used for running tests on DDB Local. (Rate Limiter cannot be used with DDB Local.) 48 | private static boolean isRunningOnDDBLocal = false; 49 | 50 | /** 51 | * Constructor for unit test purpose only. 52 | */ 53 | protected TableReader(Options options, AmazonDynamoDBClient dynamoDBClient, TableHelper tableHelper, ViolationChecker violationChecker, 54 | List attributesToGet, double taskRateLimit) { 55 | TableReader.options = options; 56 | TableReader.dynamoDBClient = dynamoDBClient; 57 | TableReader.tableHelper = tableHelper; 58 | TableReader.attributesToGet = attributesToGet; 59 | } 60 | 61 | public TableReader(Options options, AmazonDynamoDBClient dynamoDBClient, TableHelper tableHelper, boolean isRunningOnDDBLocal) 62 | throws IOException, IllegalArgumentException { 63 | TableReader.options = options; 64 | TableReader.dynamoDBClient = dynamoDBClient; 65 | TableReader.tableHelper = tableHelper; 66 | attributesToGet = tableHelper.getListOfAttributesToFetch(options.getGsiHashKeyName(), options.getGsiRangeKeyName()); 67 | itemsScanned = new AtomicLong(0); 68 | itemsScanLimit = options.getNumOfRecords(); 69 | violationsFound = new AtomicLong(0); 70 | violationsFindLimit = options.getNumOfViolations(); 71 | violationsDeleted = new AtomicLong(0); 72 | if (options.recordDetails()) { 73 | createViolationWriter(); 74 | } 75 | TableReader.isRunningOnDDBLocal = isRunningOnDDBLocal; 76 | } 77 | 78 | protected void createViolationWriter() throws IOException { 79 | String outputFilePath; 80 | if (options.isDetectionOutputS3Path()) { 81 | outputFilePath = options.getTmpDetectionOutputPath(); 82 | } else { 83 | outputFilePath = options.getDetectionOutputPath(); 84 | } 85 | ViolationWriter.getInstance().createOutputFile(outputFilePath); 86 | } 87 | 88 | public void scanTable(boolean deleteViolationsAfterFound) throws IOException { 89 | int numOfSegments = options.getNumOfSegments(); 90 | boolean parallelScan = Integer.parseInt(Options.NUM_OF_SEGMENTS_DEFAULT) != numOfSegments; 91 | PrintHelper.printScanStartInfo(parallelScan, options.getTableName(), options.getGsiHashKeyName(), options.getGsiRangeKeyName()); 92 | if (deleteViolationsAfterFound) { 93 | PrintHelper.printDeleteWarning(); 94 | } 95 | createSegmentScanThreads(numOfSegments, deleteViolationsAfterFound); 96 | if (options.recordDetails()) { 97 | ViolationWriter.getInstance().flushAndCloseWriter(); 98 | } 99 | PrintHelper.printScanSummary(itemsScanned.get(), violationsFound.get(), violationsDeleted.get(), options.getDetectionOutputPath(), options.recordDetails()); 100 | return; 101 | } 102 | 103 | protected void createSegmentScanThreads(int numOfThreadsToCreate, boolean deleteViolationsAfterFound) throws IOException { 104 | ExecutorService executor = Executors.newFixedThreadPool(numOfThreadsToCreate); 105 | for (int segment = 0; segment < numOfThreadsToCreate; segment++) { 106 | ScanSegment scanSegmenTask = new ScanSegment(options, tableHelper, dynamoDBClient, deleteViolationsAfterFound, segment); 107 | executor.execute(scanSegmenTask); 108 | } 109 | 110 | /** Wait until all threads end */ 111 | executor.shutdown(); 112 | try { 113 | executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); 114 | } catch (InterruptedException e) { 115 | // INGORE InterruptedException 116 | } 117 | } 118 | 119 | /** 120 | * For testing 121 | */ 122 | public long getViolationsFound() { 123 | return violationsFound.get(); 124 | } 125 | 126 | /** 127 | * For testing 128 | */ 129 | public long getItemsScanned() { 130 | return itemsScanned.get(); 131 | } 132 | 133 | /** 134 | * For testing 135 | */ 136 | public long getViolationsDeleted() { 137 | return violationsDeleted.get(); 138 | } 139 | 140 | private static class ScanSegment implements Runnable { 141 | private String tableName; 142 | private int numOfSegments; 143 | private int segmentNum; 144 | private long itemScannedByThread = 0; 145 | private long violationFoundByThread = 0; 146 | private long violationDeleteByThread = 0; 147 | private boolean deleteViolationAfterFound = false; 148 | private ViolationChecker violationChecker; 149 | 150 | private TableWriter tableWriter; 151 | private TableRWRateLimiter tableReadRateLimiter; 152 | 153 | public ScanSegment(Options options, TableHelper tableHelper, AmazonDynamoDBClient dynamoDBClient, boolean deleteViolationAfterFound, int segmentNum) 154 | throws IOException { 155 | this.tableName = options.getTableName(); 156 | this.numOfSegments = options.getNumOfSegments(); 157 | this.segmentNum = segmentNum; 158 | this.deleteViolationAfterFound = deleteViolationAfterFound; 159 | this.violationChecker = new ViolationChecker(options, tableHelper); 160 | this.tableWriter = new TableWriter(options, tableHelper, dynamoDBClient, numOfSegments, isRunningOnDDBLocal); 161 | this.tableReadRateLimiter = new TableRWRateLimiter(tableHelper.getReadCapacityUnits(), options.getReadWriteIOPSPercent(), 162 | options.getNumOfSegments()); 163 | /**Write header to the output file, this is not a good idea to test the first*/ 164 | if (segmentNum == 0 && options.recordDetails()) { 165 | ViolationWriter.getInstance().addViolationRecord(violationChecker.getViolationRecordHead()); 166 | } 167 | } 168 | 169 | @Override 170 | public void run() { 171 | Map exclusiveStartKey = null; 172 | ScanRequest scanRequest = new ScanRequest().withTableName(tableName).withAttributesToGet(attributesToGet) 173 | .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withTotalSegments(numOfSegments).withSegment(segmentNum); 174 | boolean scanNumLimitReached = false; 175 | while (!scanNumLimitReached) { 176 | scanRequest.withExclusiveStartKey(exclusiveStartKey); 177 | ScanResult scanResult = dynamoDBClient.scan(scanRequest); 178 | if(!isRunningOnDDBLocal) { 179 | // DDB Local does not support rate limiting 180 | tableReadRateLimiter.adjustRateWithConsumedCapacity(scanResult.getConsumedCapacity()); 181 | } 182 | 183 | for (Map item : scanResult.getItems()) { 184 | checkItemViolationAndAddDeleteRequest(item); 185 | itemsScanned.addAndGet(1); 186 | itemScannedByThread += 1; 187 | scanNumLimitReached = isScanNumberLimitReached(); 188 | if(scanNumLimitReached) { 189 | break; 190 | } 191 | } 192 | 193 | if (deleteViolationAfterFound) { 194 | sendDeleteViolations(); 195 | } 196 | PrintHelper.printScanProgress(itemsScanned.get(), itemScannedByThread, violationFoundByThread, violationDeleteByThread); 197 | 198 | if (null == (exclusiveStartKey = scanResult.getLastEvaluatedKey())) { 199 | break; 200 | } 201 | } 202 | return; 203 | } 204 | 205 | protected void checkItemViolationAndAddDeleteRequest(Map item) { 206 | try { 207 | ViolationRecord violationRecord = violationChecker.checkItemViolationAndGetRecord(item); 208 | if (violationRecord != null) { 209 | if (options.recordDetails()) { 210 | ViolationWriter.getInstance().addViolationRecord(violationRecord); 211 | } 212 | violationsFound.addAndGet(1); 213 | violationFoundByThread += 1; 214 | if (deleteViolationAfterFound) { 215 | addDeleteViolationRequests(item); 216 | } 217 | } 218 | } catch (IOException ioe) { 219 | throw new IllegalArgumentException("Error: Failed to write violation records to file."); 220 | } 221 | } 222 | 223 | protected boolean isScanNumberLimitReached() { 224 | if (itemsScanLimit > 0 && itemsScanLimit <= itemsScanned.get()) { 225 | PrintHelper.printNumOfItemReachedExitInfo(); 226 | return true; 227 | } 228 | if (violationsFindLimit > 0 && violationsFindLimit <= violationsFound.get()) { 229 | PrintHelper.printNumOfViolationReachedExitInfo(); 230 | return true; 231 | } 232 | return false; 233 | } 234 | 235 | protected void addDeleteViolationRequests(Map item) { 236 | int deletedItem = tableWriter.addDeleteRequest(item); 237 | violationsDeleted.addAndGet(deletedItem); 238 | violationDeleteByThread += deletedItem; 239 | } 240 | 241 | protected void sendDeleteViolations() { 242 | int numOfDeletedItem = tableWriter.sendDeleteRequests(); 243 | violationsDeleted.addAndGet(numOfDeletedItem); 244 | violationDeleteByThread += numOfDeletedItem; 245 | } 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/TableWriter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.util.ArrayList; 18 | import java.util.HashMap; 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | import org.apache.log4j.Logger; 23 | 24 | import com.amazonaws.AmazonServiceException; 25 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 26 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 27 | import com.amazonaws.services.dynamodbv2.model.AttributeValueUpdate; 28 | import com.amazonaws.services.dynamodbv2.model.BatchWriteItemRequest; 29 | import com.amazonaws.services.dynamodbv2.model.BatchWriteItemResult; 30 | import com.amazonaws.services.dynamodbv2.model.DeleteRequest; 31 | import com.amazonaws.services.dynamodbv2.model.ExpectedAttributeValue; 32 | import com.amazonaws.services.dynamodbv2.model.ReturnConsumedCapacity; 33 | import com.amazonaws.services.dynamodbv2.model.ReturnValue; 34 | import com.amazonaws.services.dynamodbv2.model.UpdateItemRequest; 35 | import com.amazonaws.services.dynamodbv2.model.UpdateItemResult; 36 | import com.amazonaws.services.dynamodbv2.model.WriteRequest; 37 | import com.google.common.util.concurrent.RateLimiter; 38 | 39 | /** 40 | * Write delete or update to the table. 41 | * 42 | */ 43 | public class TableWriter { 44 | private String tableName; 45 | private String tableHashKeyName; 46 | private String tableRangeKeyName; 47 | private long totalNumOfItemsDeleted; 48 | private AmazonDynamoDBClient dynamoDBClient; 49 | private List batchDeleteRequests; 50 | public final static int MAX_BATCH_WRITE_REQUEST_NUM = 25; 51 | private TableRWRateLimiter tableWriteRateLimiter; 52 | 53 | private static final Logger logger = Logger.getLogger(TableWriter.class); 54 | 55 | // Used for running tests on DDB Local. Rate Limiter cannot be used with DDB Local. 56 | private static boolean isRunningOnDDBLocal = false; 57 | 58 | /** 59 | * Constructor for unit test. 60 | */ 61 | public TableWriter(String tableName, String tableHashKeyName, String tableRangeKeyName, AmazonDynamoDBClient dynamoDBClient, RateLimiter rateLimiter, 62 | ArrayList batchDeleteRequests) { 63 | this.tableName = tableName; 64 | this.tableHashKeyName = tableHashKeyName; 65 | this.tableRangeKeyName = tableRangeKeyName; 66 | this.dynamoDBClient = dynamoDBClient; 67 | this.batchDeleteRequests = batchDeleteRequests; 68 | } 69 | 70 | public TableWriter(Options options, TableHelper tableHelper, AmazonDynamoDBClient dynamoDBClient, int numOfTasks, boolean isRunningOnDDBLocal) { 71 | this.tableName = options.getTableName(); 72 | this.tableHashKeyName = tableHelper.getTableHashKeyName(); 73 | this.tableRangeKeyName = tableHelper.getTableRangeKeyName(); 74 | this.dynamoDBClient = dynamoDBClient; 75 | this.totalNumOfItemsDeleted = 0; 76 | batchDeleteRequests = new ArrayList(); 77 | tableWriteRateLimiter = new TableRWRateLimiter(tableHelper.getWriteCapacityUnits(), options.getReadWriteIOPSPercent(), numOfTasks); 78 | TableWriter.isRunningOnDDBLocal = isRunningOnDDBLocal; 79 | } 80 | 81 | /** 82 | * Add delete request to the bath write requests, since batch write has a 83 | * limit on number of requests, will send the request automatically if the 84 | * requests number limit reached. 85 | */ 86 | public int addDeleteRequest(Map item) { 87 | AttributeValue tableHashKey = item.get(tableHashKeyName); 88 | AttributeValue tableRangeKey = item.get(tableRangeKeyName); 89 | return addDeleteRequest(tableHashKey, tableRangeKey); 90 | } 91 | 92 | protected int addDeleteRequest(AttributeValue tableHashKey, AttributeValue tableRangeKey) { 93 | Map primaryKey = genTablePrimaryKey(tableHashKey, tableRangeKey); 94 | batchDeleteRequests.add(new WriteRequest().withDeleteRequest(new DeleteRequest().withKey(primaryKey))); 95 | int deletedItems = 0; 96 | if (batchDeleteRequests.size() == MAX_BATCH_WRITE_REQUEST_NUM) { 97 | deletedItems = sendDeleteRequests(); 98 | } 99 | return deletedItems; 100 | } 101 | 102 | public Map genTablePrimaryKey(AttributeValue tableHashKeyValue, AttributeValue tableRangeKeyValue) { 103 | Map primaryKey = new HashMap(); 104 | primaryKey.put(tableHashKeyName, tableHashKeyValue); 105 | if (tableRangeKeyValue != null) { 106 | primaryKey.put(tableRangeKeyName, tableRangeKeyValue); 107 | } 108 | return primaryKey; 109 | } 110 | 111 | public int sendDeleteRequests() throws IllegalArgumentException { 112 | if (batchDeleteRequests.isEmpty()) { 113 | return 0; 114 | } 115 | BatchWriteItemRequest batchWriteItemRequest = genBatchWriteItemRequest(); 116 | BatchWriteItemResult bathWriteResult = sendBatchWriteRequest(batchWriteItemRequest); 117 | int undeletedItemNum = countAndPrintUndeletedItems(bathWriteResult); 118 | if(!isRunningOnDDBLocal) { 119 | // DDB Local does not support rate limiting 120 | tableWriteRateLimiter.adjustRateWithConsumedCapacity(bathWriteResult.getConsumedCapacity()); 121 | } 122 | int deletedRequest = batchDeleteRequests.size() - undeletedItemNum; 123 | totalNumOfItemsDeleted += deletedRequest; 124 | PrintHelper.printDeleteProgressInfo(deletedRequest, totalNumOfItemsDeleted); 125 | batchDeleteRequests = new ArrayList(); 126 | return deletedRequest; 127 | } 128 | 129 | protected BatchWriteItemRequest genBatchWriteItemRequest() { 130 | Map> requestItems = new HashMap>(); 131 | requestItems.put(tableName, batchDeleteRequests); 132 | return new BatchWriteItemRequest().withRequestItems(requestItems).withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); 133 | } 134 | 135 | protected BatchWriteItemResult sendBatchWriteRequest(BatchWriteItemRequest batchWriteItemRequest) { 136 | try { 137 | return dynamoDBClient.batchWriteItem(batchWriteItemRequest); 138 | } catch (AmazonServiceException ase) { 139 | throw new IllegalArgumentException("Error: Failed to delete " + ase.getMessage()); 140 | } catch (IllegalArgumentException iae) { 141 | throw new IllegalArgumentException("Error: Invalid argument: " + iae.getMessage()); 142 | } 143 | } 144 | 145 | protected int countAndPrintUndeletedItems(BatchWriteItemResult batchWriteResult) { 146 | if (!batchWriteResult.getUnprocessedItems().isEmpty()) { 147 | logger.warn("WARNING: UNPROCESSED ITEMS:"); 148 | List unprocessedRequests = (batchWriteResult.getUnprocessedItems()).get(tableName); 149 | int count = 0; 150 | for (WriteRequest w : unprocessedRequests) { 151 | PrintHelper.printItem(0, count++, w.getDeleteRequest().getKey()); 152 | } 153 | return unprocessedRequests.size(); 154 | } 155 | return 0; 156 | } 157 | 158 | /** 159 | * Null or empty key value for table key is not allowed. 160 | */ 161 | public AttributeValue genAttributeValueForTableKey(String attributeType, String keyValue) throws IllegalArgumentException { 162 | if (keyValue == null || keyValue.isEmpty()) { 163 | throw new IllegalArgumentException("Error: Key value must not be empty."); 164 | } 165 | return genAttributeValueForKey(attributeType, keyValue); 166 | } 167 | 168 | /** 169 | * Empty GSI key value for GSI is allowed, since it will be interpreted as 170 | * delete operation. 171 | */ 172 | public AttributeValue genAttributeValueForGSIKey(String attributeType, String keyValue) throws IllegalArgumentException { 173 | if (keyValue == null || keyValue.isEmpty()) { 174 | return null; 175 | } 176 | return genAttributeValueForKey(attributeType, keyValue); 177 | } 178 | 179 | /** 180 | * For key, they are stored as blank value, so parse them as blank value 181 | */ 182 | protected AttributeValue genAttributeValueForKey(String attributeType, String keyValue) throws IllegalArgumentException { 183 | if (!attributeType.equals("S") && !attributeType.equals("N") && !attributeType.equals("B")) { 184 | throw new IllegalArgumentException("Error: Key attribute value must be S, B or N"); 185 | } 186 | return AttributeValueConverter.parseFromBlankString(attributeType, keyValue); 187 | } 188 | 189 | /** 190 | * For expected value, they are stored with their attribute type, so parse 191 | * them with their value. 192 | */ 193 | protected ExpectedAttributeValue genExpectedAttributeValue(String value) throws IllegalArgumentException { 194 | AttributeValue attributeValue = AttributeValueConverter.parseFromWithAttributeTypeString(value); 195 | return new ExpectedAttributeValue().withExists(true).withValue(attributeValue); 196 | } 197 | 198 | /** 199 | * Sends an update request to the service and returns true if the request is successful. 200 | */ 201 | public boolean sendUpdateRequest(Map primaryKey, Map updateItems, 202 | Map expectedItems) throws Exception { 203 | if (updateItems.isEmpty()) { 204 | return false; // No update, return false 205 | } 206 | UpdateItemRequest updateItemRequest = new UpdateItemRequest().withTableName(tableName).withKey(primaryKey).withReturnValues(ReturnValue.UPDATED_NEW) 207 | .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL).withAttributeUpdates(updateItems); 208 | if (expectedItems != null) { 209 | updateItemRequest.withExpected(expectedItems); 210 | } 211 | 212 | UpdateItemResult result = dynamoDBClient.updateItem(updateItemRequest); 213 | if(!isRunningOnDDBLocal) { 214 | // DDB Local does not support rate limiting 215 | tableWriteRateLimiter.adjustRateWithConsumedCapacity(result.getConsumedCapacity()); 216 | } 217 | return true; 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/ViolationChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.io.IOException; 18 | import java.nio.charset.Charset; 19 | import java.util.Map; 20 | 21 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 22 | import com.amazonaws.services.dynamodbv2.model.KeyType; 23 | import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; 24 | 25 | /** 26 | * Check item or attribute violations. 27 | * 28 | */ 29 | public class ViolationChecker { 30 | 31 | /** Violation Types */ 32 | public static final String SIZE_VIOLATION = "Size Violation"; 33 | public static final String TYPE_VIOLATION = "Type Violation"; 34 | 35 | /** Maximum Values in Bytes */ 36 | public static final int MAX_HASH_KEY_SIZE = 2048; 37 | public static final int MAX_RANGE_KEY_SIZE = 1024; 38 | 39 | /** Set Type */ 40 | public static final String SS = "SS"; 41 | public static final String NS = "NS"; 42 | public static final String BS = "BS"; 43 | 44 | /** Character set for encoding */ 45 | public final static Charset UTF8 = Charset.forName("UTF-8"); 46 | 47 | private boolean recordViolation = true; 48 | private boolean recordGsiValueInViolationRecord = false; 49 | private boolean tableHasRangeKey = false; 50 | private boolean checkGSIHashKey = false; 51 | private boolean checkGSIRangeKey = false; 52 | private String tableHashkeyName; 53 | private String tableRangeKeyName; 54 | private String GSIHashKeyName; 55 | private String GSIHashKeyType; 56 | private String GSIRangeKeyName; 57 | private String GSIRangeKeyType; 58 | private ViolationRecord violationRecord; 59 | private boolean isHashKeyViolation = false; 60 | private boolean isRangeKeyViolation = false; 61 | 62 | /** 63 | * Constructor for unit tests 64 | */ 65 | protected ViolationChecker(String outputFilePath, String tableHashKeyName, String tableRangeKeyName, String GSIHashKeyName, String GSIHashKeyType, 66 | String GSIRangeKeyName, String GSIRangeKeyType, boolean recordViolation, boolean recordGsiValueInViolationRecord, ViolationRecord violationRecord, 67 | ViolationWriter violationWriter) { 68 | this.tableHashkeyName = tableHashKeyName; 69 | this.tableRangeKeyName = tableRangeKeyName; 70 | if (tableRangeKeyName != null) { 71 | tableHasRangeKey = true; 72 | } 73 | 74 | if (GSIHashKeyName != null) { 75 | this.checkGSIHashKey = true; 76 | this.GSIHashKeyName = GSIHashKeyName; 77 | this.GSIHashKeyType = GSIHashKeyType; 78 | } 79 | 80 | if (GSIRangeKeyName != null) { 81 | this.checkGSIRangeKey = true; 82 | this.GSIRangeKeyName = GSIRangeKeyName; 83 | this.GSIRangeKeyType = GSIRangeKeyType; 84 | } 85 | 86 | this.recordViolation = recordViolation; 87 | this.recordGsiValueInViolationRecord = recordGsiValueInViolationRecord; 88 | this.violationRecord = violationRecord; 89 | } 90 | 91 | public ViolationChecker(Options options, TableHelper tableHelper) { 92 | this.tableHashkeyName = tableHelper.getTableHashKeyName(); 93 | this.tableRangeKeyName = tableHelper.getTableRangeKeyName(); 94 | if (tableRangeKeyName != null) { 95 | tableHasRangeKey = true; 96 | } 97 | 98 | this.GSIHashKeyName = options.getGsiHashKeyName(); 99 | if (GSIHashKeyName != null) { 100 | this.checkGSIHashKey = true; 101 | this.GSIHashKeyName = options.getGsiHashKeyName(); 102 | this.GSIHashKeyType = options.getGsiHashKeyType(); 103 | } 104 | 105 | this.GSIRangeKeyName = options.getGsiRangeKeyName(); 106 | if (GSIRangeKeyName != null) { 107 | this.checkGSIRangeKey = true; 108 | this.GSIRangeKeyName = options.getGsiRangeKeyName(); 109 | this.GSIRangeKeyType = options.getGsiRangeKeyType(); 110 | } 111 | 112 | this.recordViolation = options.recordDetails(); 113 | this.recordGsiValueInViolationRecord = options.recordGsiValueInViolationRecord(); 114 | if (recordViolation) { 115 | violationRecord = new ViolationRecord(tableHasRangeKey, checkGSIHashKey, checkGSIRangeKey, recordGsiValueInViolationRecord); 116 | } 117 | } 118 | 119 | public ViolationRecord getViolationRecordHead() throws IOException { 120 | return violationRecord.getViolationRecordHead(); 121 | } 122 | 123 | public ViolationRecord checkItemViolationAndGetRecord(Map item) { 124 | isHashKeyViolation = false; 125 | isRangeKeyViolation = false; 126 | 127 | if (recordViolation) { 128 | violationRecord.clear(); 129 | } 130 | 131 | if (checkGSIHashKey) { 132 | AttributeValue GSIHashKeyValue = item.get(GSIHashKeyName); 133 | /** If that value does not exist, ignore */ 134 | if (GSIHashKeyValue != null) { 135 | isHashKeyViolation = checkAttributeViolation(GSIHashKeyValue, GSIHashKeyType, KeyType.HASH); 136 | } 137 | } 138 | 139 | if (checkGSIRangeKey) { 140 | AttributeValue GSIRangeKeyValue = item.get(GSIRangeKeyName); 141 | if (GSIRangeKeyValue != null) { 142 | isRangeKeyViolation = checkAttributeViolation(GSIRangeKeyValue, GSIRangeKeyType, KeyType.RANGE); 143 | } 144 | } 145 | 146 | if (recordViolation && (isHashKeyViolation || isRangeKeyViolation)) { 147 | AttributeValue tableHashKeyValue = item.get(tableHashkeyName); 148 | AttributeValue tableRangeKeyValue = tableHasRangeKey ? item.get(tableRangeKeyName) : null; 149 | recordItemTablePrimaryKey(tableHashKeyValue, tableRangeKeyValue); 150 | } 151 | 152 | if (isHashKeyViolation || isRangeKeyViolation) { 153 | if(recordViolation) { 154 | return violationRecord; 155 | } else { 156 | // return any non-null violation record because null would mean that there is no violation. 157 | // Since we are not recording violations, this value will not be used. 158 | return new ViolationRecord(tableHasRangeKey, checkGSIHashKey, checkGSIRangeKey, recordGsiValueInViolationRecord); 159 | } 160 | } 161 | return null; 162 | } 163 | 164 | protected boolean checkAttributeViolation(AttributeValue keyValue, String expectedDatatype, KeyType keyType) { 165 | int maxKeySize = keyType.name().equals(KeyType.HASH.name()) ? MAX_HASH_KEY_SIZE : MAX_RANGE_KEY_SIZE; 166 | if (keyValue.getS() != null) { 167 | if (!ScalarAttributeType.S.name().equals(expectedDatatype)) { 168 | if (recordViolation) { 169 | recordTypeViolation(keyValue, keyType, expectedDatatype, ScalarAttributeType.S.name()); 170 | } 171 | return true; 172 | } else { 173 | int size = keyValue.getS().getBytes(UTF8).length; 174 | if (size > maxKeySize) { 175 | if (recordViolation) { 176 | recordSizeViolation(keyValue, size, keyType); 177 | } 178 | return true; 179 | } else { 180 | return false; 181 | } 182 | } 183 | } else if (keyValue.getN() != null) { 184 | if (!ScalarAttributeType.N.name().equals(expectedDatatype)) { 185 | if (recordViolation) { 186 | recordTypeViolation(keyValue, keyType, expectedDatatype, ScalarAttributeType.N.name()); 187 | } 188 | return true; 189 | } else { 190 | // There can be no size violation for Number datatype 191 | return false; 192 | } 193 | } else if (keyValue.getB() != null) { 194 | if (!ScalarAttributeType.B.name().equals(expectedDatatype)) { 195 | if (recordViolation) { 196 | recordTypeViolation(keyValue, keyType, expectedDatatype, ScalarAttributeType.B.name()); 197 | } 198 | return true; 199 | } else { 200 | int size = keyValue.getB().array().length; 201 | if (size > maxKeySize) { 202 | if (recordViolation) { 203 | recordSizeViolation(keyValue, size, keyType); 204 | } 205 | return true; 206 | } else { 207 | return false; 208 | } 209 | } 210 | } else if (keyValue.getSS() != null) { 211 | if (recordViolation) { 212 | recordTypeViolation(keyValue, keyType, expectedDatatype, SS); 213 | } 214 | return true; 215 | } else if (keyValue.getNS() != null) { 216 | if (recordViolation) { 217 | recordTypeViolation(keyValue, keyType, expectedDatatype, NS); 218 | } 219 | return true; 220 | } else if (keyValue.getBS() != null) { 221 | if (recordViolation) { 222 | recordTypeViolation(keyValue, keyType, expectedDatatype, BS); 223 | } 224 | return true; 225 | } 226 | return false; 227 | } 228 | 229 | /** For GSI Violation value, should store them with their attribute type */ 230 | protected void recordSizeViolation(AttributeValue keyValue, int size, KeyType keyType) { 231 | if (keyType == KeyType.HASH) { 232 | if (recordGsiValueInViolationRecord) { 233 | violationRecord.setGSIHashKey(AttributeValueConverter.toStringWithAttributeType(keyValue)); 234 | } 235 | violationRecord.setGSIHashKeyViolationType(SIZE_VIOLATION); 236 | violationRecord.setGSIHashKeyViolationDesc("Max Bytes Allowed: " + MAX_HASH_KEY_SIZE + " Found: " + size); 237 | } else if (keyType == KeyType.RANGE) { 238 | if (recordGsiValueInViolationRecord) { 239 | violationRecord.setGSIRangeKey(AttributeValueConverter.toStringWithAttributeType(keyValue)); 240 | } 241 | violationRecord.setGSIRangeKeyViolationType(SIZE_VIOLATION); 242 | violationRecord.setGSIRangeKeyViolationDesc("Max Bytes Allowed: " + MAX_RANGE_KEY_SIZE + " Found: " + size); 243 | } 244 | } 245 | 246 | protected void recordTypeViolation(AttributeValue keyValue, KeyType keyType, String expectedDatatype, String foundDatatype) { 247 | if (keyType == KeyType.HASH) { 248 | if (recordGsiValueInViolationRecord) { 249 | violationRecord.setGSIHashKey(AttributeValueConverter.toStringWithAttributeType(keyValue)); 250 | } 251 | violationRecord.setGSIHashKeyViolationType(TYPE_VIOLATION); 252 | violationRecord.setGSIHashKeyViolationDesc("Expected: " + expectedDatatype + " Found: " + foundDatatype); 253 | } else { 254 | if (recordGsiValueInViolationRecord) { 255 | violationRecord.setGSIRangeKey(AttributeValueConverter.toStringWithAttributeType(keyValue)); 256 | } 257 | violationRecord.setGSIRangeKeyViolationType(TYPE_VIOLATION); 258 | violationRecord.setGSIRangeKeyViolationDesc("Expected: " + expectedDatatype + " Found: " + foundDatatype); 259 | } 260 | } 261 | 262 | public void recordItemTablePrimaryKey(AttributeValue itemHashKey, AttributeValue itemRangeKey) { 263 | recordItemTableHashKey(itemHashKey); 264 | recordItemTableRangeKey(itemRangeKey); 265 | } 266 | 267 | /** For table hash key or range key, store them as blank strings */ 268 | protected void recordItemTableHashKey(AttributeValue itemHashKey) { 269 | if (itemHashKey.getS() == null && itemHashKey.getN() == null && itemHashKey.getB() == null) { 270 | throw new IllegalArgumentException("Error: Invalid hash key, should contains S, N or B only."); 271 | } 272 | violationRecord.setTableHashKey(AttributeValueConverter.toBlankString(itemHashKey)); 273 | } 274 | 275 | protected void recordItemTableRangeKey(AttributeValue itemRangeKey) { 276 | if (tableHasRangeKey == false) { 277 | if (itemRangeKey != null) { 278 | throw new IllegalArgumentException("Error: Table does not have range key but was provided."); 279 | } 280 | return; 281 | } 282 | if (itemRangeKey.getS() == null && itemRangeKey.getN() == null && itemRangeKey.getB() == null) { 283 | throw new IllegalArgumentException("Error: Invalid range key, should contains S, N or B only."); 284 | } 285 | violationRecord.setTableRangeKey(AttributeValueConverter.toBlankString(itemRangeKey)); 286 | } 287 | } 288 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/ViolationRecord.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | 20 | /** 21 | * Define the format of violation record. 22 | * 23 | */ 24 | public class ViolationRecord { 25 | 26 | /** Head for output records */ 27 | public static final String TABLE_HASH_KEY = "Table Hash Key"; 28 | public static final String TABLE_RANGE_KEY = "Table Range Key"; 29 | public static final String GSI_HASH_KEY = "GSI Hash Key Value"; 30 | public static final String GSI_HASH_KEY_VIOLATION_TYPE = "GSI Hash Key Violation Type"; 31 | public static final String GSI_HASH_KEY_VIOLATION_DESC = "GSI Hash Key Violation Description"; 32 | public static final String GSI_HASH_KEY_UPDATE_VALUE = "GSI Hash Key Update Value(FOR USER)"; 33 | public static final String GSI_RANGE_KEY = "GSI Range Key Value"; 34 | public static final String GSI_RANGE_KEY_VIOLATION_TYPE = "GSI Range Key Violation Type"; 35 | public static final String GSI_RANGE_KEY_VIOLATION_DESC = "GSI Range Key Violation Description"; 36 | public static final String GSI_RANGE_KEY_UPDATE_VALUE = "GSI Range Key Update Value(FOR USER)"; 37 | public static final String GSI_CORRECTION_DELETE_BLANK = "Delete Blank Attributes When Updating?(Y/N)"; 38 | public static final String GSI_CORRECTION_DELETE_BLANK_YES = "Y"; 39 | public static final String GSI_CORRECTION_DELETE_BLANK_NO = "N"; 40 | 41 | // This option is used only for violation correction output 42 | public static final String GSI_VALUE_UPDATE_ERROR = "Error While Updating Value"; 43 | 44 | private String tableHashKey = null; 45 | private String tableRangeKey = null; 46 | private String GSIHashKeyViolationType = null; 47 | private String GSIHashKey = null; 48 | private String GSIHashKeyViolationDesc = null; 49 | private String GSIHashKeyUpdateValue = ""; 50 | private String GSIRangeKeyViolationType = null; 51 | private String GSIRangeKey = null; 52 | private String GSIRangeKeyViolationDesc = null; 53 | private String GSIRangekeyUpdateValue = ""; 54 | private String GSIDeleteBlankOnUpdate = ""; 55 | 56 | /** Error messages */ 57 | private final String ERROR_CHECK_NO_GSI_KEY = "Error: Should check one GSI Key at least."; 58 | private final String ERROR_SET_TABLE_RANGE_KEY = "Error: tableHasRangeKey false, cannot set table range key"; 59 | private final String ERROR_SET_GSI_HASH_KEY = "Error: checkGSIHashKey false, cannot set GSI hash key"; 60 | private final String ERROR_SET_GSI_HASH_KEY_VIO_TYPE = "Error: checkGSIHashKey false, cannot set GSI hash key violation type"; 61 | private final String ERROR_SET_GSI_HASH_KEY_VIO_DESC = "Error: checkGSIHashKey false, cannot set GSI hash key violation description"; 62 | private final String ERROR_SET_GSI_HASH_KEY_UPDATE = "Error: checkGSIHashKey false, cannot set GSI hash key update value"; 63 | private final String ERROR_SET_GSI_RANGE_KEY = "Error: checkGSIRangeKey false, cannot set GSI range key"; 64 | private final String ERROR_MSG_SET_GSI_RANGE_KEY_VIO_TYPE = "Error: checkGSIRangeKey false, cannot set GSI range key violation type"; 65 | private final String ERROR_MSG_SET_GSI_RANGE_KEY_VIO_DESC = "Error: checkGSIRangeKey false, cannot set GSI range violation description"; 66 | private final String ERROR_MSG_GSI_RANGE_KEY_UPDATE = "Error: checkGSIRangeKey false, cannot set GSI range key update value"; 67 | 68 | private boolean tableHasRangeKey = false; 69 | private boolean checkGSIHashKey = false; 70 | private boolean checkGSIRangeKey = false; 71 | private boolean recordGsiValueInViolationRecord = false; 72 | private boolean GSIHashKeyViolationRecorded = false; 73 | private boolean GSIRangeKeyViolationRecorded = false; 74 | 75 | /** 76 | * User can choose to check GSI hash key or GSI range key independently by 77 | * enabling or disabling the two options. 78 | */ 79 | public ViolationRecord(boolean tableHasRangeKey, boolean checkGSIHashKey, boolean checkGSIRangeKey, boolean recordGsiValueInViolationRecord) { 80 | if (checkGSIHashKey == false && checkGSIRangeKey == false) { 81 | throw new IllegalArgumentException(ERROR_CHECK_NO_GSI_KEY); 82 | } 83 | 84 | this.tableHasRangeKey = tableHasRangeKey; 85 | this.checkGSIHashKey = checkGSIHashKey; 86 | this.checkGSIRangeKey = checkGSIRangeKey; 87 | this.recordGsiValueInViolationRecord = recordGsiValueInViolationRecord; 88 | } 89 | 90 | public ViolationRecord getViolationRecordHead() { 91 | ViolationRecord violationRecordHead = new ViolationRecord(tableHasRangeKey, checkGSIHashKey, checkGSIRangeKey, recordGsiValueInViolationRecord); 92 | violationRecordHead.setTableHashKey(TABLE_HASH_KEY); 93 | 94 | if (tableHasRangeKey) { 95 | violationRecordHead.setTableRangeKey(TABLE_RANGE_KEY); 96 | } 97 | 98 | if (checkGSIHashKey) { 99 | violationRecordHead.setGSIHashKey(GSI_HASH_KEY); 100 | violationRecordHead.setGSIHashKeyViolationType(GSI_HASH_KEY_VIOLATION_TYPE); 101 | violationRecordHead.setGSIHashKeyViolationDesc(GSI_HASH_KEY_VIOLATION_DESC); 102 | violationRecordHead.setGSIHashKeyUpdateValue(GSI_HASH_KEY_UPDATE_VALUE); 103 | } 104 | 105 | if (checkGSIRangeKey) { 106 | violationRecordHead.setGSIRangeKey(GSI_RANGE_KEY); 107 | violationRecordHead.setGSIRangeKeyViolationType(GSI_RANGE_KEY_VIOLATION_TYPE); 108 | violationRecordHead.setGSIRangeKeyViolationDesc(GSI_RANGE_KEY_VIOLATION_DESC); 109 | violationRecordHead.setGSIRangeKeyUpdateValue(GSI_RANGE_KEY_UPDATE_VALUE); 110 | } 111 | 112 | violationRecordHead.setGSIDeleteBlankOnUpdate(GSI_CORRECTION_DELETE_BLANK); 113 | return violationRecordHead; 114 | } 115 | 116 | /** 117 | * Convert the record into string list. If key checked, but violation not 118 | * found, will store empty string to keep the format consistent. 119 | */ 120 | public List toStringList() { 121 | List record = new ArrayList(); 122 | record.add(tableHashKey); 123 | if (tableHasRangeKey) { 124 | record.add(tableRangeKey); 125 | } 126 | 127 | if (checkGSIHashKey) { 128 | if (GSIHashKeyViolationRecorded) { 129 | if (recordGsiValueInViolationRecord) { 130 | record.add(GSIHashKey); 131 | } 132 | record.add(GSIHashKeyViolationType); 133 | record.add(GSIHashKeyViolationDesc); 134 | record.add(GSIHashKeyUpdateValue); 135 | } else { 136 | if (recordGsiValueInViolationRecord) { 137 | record.add(""); 138 | } 139 | record.add(""); 140 | record.add(""); 141 | record.add(""); 142 | } 143 | } 144 | 145 | if (checkGSIRangeKey) { 146 | if (GSIRangeKeyViolationRecorded) { 147 | if (recordGsiValueInViolationRecord) { 148 | record.add(GSIRangeKey); 149 | } 150 | record.add(GSIRangeKeyViolationType); 151 | record.add(GSIRangeKeyViolationDesc); 152 | record.add(GSIRangekeyUpdateValue); 153 | } else { 154 | if (recordGsiValueInViolationRecord) { 155 | record.add(""); 156 | } 157 | record.add(""); 158 | record.add(""); 159 | record.add(""); 160 | } 161 | } 162 | 163 | record.add(GSIDeleteBlankOnUpdate); 164 | return record; 165 | } 166 | 167 | public void clear() { 168 | GSIHashKeyViolationRecorded = false; 169 | GSIRangeKeyViolationRecorded = false; 170 | } 171 | 172 | public void setTableHashKey(String tableHashKey) { 173 | this.tableHashKey = tableHashKey; 174 | } 175 | 176 | public void setTableRangeKey(String tableRangeKey) { 177 | if (!tableHasRangeKey) { 178 | throw new IllegalArgumentException(ERROR_SET_TABLE_RANGE_KEY); 179 | } 180 | this.tableRangeKey = tableRangeKey; 181 | } 182 | 183 | public void setGSIHashKey(String GSIHashKey) { 184 | if (!checkGSIHashKey) { 185 | throw new IllegalArgumentException(ERROR_SET_GSI_HASH_KEY); 186 | } 187 | this.GSIHashKey = GSIHashKey; 188 | GSIHashKeyViolationRecorded = true; 189 | } 190 | 191 | public void setGSIHashKeyViolationType(String GSIHashKeyViolationType) { 192 | if (!checkGSIHashKey) { 193 | throw new IllegalArgumentException(ERROR_SET_GSI_HASH_KEY_VIO_TYPE); 194 | } 195 | this.GSIHashKeyViolationType = GSIHashKeyViolationType; 196 | GSIHashKeyViolationRecorded = true; 197 | } 198 | 199 | public void setGSIHashKeyViolationDesc(String GSIHashKeyViolationDesc) { 200 | if (!checkGSIHashKey) { 201 | throw new IllegalArgumentException(ERROR_SET_GSI_HASH_KEY_VIO_DESC); 202 | } 203 | this.GSIHashKeyViolationDesc = GSIHashKeyViolationDesc; 204 | } 205 | 206 | public void setGSIHashKeyUpdateValue(String GSIHashKeyUpdateValue) { 207 | if (!checkGSIHashKey) { 208 | throw new IllegalArgumentException(ERROR_SET_GSI_HASH_KEY_UPDATE); 209 | } 210 | this.GSIHashKeyUpdateValue = GSIHashKeyUpdateValue; 211 | } 212 | 213 | public void setGSIRangeKey(String GSIRangeKey) { 214 | if (!checkGSIRangeKey) { 215 | throw new IllegalArgumentException(ERROR_SET_GSI_RANGE_KEY); 216 | } 217 | this.GSIRangeKey = GSIRangeKey; 218 | GSIRangeKeyViolationRecorded = true; 219 | } 220 | 221 | public void setGSIRangeKeyViolationType(String GSIRangeKeyViolationType) { 222 | if (!checkGSIRangeKey) { 223 | throw new IllegalArgumentException(ERROR_MSG_SET_GSI_RANGE_KEY_VIO_TYPE); 224 | } 225 | this.GSIRangeKeyViolationType = GSIRangeKeyViolationType; 226 | GSIRangeKeyViolationRecorded = true; 227 | } 228 | 229 | public void setGSIRangeKeyViolationDesc(String GSIRangeKeyViolationDesc) { 230 | if (!checkGSIRangeKey) { 231 | throw new IllegalArgumentException(ERROR_MSG_SET_GSI_RANGE_KEY_VIO_DESC); 232 | } 233 | this.GSIRangeKeyViolationDesc = GSIRangeKeyViolationDesc; 234 | } 235 | 236 | public void setGSIRangeKeyUpdateValue(String GSIRangekeyUpdateValue) { 237 | if (!checkGSIRangeKey) { 238 | throw new IllegalArgumentException(ERROR_MSG_GSI_RANGE_KEY_UPDATE); 239 | } 240 | this.GSIRangekeyUpdateValue = GSIRangekeyUpdateValue; 241 | } 242 | 243 | public void setGSIDeleteBlankOnUpdate(String GSIDeleteBlankOnUpdate) { 244 | this.GSIDeleteBlankOnUpdate = GSIDeleteBlankOnUpdate; 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /src/main/java/com/amazonaws/services/dynamodbv2/online/index/ViolationWriter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.io.BufferedWriter; 18 | import java.io.File; 19 | import java.io.FileWriter; 20 | import java.io.IOException; 21 | import java.util.List; 22 | 23 | import org.apache.commons.csv.CSVFormat; 24 | import org.apache.commons.csv.CSVPrinter; 25 | 26 | /** 27 | * Write violation records to file. 28 | * 29 | */ 30 | public class ViolationWriter { 31 | private BufferedWriter bufferWriter = null; 32 | private CSVPrinter printer = null; 33 | private CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(',').withIgnoreEmptyLines(true); 34 | private static ViolationWriter instance = new ViolationWriter(); 35 | 36 | private ViolationWriter() { 37 | }; 38 | 39 | public static ViolationWriter getInstance() { 40 | return instance; 41 | } 42 | 43 | public void createOutputFile(String outputFilePath) throws IOException { 44 | File outputFile = new File(outputFilePath); 45 | if (outputFile.exists()) { 46 | outputFile.delete(); 47 | } 48 | outputFile.createNewFile(); 49 | FileWriter out = new FileWriter(outputFilePath, true); 50 | bufferWriter = new BufferedWriter(out); 51 | printer = new CSVPrinter(bufferWriter, format); 52 | } 53 | 54 | public void addViolationRecord(ViolationRecord violationRecord) throws IOException { 55 | synchronized (this) { 56 | if (violationRecord != null) { 57 | printer.printRecord(violationRecord.toStringList()); 58 | } 59 | } 60 | } 61 | 62 | public void addViolationRecord(List record) throws IOException { 63 | synchronized (this) { 64 | if (record != null) { 65 | printer.printRecord(record); 66 | } 67 | } 68 | } 69 | 70 | public void flushAndCloseWriter() throws IOException { 71 | if(bufferWriter != null) { 72 | bufferWriter.flush(); 73 | bufferWriter.close(); 74 | } 75 | 76 | if(printer != null) { 77 | printer.close(); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger = INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.Target=System.out 5 | log4j.appender.stdout.ImmediateFlush=true 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.conversionPattern=%d %p [%c] - %m%n -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/AWSConnectionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertTrue; 18 | 19 | import java.io.FileNotFoundException; 20 | import java.io.IOException; 21 | 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | import org.mockito.Mockito; 25 | 26 | import com.amazonaws.auth.AWSCredentials; 27 | import com.amazonaws.regions.Region; 28 | import com.amazonaws.regions.Regions; 29 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 30 | import com.amazonaws.services.dynamodbv2.online.index.AWSConnection; 31 | import com.amazonaws.services.s3.AmazonS3Client; 32 | 33 | /** 34 | * Tests for AWSConnection. 35 | * 36 | */ 37 | public class AWSConnectionTest { 38 | /** Mock objects */ 39 | private AWSCredentials mockAWSCredentials = Mockito.mock(AWSCredentials.class); 40 | 41 | private AWSConnection awsConnection; 42 | 43 | @Before 44 | public void setupBeforeTest() { 45 | awsConnection = new AWSConnection(mockAWSCredentials); 46 | } 47 | 48 | @Test(expected = IllegalArgumentException.class) 49 | public void testConstructWithInvalidCredentialFilePath() throws FileNotFoundException, IOException { 50 | new AWSConnection("invalid file path"); 51 | } 52 | 53 | @Test 54 | public void testGetDynamoDBClient() { 55 | assertTrue(awsConnection.getDynamoDBClient(Region.getRegion(Regions.EU_WEST_1), false /*runOnDDBLocal*/) instanceof AmazonDynamoDBClient); 56 | } 57 | 58 | @Test 59 | public void testGetS3Client() { 60 | assertTrue(awsConnection.getS3Client() instanceof AmazonS3Client); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/CorrectionReaderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertFalse; 18 | import static org.junit.Assert.assertTrue; 19 | 20 | import java.io.FileNotFoundException; 21 | import java.io.IOException; 22 | import java.io.Reader; 23 | import java.util.Iterator; 24 | 25 | import org.apache.commons.csv.CSVRecord; 26 | import org.junit.Before; 27 | import org.junit.Test; 28 | import org.mockito.Mockito; 29 | 30 | import com.amazonaws.services.dynamodbv2.online.index.CorrectionReader; 31 | 32 | /** 33 | * 34 | * Unit test for ViolationWriter 35 | * 36 | */ 37 | public class CorrectionReaderTest { 38 | 39 | /** Mock objects */ 40 | @SuppressWarnings("unchecked") 41 | Iterator mockIterator = Mockito.mock(Iterator.class); 42 | Reader mockReader = Mockito.mock(Reader.class); 43 | 44 | CorrectionReader correctionReader; 45 | RandomDataGenerator randDataGenerator = new RandomDataGenerator(); 46 | 47 | @Before 48 | public void setupBeforeTest() { 49 | correctionReader = new CorrectionReader(mockReader, mockIterator); 50 | } 51 | 52 | @Test(expected = FileNotFoundException.class) 53 | public void testLoadCSVFileWithInvalidFilePath() throws IOException { 54 | CorrectionReader correctionReader = new CorrectionReader(); 55 | correctionReader.loadCSVFile("invalidFilePath"); 56 | } 57 | 58 | @Test 59 | public void testMovoToNextRecordIfHasWithValidNextValue() { 60 | Mockito.when(mockIterator.hasNext()).thenReturn(true); 61 | Mockito.when(mockIterator.next()).thenReturn(null); 62 | assertTrue(correctionReader.moveToNextRecordIfHas()); 63 | } 64 | 65 | @Test 66 | public void testMovoToNextRecordIfHasWithNoNextValue() { 67 | Mockito.when(mockIterator.hasNext()).thenReturn(false); 68 | assertFalse(correctionReader.moveToNextRecordIfHas()); 69 | } 70 | 71 | @Test 72 | public void testGetValueInRecordByName() throws IOException { 73 | // CSVRecord cannot be constructed and mocked 74 | } 75 | 76 | @Test 77 | public void testGetValueInRecordByNameWithIllegalName() { 78 | // CSVRecord cannot be constructed and mocked 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/CorrectionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | import static org.junit.Assert.assertFalse; 19 | import static org.junit.Assert.assertTrue; 20 | 21 | import java.io.FileNotFoundException; 22 | import java.io.IOException; 23 | 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | import org.mockito.Mockito; 27 | 28 | import com.amazonaws.services.dynamodbv2.online.index.Correction; 29 | import com.amazonaws.services.dynamodbv2.online.index.CorrectionReader; 30 | import com.amazonaws.services.dynamodbv2.online.index.Options; 31 | import com.amazonaws.services.dynamodbv2.online.index.TableHelper; 32 | import com.amazonaws.services.dynamodbv2.online.index.TableWriter; 33 | import com.amazonaws.services.dynamodbv2.online.index.ViolationRecord; 34 | 35 | /** 36 | * 37 | * Unit tests for Correction.java 38 | * 39 | */ 40 | public class CorrectionTest { 41 | private RandomDataGenerator randDataGenerator = new RandomDataGenerator(); 42 | 43 | /** Mock objects */ 44 | private Options mockOptions = Mockito.mock(Options.class); 45 | private CorrectionReader mockCorrectionReader = Mockito.mock(CorrectionReader.class); 46 | private TableHelper mockTableHelper = Mockito.mock(TableHelper.class); 47 | private TableWriter mockTableWriter = Mockito.mock(TableWriter.class); 48 | 49 | private Correction correction; 50 | private String rangeKeyName = "rangeKey"; 51 | private String gsiHashKeyName = "gsiHashKey"; 52 | private String gsiRangeKeyName = "gsiRangeKey"; 53 | private String gsiHashKeyType = "S"; 54 | private String gsiRangeKyeType = "N"; 55 | private String correctionFilePath = "/correction/file"; 56 | 57 | @Before 58 | public void setupBeforeTests() throws Exception { 59 | correction = new Correction(mockOptions, mockCorrectionReader, mockTableHelper, mockTableWriter); 60 | /** Setup stubs for testing */ 61 | Mockito.when(mockOptions.getCorrectionInputPath()).thenReturn("randompath"); 62 | Mockito.when(mockOptions.isCorrectionInputS3Path()).thenReturn(false); 63 | Mockito.when(mockOptions.getGsiHashKeyName()).thenReturn(gsiHashKeyName); 64 | Mockito.when(mockOptions.getGsiHashKeyType()).thenReturn(gsiHashKeyType); 65 | Mockito.when(mockOptions.getGsiRangeKeyName()).thenReturn(gsiRangeKeyName); 66 | Mockito.when(mockOptions.getGsiRangeKeyType()).thenReturn(gsiRangeKyeType); 67 | } 68 | 69 | @Test 70 | public void testLoadCorrectionFile() throws IOException { 71 | correction.loadRecordsFromCorrectionFile(correctionFilePath); 72 | Mockito.verify(mockCorrectionReader).loadCSVFile(correctionFilePath); 73 | } 74 | 75 | @Test(expected = IllegalArgumentException.class) 76 | public void testLoadCorrectionFileWithInvalidFilePath() throws IOException { 77 | Mockito.doThrow(new FileNotFoundException()).when(mockCorrectionReader).loadCSVFile(Mockito.anyString()); 78 | correction.loadRecordsFromCorrectionFile(correctionFilePath); 79 | } 80 | 81 | @Test(expected = IllegalArgumentException.class) 82 | public void testLoadCorrectionFileWithIOException() throws IOException { 83 | Mockito.doThrow(new IOException()).when(mockCorrectionReader).loadCSVFile(Mockito.anyString()); 84 | correction.loadRecordsFromCorrectionFile(correctionFilePath); 85 | } 86 | 87 | @Test 88 | public void testGetNextTableHashKey() { 89 | String testTableHashKey = randDataGenerator.nextRadomString(10); 90 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.TABLE_HASH_KEY)).thenReturn(testTableHashKey); 91 | assertEquals("Should return the next table hash key", testTableHashKey, correction.getNextTableHashKey()); 92 | } 93 | 94 | @Test(expected = IllegalArgumentException.class) 95 | public void testGetNextTableHashKeyWithValueNotFound() { 96 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.TABLE_HASH_KEY)).thenReturn(null); 97 | correction.getNextTableHashKey(); 98 | } 99 | 100 | @Test 101 | public void testGetNextTableRangeKey() { 102 | String testTableRangeKey = randDataGenerator.nextRadomString(10); 103 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.TABLE_RANGE_KEY)).thenReturn(testTableRangeKey); 104 | Mockito.when(mockTableHelper.getTableRangeKeyName()).thenReturn(rangeKeyName); 105 | assertEquals("Should get the next table range key", testTableRangeKey, correction.getNextTableRangeKey()); 106 | } 107 | 108 | @Test 109 | public void testGetNextTableRangeKeyWithNoRangeKeyOnTable() { 110 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.TABLE_RANGE_KEY)).thenReturn(null); 111 | Mockito.when(mockTableHelper.getTableRangeKeyName()).thenReturn(null); 112 | assertEquals("Should get the next table range key", null, correction.getNextTableRangeKey()); 113 | } 114 | 115 | @Test(expected = IllegalArgumentException.class) 116 | public void testGetNextTableRangeKeyWithNoTableRangeKeyButFoundOnInputFile() { 117 | String testTableRangeKey = randDataGenerator.nextRadomString(10); 118 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.TABLE_RANGE_KEY)).thenReturn(testTableRangeKey); 119 | Mockito.when(mockTableHelper.getTableRangeKeyName()).thenReturn(null); 120 | correction.getNextTableRangeKey(); 121 | } 122 | 123 | @Test(expected = IllegalArgumentException.class) 124 | public void testGetNextTableRangeKeyWithTableRangeKeyButNotFoundOnFile() { 125 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.TABLE_RANGE_KEY)).thenReturn(null); 126 | Mockito.when(mockTableHelper.getTableRangeKeyName()).thenReturn(rangeKeyName); 127 | correction.getNextTableRangeKey(); 128 | } 129 | 130 | @Test 131 | public void testGetNextGsiHashKeyUpdateValue() { 132 | String testGsiHashKeyUpdateValue = randDataGenerator.nextRadomString(10); 133 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_HASH_KEY_UPDATE_VALUE)).thenReturn(testGsiHashKeyUpdateValue); 134 | Mockito.when(mockOptions.getGsiHashKeyName()).thenReturn(gsiHashKeyName); 135 | assertEquals("Should return the given gsi hash key update value", testGsiHashKeyUpdateValue, correction.getNextGsiHashKeyUpdateValue()); 136 | } 137 | 138 | @Test 139 | public void testGetNextGsiHashKeyUpdateValueWithNoGsiHashKey() { 140 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_HASH_KEY_UPDATE_VALUE)).thenReturn(null); 141 | Mockito.when(mockOptions.getGsiHashKeyName()).thenReturn(null); 142 | assertEquals("Should return the given gsi hash key update value", null, correction.getNextGsiHashKeyUpdateValue()); 143 | } 144 | 145 | @Test 146 | public void testGetNextGsiHashKeyUpdateValueWithNoGsiHashKeyButProvidedOnInputFile() { 147 | String testGsiHashKeyUpdateValue = randDataGenerator.nextRadomString(10); 148 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_HASH_KEY_UPDATE_VALUE)).thenReturn(testGsiHashKeyUpdateValue); 149 | Mockito.when(mockOptions.getGsiHashKeyName()).thenReturn(null); 150 | assertEquals("Should return the given gsi hash key update value", testGsiHashKeyUpdateValue, correction.getNextGsiHashKeyUpdateValue()); 151 | } 152 | 153 | @Test 154 | public void testGetNextGsiHashKeyUpdateValueWithGsiHashKeyButNotProvidedOnInputFile() { 155 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_HASH_KEY_UPDATE_VALUE)).thenReturn(null); 156 | Mockito.when(mockOptions.getGsiHashKeyName()).thenReturn(gsiHashKeyName); 157 | assertEquals("Should return null", null, correction.getNextGsiHashKeyUpdateValue()); 158 | } 159 | 160 | @Test 161 | public void testGetNextGsiRangeKeyUpdateValue() { 162 | String testGsiRangeKeyUpdateValue = randDataGenerator.nextRadomString(10); 163 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_RANGE_KEY_UPDATE_VALUE)).thenReturn(testGsiRangeKeyUpdateValue); 164 | Mockito.when(mockOptions.getGsiRangeKeyName()).thenReturn(gsiRangeKeyName); 165 | assertEquals("Should return next range key update value", testGsiRangeKeyUpdateValue, correction.getNextGsiRangeKeyUpdateValue()); 166 | } 167 | 168 | @Test 169 | public void testGetNextGsiRangeKeyUpdateValueWithNoGsiRangeKey() { 170 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_RANGE_KEY_UPDATE_VALUE)).thenReturn(null); 171 | Mockito.when(mockOptions.getGsiRangeKeyName()).thenReturn(null); 172 | assertEquals("Should return next range key update value", null, correction.getNextGsiRangeKeyUpdateValue()); 173 | } 174 | 175 | @Test 176 | public void testGetNextGsiRangeKeyUpdateValueWithNoGsiRangeKeyButFoundOnInputFile() { 177 | String testGsiRangeKeyUpdateValue = randDataGenerator.nextRadomString(10); 178 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_RANGE_KEY_UPDATE_VALUE)).thenReturn(testGsiRangeKeyUpdateValue); 179 | Mockito.when(mockOptions.getGsiRangeKeyName()).thenReturn(null); 180 | assertEquals("Should return next range key update value", testGsiRangeKeyUpdateValue, correction.getNextGsiRangeKeyUpdateValue()); 181 | } 182 | 183 | @Test 184 | public void testGetNextGsiRangeKeyUpdateValueWithGsiRangeKeyButNotFoundOnInputFile() { 185 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_RANGE_KEY_UPDATE_VALUE)).thenReturn(null); 186 | Mockito.when(mockOptions.getGsiRangeKeyName()).thenReturn(gsiRangeKeyName); 187 | assertEquals("Should have returned null", null, correction.getNextGsiRangeKeyUpdateValue()); 188 | } 189 | 190 | @Test 191 | public void testGetNextDeleteBlankAttributeYes() { 192 | String testNextDeleteBlankAttributeString = "Y"; 193 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_CORRECTION_DELETE_BLANK)).thenReturn(testNextDeleteBlankAttributeString); 194 | assertTrue(correction.getNextDeleteBlankAttribute()); 195 | } 196 | 197 | @Test 198 | public void testGetNextDeleteBlankAttributeNO() { 199 | String testNextDeleteBlankAttributeString = "N"; 200 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_CORRECTION_DELETE_BLANK)).thenReturn(testNextDeleteBlankAttributeString); 201 | assertFalse(correction.getNextDeleteBlankAttribute()); 202 | } 203 | 204 | @Test 205 | public void testGetNextDeleteBlankAttributeEmpty() { 206 | String testNextDeleteBlankAttributeString = ""; 207 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_CORRECTION_DELETE_BLANK)).thenReturn(testNextDeleteBlankAttributeString); 208 | assertFalse(correction.getNextDeleteBlankAttribute()); 209 | } 210 | @Test 211 | public void testGetNextDeleteBlankAttributeWithNoColumn() { 212 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_CORRECTION_DELETE_BLANK)).thenReturn(null); 213 | correction.getNextDeleteBlankAttribute(); 214 | assertFalse(correction.getNextDeleteBlankAttribute()); 215 | } 216 | 217 | @Test(expected = IllegalArgumentException.class) 218 | public void testGetNextDeleteBlankAttributeWithInvalidValue() { 219 | String testNextDeleteBlankAttributeString = "invalidValue"; 220 | Mockito.when(mockCorrectionReader.getValueInRecordByName(ViolationRecord.GSI_CORRECTION_DELETE_BLANK)).thenReturn(testNextDeleteBlankAttributeString); 221 | assertFalse(correction.getNextDeleteBlankAttribute()); 222 | } 223 | 224 | 225 | } 226 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/OptionCheckerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertFalse; 18 | import static org.junit.Assert.assertTrue; 19 | 20 | import org.junit.Test; 21 | 22 | import com.amazonaws.services.dynamodbv2.online.index.OptionChecker; 23 | 24 | /** 25 | * 26 | * Unit tests for OptionChecker. 27 | * 28 | */ 29 | public class OptionCheckerTest { 30 | RandomDataGenerator randDataGenerator = new RandomDataGenerator(); 31 | OptionChecker optionChecker = new OptionChecker(); 32 | 33 | @Test 34 | public void testIsValidKeyTypeWithValidKeyType() { 35 | assertTrue(optionChecker.isValidKeyType("S")); 36 | assertTrue(optionChecker.isValidKeyType("N")); 37 | assertTrue(optionChecker.isValidKeyType("B")); 38 | } 39 | 40 | @Test 41 | public void testIsValidKeyTypeWithInvalidKeyType(){ 42 | assertFalse(optionChecker.isValidKeyType("NS")); 43 | } 44 | 45 | @Test 46 | public void testIsS3PathWithValidS3Path() { 47 | assertTrue(optionChecker.isS3Path("s3://fads")); 48 | } 49 | 50 | @Test 51 | public void testIsS3PathWithInvalidS3Path(){ 52 | assertFalse(optionChecker.isS3Path("fadsfasd")); 53 | } 54 | 55 | @Test 56 | public void testCheckNumberRangeWithInRange() { 57 | int lowerBound = Math.abs(randDataGenerator.nextRandomInt(1000)) + 1; 58 | int number = 2 * lowerBound; 59 | int higherBound = 3 * lowerBound; 60 | assertTrue(optionChecker.isNumberInRange(number, lowerBound, higherBound)); 61 | } 62 | 63 | @Test 64 | public void testCheckNumberRangeBeyondRange(){ 65 | int lowerBound = Math.abs(randDataGenerator.nextRandomInt(1000)) + 1; 66 | int number = 3 * lowerBound; 67 | int higherBound = 2 * lowerBound; 68 | assertFalse(optionChecker.isNumberInRange(number, lowerBound, higherBound)); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/OptionLoaderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | 19 | import java.util.Properties; 20 | 21 | import org.junit.Before; 22 | import org.junit.Test; 23 | import org.mockito.Mockito; 24 | 25 | import com.amazonaws.regions.Region; 26 | import com.amazonaws.regions.Regions; 27 | import com.amazonaws.services.dynamodbv2.online.index.OptionChecker; 28 | import com.amazonaws.services.dynamodbv2.online.index.OptionLoader; 29 | import com.amazonaws.services.dynamodbv2.online.index.Options; 30 | 31 | /** 32 | * Unit tests for OptionManager. 33 | * 34 | * 35 | */ 36 | public class OptionLoaderTest { 37 | private Properties mockProperties = Mockito.mock(Properties.class); 38 | private OptionChecker mockOptionChecker = Mockito.mock(OptionChecker.class); 39 | private Options mockOptions = Mockito.mock(Options.class); 40 | private OptionLoader optionLoader; 41 | 42 | @Before 43 | public void setupBeforeTest() { 44 | optionLoader = new OptionLoader(mockProperties, mockOptionChecker, mockOptions); 45 | } 46 | 47 | @Test 48 | public void testGetOptions() { 49 | assertEquals("Should return the options instance", optionLoader.getOptions(), mockOptions); 50 | } 51 | 52 | @Test 53 | public void testLoadCredentialFilePath() { 54 | String awsCredentialsFile = "/Documents/credentialfilePath"; 55 | Mockito.when(mockProperties.getProperty(Options.AWS_CREDENTIAL_FILE)).thenReturn(awsCredentialsFile); 56 | assertEquals("Should return the credential fiel on property file", awsCredentialsFile, optionLoader.loadCredentialFilePath()); 57 | } 58 | 59 | @Test(expected = IllegalArgumentException.class) 60 | public void testLoadCredentialFilePathWithValueMissing() { 61 | Mockito.when(mockProperties.getProperty(Options.AWS_CREDENTIAL_FILE)).thenReturn(null); 62 | optionLoader.loadCredentialFilePath(); 63 | } 64 | 65 | @Test 66 | public void testLoadDynamoDBRegion() { 67 | String dynamoDBRegion = "us-west-2"; 68 | Mockito.when(mockProperties.getProperty(Options.DYNAMODB_REGION)).thenReturn(dynamoDBRegion); 69 | assertEquals("Should return the valid region", Region.getRegion(Regions.fromName(dynamoDBRegion)), optionLoader.loadDynamoDBRegion()); 70 | } 71 | 72 | @Test(expected = IllegalArgumentException.class) 73 | public void testLoadDynamoDBRegionWithInvalidRegionname() { 74 | Mockito.when(mockProperties.getProperty(Options.DYNAMODB_REGION)).thenReturn("invalidregion"); 75 | optionLoader.loadDynamoDBRegion(); 76 | } 77 | 78 | @Test 79 | public void testLoadTableName() { 80 | String tableName = "Table1"; 81 | Mockito.when(mockProperties.getProperty(Options.TABLE_NAME)).thenReturn(tableName); 82 | assertEquals("Should return the right table name", tableName, optionLoader.loadTableName()); 83 | } 84 | 85 | @Test(expected = IllegalArgumentException.class) 86 | public void testLoadTableNameWithValueMissing() { 87 | Mockito.when(mockProperties.getProperty(Options.TABLE_NAME)).thenReturn(null); 88 | optionLoader.loadTableName(); 89 | } 90 | 91 | @Test 92 | public void testLoadGsiHashKeyName() { 93 | String gsiHashKeyName = "gsiHashKeyName"; 94 | Mockito.when(mockProperties.getProperty(Options.GSI_HASH_KEY_NAME, null)).thenReturn(gsiHashKeyName); 95 | assertEquals("Should return the right gsi hash key name", gsiHashKeyName, optionLoader.loadGsiHashKeyName()); 96 | } 97 | 98 | @Test 99 | public void testLoadGsiHashKeyType() { 100 | String gsiHashKeyType = "N"; 101 | Mockito.when(mockProperties.getProperty(Options.GSI_HASH_KEY_TYPE, null)).thenReturn(gsiHashKeyType); 102 | Mockito.when(mockOptionChecker.isValidKeyType(gsiHashKeyType)).thenReturn(true); 103 | assertEquals("Should return the right gsi hash key tyype", gsiHashKeyType, optionLoader.loadGsiHashKeyType()); 104 | } 105 | 106 | @Test(expected = IllegalArgumentException.class) 107 | public void testLoadGsiHashKeyTypeWithInvalidKeyType() { 108 | String invalidGsiRangeKeyType = "invalidKeyType"; 109 | Mockito.when(mockProperties.getProperty(Options.GSI_HASH_KEY_TYPE, null)).thenReturn(invalidGsiRangeKeyType); 110 | Mockito.when(mockOptionChecker.isValidKeyType(invalidGsiRangeKeyType)).thenReturn(false); 111 | optionLoader.loadDetectionOptions(); 112 | } 113 | 114 | @Test 115 | public void testCheckGsiHashKey() { 116 | optionLoader.checkGsiHashKey("gsiHashKeyName", "N"); 117 | } 118 | 119 | @Test(expected = IllegalArgumentException.class) 120 | public void testCheckGsiHashKeyWithHashKeyNameMissing() { 121 | optionLoader.checkGsiHashKey(null, "N"); 122 | } 123 | 124 | @Test(expected = IllegalArgumentException.class) 125 | public void testCheckGsiHashKeyWithHashKeyTypeMissing() { 126 | optionLoader.checkGsiHashKey("gsiHashKeyName", null); 127 | } 128 | 129 | @Test 130 | public void testLoadGsiRangeKeyName() { 131 | String gsiRangeKeyName = "gsiRangeKeyName"; 132 | Mockito.when(mockProperties.getProperty(Options.GSI_RANGE_KEY_NAME, null)).thenReturn(gsiRangeKeyName); 133 | assertEquals("Should return the right gsi range key name", gsiRangeKeyName, optionLoader.loadGsiRangeKeyName()); 134 | } 135 | 136 | @Test 137 | public void testLoadGsiRangeKeyType(){ 138 | String gsiRangeKeyType = "S"; 139 | Mockito.when(mockProperties.getProperty(Options.GSI_RANGE_KEY_TYPE, null)).thenReturn(gsiRangeKeyType); 140 | Mockito.when(mockOptionChecker.isValidKeyType(gsiRangeKeyType)).thenReturn(true); 141 | assertEquals("Should return right gsi range key type", gsiRangeKeyType, optionLoader.loadGsiRangeKeyType()); 142 | } 143 | 144 | @Test(expected = IllegalArgumentException.class) 145 | public void testLoadGsiRangeKeyTypeWithInvalidType() { 146 | Mockito.when(mockProperties.getProperty(Options.GSI_RANGE_KEY_TYPE, null)).thenReturn("invalidtype"); 147 | optionLoader.loadDetectionOptions(); 148 | } 149 | 150 | @Test 151 | public void testCheckGsiRangeKey(){ 152 | optionLoader.checkGsiRangeKey("gsiRangeKeyName", "S"); 153 | } 154 | 155 | @Test(expected = IllegalArgumentException.class) 156 | public void testCheckGsiRangeKeyWithRangeKeyNameMissing(){ 157 | optionLoader.checkGsiRangeKey(null, "N"); 158 | } 159 | 160 | @Test(expected = IllegalArgumentException.class) 161 | public void testCheckGsiRangeKeyWithRangeKeyTypeMissing(){ 162 | optionLoader.checkGsiRangeKey("gsiRangeKeyName", null); 163 | } 164 | 165 | @Test 166 | public void testCheckGsiHashKeyAndRangeKey(){ 167 | optionLoader.checkGsiHashKeyAndRangeKey("gsiHashKeyName", "gsiRangeKeyName"); 168 | } 169 | 170 | @Test 171 | public void testCheckGsiHashKeyAndRangeKeyWithRangeKeyNameEmpty(){ 172 | optionLoader.checkGsiHashKeyAndRangeKey("gsiHashKeyName", null); 173 | } 174 | 175 | @Test 176 | public void testCheckGsiHashKeyAndRangeKeyWithHashKeyNameEmpty(){ 177 | optionLoader.checkGsiHashKeyAndRangeKey(null, "gsiRangeKeyName"); 178 | } 179 | 180 | @Test(expected = IllegalArgumentException.class) 181 | public void testCheckGsiHashKeyAndRangeKeyWithSameName(){ 182 | optionLoader.checkGsiHashKeyAndRangeKey("gsiHashKeyName", "gsiHashKeyName"); 183 | } 184 | 185 | @Test(expected = IllegalArgumentException.class) 186 | public void testCheckGsiHashKeyAndRangeKeyWithBothNameEmpty(){ 187 | optionLoader.checkGsiHashKeyAndRangeKey(null, null); 188 | } 189 | 190 | @Test 191 | public void testLoadScanIOPSPercent(){ 192 | String ScanIOPSPercent = "23"; 193 | Mockito.when(mockProperties.getProperty(Options.READ_WRITE_IOPS_PERCENT, Options.READ_WRITE_IOPS_PERCENT_DEFAULT)).thenReturn(ScanIOPSPercent); 194 | Mockito.when(mockOptionChecker.isNumberInRange(Integer.parseInt(ScanIOPSPercent), Options.MIN_READ_WRITE_IOPS_PERCENT, Options.MAX_READ_WRITE_IOPS_PERCENT)).thenReturn(true); 195 | assertEquals("Should return the scan IOPS set", Integer.parseInt(ScanIOPSPercent), optionLoader.loadScanIOPSPercent()); 196 | } 197 | 198 | @Test(expected = IllegalArgumentException.class) 199 | public void testLoadScanIOPSPercentWithInvalidInteger() { 200 | Mockito.when(mockProperties.getProperty(Options.READ_WRITE_IOPS_PERCENT, Options.READ_WRITE_IOPS_PERCENT_DEFAULT)).thenReturn("notaninteger"); 201 | optionLoader.loadScanIOPSPercent(); 202 | } 203 | 204 | @Test(expected = IllegalArgumentException.class) 205 | public void testLoadScanIOPSPercentWithIntegerExceedsRange() { 206 | String invalidScanIOPSPercent = "-23"; 207 | Mockito.when(mockProperties.getProperty(Options.READ_WRITE_IOPS_PERCENT, Options.READ_WRITE_IOPS_PERCENT_DEFAULT)).thenReturn(invalidScanIOPSPercent); 208 | Mockito.when(mockOptionChecker.isNumberInRange(Integer.parseInt(invalidScanIOPSPercent), Options.MIN_READ_WRITE_IOPS_PERCENT, Options.MAX_READ_WRITE_IOPS_PERCENT)).thenReturn(false); 209 | optionLoader.loadScanIOPSPercent(); 210 | } 211 | 212 | @Test 213 | public void testLoadRecordDetails(){ 214 | String recordDeteils = "true"; 215 | Mockito.when(mockProperties.getProperty(Options.RECORD_DETAILS, Options.RECORD_DETAILS_DEFAULT)).thenReturn(recordDeteils); 216 | assertEquals("Should return the boolean set for record details", Boolean.parseBoolean(recordDeteils), optionLoader.loadRecordDetails()); 217 | } 218 | 219 | @Test(expected = IllegalArgumentException.class) 220 | public void testLoadRecordDetailsWithInvalidBoolean() { 221 | Mockito.when(mockProperties.getProperty(Options.RECORD_DETAILS, Options.RECORD_DETAILS_DEFAULT)).thenReturn("invalidboolean"); 222 | optionLoader.loadRecordDetails(); 223 | } 224 | 225 | @Test 226 | public void testLoadOutputPath(){ 227 | String outputPath = "/Document/outputPath"; 228 | Mockito.when(mockProperties.getProperty(Options.DETECTION_OUTPUT_PATH, Options.DETECTION_OUTPUT_PATH_DEFAULT)).thenReturn(outputPath); 229 | assertEquals("Should return the the given output path", outputPath, optionLoader.loadDetectionOutputPath()); 230 | } 231 | 232 | @Test 233 | public void testLoadNumOfSegments(){ 234 | String numOfSegments = "12"; 235 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_SEGMENTS, Options.NUM_OF_SEGMENTS_DEFAULT)).thenReturn(numOfSegments); 236 | Mockito.when(mockOptionChecker.isNumberInRange(Integer.parseInt(numOfSegments), Options.MIN_NUM_OF_SEGMENTS, Options.MAX_NUM_OF_SEGMENTS)).thenReturn(true); 237 | assertEquals("Should return the given num of segments", Integer.parseInt(numOfSegments), optionLoader.loadNumOfSegments()); 238 | } 239 | 240 | @Test(expected = IllegalArgumentException.class) 241 | public void testLoadNumOfSegmentsWithInvalidInteger() { 242 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_SEGMENTS, Options.NUM_OF_SEGMENTS_DEFAULT)).thenReturn("invalidint"); 243 | optionLoader.loadNumOfSegments(); 244 | } 245 | 246 | @Test(expected = IllegalArgumentException.class) 247 | public void testLoadNumOfSegmentsWithNumberExceedsRange() { 248 | String numOfSegments = "-12"; 249 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_SEGMENTS, Options.NUM_OF_SEGMENTS_DEFAULT)).thenReturn(numOfSegments); 250 | Mockito.when(mockOptionChecker.isNumberInRange(Integer.parseInt(numOfSegments), Options.MIN_NUM_OF_SEGMENTS, Options.MAX_NUM_OF_SEGMENTS)).thenReturn(false); 251 | optionLoader.loadNumOfSegments(); 252 | } 253 | 254 | @Test 255 | public void testLoadNumOfViolations(){ 256 | String numOfViolaitons = "1234"; 257 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_VIOLATIONS)).thenReturn(numOfViolaitons); 258 | assertEquals("Should return the given num of violation", Integer.parseInt(numOfViolaitons), optionLoader.loadNumOfViolations()); 259 | } 260 | 261 | @Test(expected = IllegalArgumentException.class) 262 | public void testLoadNumOfViolationsWithInvalidInteger() { 263 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_VIOLATIONS)).thenReturn("invalidInteger"); 264 | optionLoader.loadNumOfViolations(); 265 | } 266 | 267 | @Test(expected = IllegalArgumentException.class) 268 | public void testLoadNumOfViolationsWithValueExceedsRange() { 269 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_VIOLATIONS)).thenReturn("-20"); 270 | optionLoader.loadNumOfViolations(); 271 | } 272 | 273 | @Test 274 | public void testLoadNumOfRecords(){ 275 | String numOfRecords = "1234"; 276 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_RECORDS)).thenReturn(numOfRecords); 277 | assertEquals("Should return the number of records set", Integer.parseInt(numOfRecords), optionLoader.loadNumOfRecords()); 278 | } 279 | 280 | @Test(expected = IllegalArgumentException.class) 281 | public void testLoadNumOfRecordsWithInvalidInteger() { 282 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_RECORDS)).thenReturn("invalidInt"); 283 | optionLoader.loadNumOfRecords(); 284 | } 285 | 286 | @Test(expected = IllegalArgumentException.class) 287 | public void testLoadNumOfRecordsWithValueExceedsRange() { 288 | Mockito.when(mockProperties.getProperty(Options.NUM_OF_RECORDS)).thenReturn("-20"); 289 | optionLoader.loadNumOfRecords(); 290 | } 291 | 292 | @Test 293 | public void testloadCorrectionInputPath(){ 294 | String correctionInputPath = "/Documents/CorrectionInputPath"; 295 | Mockito.when(mockProperties.getProperty(Options.CORRECTION_INPUT_PATH)).thenReturn(correctionInputPath); 296 | assertEquals("Should return the path set for correction input", correctionInputPath, optionLoader.loadCorrectionInputPath()); 297 | } 298 | 299 | @Test(expected = IllegalArgumentException.class) 300 | public void testloadCorrectionInputPathWithPathMissing() { 301 | Mockito.when(mockProperties.getProperty(Options.CORRECTION_INPUT_PATH)).thenReturn(null); 302 | optionLoader.loadCorrectionInputPath(); 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/OptionsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | 19 | import org.junit.Test; 20 | 21 | import com.amazonaws.services.dynamodbv2.online.index.Options; 22 | 23 | /** 24 | * 25 | * Unit tests for Options. 26 | * 27 | */ 28 | public class OptionsTest { 29 | 30 | @Test 31 | public void testSingleInstance() { 32 | assertEquals("Should be the same instnace ", Options.getInstance(), Options.getInstance()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/RandomDataGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.nio.ByteBuffer; 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | import java.util.Random; 21 | 22 | /** 23 | * 24 | * Generator random data. 25 | * 26 | */ 27 | public class RandomDataGenerator { 28 | private static char[] symbols; 29 | private final Random random = new Random(); 30 | 31 | static { 32 | StringBuilder tmp = new StringBuilder(); 33 | for (char ch = '0'; ch < '9'; ++ch) 34 | tmp.append(ch); 35 | for (char ch = 'a'; ch <= 'z'; ++ch) 36 | tmp.append(ch); 37 | symbols = tmp.toString().toCharArray(); 38 | } 39 | 40 | /** 41 | * 42 | * Generate next random string 43 | * 44 | */ 45 | public String nextRadomString(int length) { 46 | char[] buf = new char[length]; 47 | for (int index = 0; index < buf.length; ++index) 48 | buf[index] = symbols[random.nextInt(symbols.length)]; 49 | return new String(buf); 50 | } 51 | 52 | /** 53 | * 54 | * Generate next random string array 55 | * 56 | */ 57 | public List nextRandomStringArray(int stringLength) { 58 | int arrayLength = random.nextInt(10) + 1; 59 | return this.nextRandomStringArray(stringLength, arrayLength); 60 | } 61 | 62 | public List nextRandomStringArray(int stringLength, int arrayLength) { 63 | List array = new ArrayList(); 64 | for (int i = 0; i < arrayLength; i++) { 65 | array.add(this.nextRadomString(stringLength)); 66 | } 67 | return array; 68 | } 69 | 70 | /** 71 | * 72 | * Return random array of integer 73 | * 74 | */ 75 | public List nextRandomIntArray() { 76 | List array = new ArrayList(); 77 | int arrayLength = random.nextInt(10) + 1; 78 | for (int i = 0; i < arrayLength; i++) { 79 | array.add(Integer.toString(this.nextRandomInt())); 80 | } 81 | return array; 82 | } 83 | 84 | /** 85 | * 86 | * Return next random binary array 87 | * 88 | */ 89 | public List nextRandomBinaryArray(int binaryLength) { 90 | List array = new ArrayList(); 91 | int arrayLength = random.nextInt(10) + 1; 92 | for (int i = 0; i < arrayLength; i++) { 93 | array.add(this.nextRandomBinary(binaryLength)); 94 | } 95 | return array; 96 | } 97 | 98 | /** 99 | * 100 | * Generate next binary buffer 101 | * 102 | */ 103 | public ByteBuffer nextRandomBinary(int length) { 104 | char[] buf = new char[length]; 105 | ByteBuffer byteBuffer = ByteBuffer.allocate(length); 106 | for (int index = 0; index < buf.length; ++index) { 107 | byte tmp = (byte) symbols[random.nextInt(symbols.length)]; 108 | byteBuffer.put(tmp); 109 | } 110 | /** Should reset the position */ 111 | byteBuffer.position(0); 112 | return byteBuffer; 113 | } 114 | 115 | /** 116 | * 117 | * Generate next random integer 118 | * 119 | */ 120 | public int nextRandomInt() { 121 | return random.nextInt(); 122 | } 123 | 124 | /** 125 | * 126 | * Generate next random integer within limit 127 | * 128 | */ 129 | public int nextRandomInt(int limit) { 130 | return random.nextInt(limit); 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/TableHelperTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | import static org.junit.Assert.assertFalse; 19 | import static org.junit.Assert.assertTrue; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | import org.mockito.Mockito; 27 | 28 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 29 | import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; 30 | import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndexDescription; 31 | import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; 32 | import com.amazonaws.services.dynamodbv2.model.KeyType; 33 | import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription; 34 | import com.amazonaws.services.dynamodbv2.model.TableDescription; 35 | import com.amazonaws.services.dynamodbv2.online.index.TableHelper; 36 | 37 | /** 38 | * 39 | * Unit tests for table helper. 40 | * 41 | */ 42 | public class TableHelperTest { 43 | private static String hashKeyName = "hashKey"; 44 | private static String hashKeyType = "S"; 45 | private static String rangeKeyName = "rangeKey"; 46 | private static String rangeKeyType = "N"; 47 | private String gsiName = "gsiName"; 48 | private static long readCapacity = 30L; 49 | private static long writeCapacity = 100L; 50 | 51 | /** Mock Objects */ 52 | AmazonDynamoDBClient mockDynamoDBClient = Mockito.mock(AmazonDynamoDBClient.class); 53 | TableDescription mockTableDescription = Mockito.mock(TableDescription.class); 54 | 55 | TableHelper tableHelper = new TableHelper(mockDynamoDBClient, mockTableDescription); 56 | 57 | @Before 58 | public void setupBeforeTest() { 59 | /** Setup key schema */ 60 | List keySchema = new ArrayList(); 61 | KeySchemaElement hashKey = new KeySchemaElement().withAttributeName(hashKeyName).withKeyType(KeyType.HASH); 62 | KeySchemaElement rangeKey = new KeySchemaElement().withAttributeName(rangeKeyName).withKeyType(KeyType.RANGE); 63 | keySchema.add(hashKey); 64 | keySchema.add(rangeKey); 65 | Mockito.when(mockTableDescription.getKeySchema()).thenReturn(keySchema); 66 | 67 | /** Setup attribute definition */ 68 | List attribtueDefinitions = new ArrayList(); 69 | AttributeDefinition hashKeyDefinition = new AttributeDefinition().withAttributeName(hashKeyName).withAttributeType(hashKeyType); 70 | AttributeDefinition rangeKeyDefinition = new AttributeDefinition().withAttributeName(rangeKeyName).withAttributeType(rangeKeyType); 71 | attribtueDefinitions.add(hashKeyDefinition); 72 | attribtueDefinitions.add(rangeKeyDefinition); 73 | Mockito.when(mockTableDescription.getAttributeDefinitions()).thenReturn(attribtueDefinitions); 74 | 75 | /** Setup GSI */ 76 | List globalSecondaryIndexes = new ArrayList(); 77 | GlobalSecondaryIndexDescription gsiTest = new GlobalSecondaryIndexDescription().withIndexName(gsiName); 78 | globalSecondaryIndexes.add(gsiTest); 79 | Mockito.when(mockTableDescription.getGlobalSecondaryIndexes()).thenReturn(globalSecondaryIndexes); 80 | 81 | /** Setup provisioned throughput */ 82 | ProvisionedThroughputDescription provisionedThroughPut = new ProvisionedThroughputDescription().withReadCapacityUnits(readCapacity) 83 | .withWriteCapacityUnits(writeCapacity); 84 | Mockito.when(mockTableDescription.getProvisionedThroughput()).thenReturn(provisionedThroughPut); 85 | } 86 | 87 | @Test 88 | public void testGetTableHashKeyName() { 89 | assertEquals("Should return the hash key name", tableHelper.getTableHashKeyName(), hashKeyName); 90 | } 91 | 92 | @Test 93 | public void testGetTableHashKeyType() { 94 | assertEquals("Should return the hash key type", tableHelper.getTableHashKeyType(), hashKeyType); 95 | } 96 | 97 | @Test 98 | public void testGetRangeKeyName() { 99 | assertEquals("Should return the range key name", tableHelper.getTableRangeKeyName(), rangeKeyName); 100 | } 101 | 102 | @Test 103 | public void testGetRangeKeyType() { 104 | assertEquals("Should return the hash key type", tableHelper.getTableRangeKeyType(), rangeKeyType); 105 | } 106 | 107 | @Test 108 | public void testGsiExists() { 109 | assertTrue(tableHelper.isGsiExists(gsiName)); 110 | assertFalse(tableHelper.isGsiExists("randomName")); 111 | } 112 | 113 | @Test 114 | public void testGetReadCapacityUnits() { 115 | assertEquals("Should return read capacity units", tableHelper.getReadCapacityUnits(), readCapacity); 116 | } 117 | 118 | @Test 119 | public void testGetWriteCapacityUnits() { 120 | assertEquals("Should return write capacity units", tableHelper.getWriteCapacityUnits(), writeCapacity); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/TableReaderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | 20 | import org.junit.Test; 21 | import org.mockito.Mockito; 22 | 23 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 24 | import com.amazonaws.services.dynamodbv2.online.index.Options; 25 | import com.amazonaws.services.dynamodbv2.online.index.TableHelper; 26 | import com.amazonaws.services.dynamodbv2.online.index.TableReader; 27 | import com.amazonaws.services.dynamodbv2.online.index.ViolationChecker; 28 | 29 | /** 30 | * 31 | * Unit tests for tableReader. 32 | * 33 | */ 34 | public class TableReaderTest { 35 | /** Mock objects */ 36 | Options mockOptions = Mockito.mock(Options.class); 37 | AmazonDynamoDBClient mockDynamoDBClient = Mockito.mock(AmazonDynamoDBClient.class); 38 | TableHelper mockTableHelper = Mockito.mock(TableHelper.class); 39 | ViolationChecker mockViolationChecker = Mockito.mock(ViolationChecker.class); 40 | List attributesToGet = new ArrayList(); 41 | double taskRateLimit = 100.0; 42 | 43 | TableReader reader = new TableReader(mockOptions, mockDynamoDBClient, mockTableHelper, mockViolationChecker, 44 | attributesToGet, taskRateLimit); 45 | 46 | @Test 47 | public void testScanEntireTable() { 48 | 49 | } 50 | 51 | @Test 52 | public void testScanGivenNumOfRecords() { 53 | 54 | } 55 | 56 | @Test 57 | public void testScanGivenNumOfViolations() { 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/TableWriterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | import static org.junit.Assert.assertNull; 19 | 20 | import java.util.ArrayList; 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | import org.mockito.Mockito; 28 | 29 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 30 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 31 | import com.amazonaws.services.dynamodbv2.model.BatchWriteItemResult; 32 | import com.amazonaws.services.dynamodbv2.model.WriteRequest; 33 | import com.amazonaws.services.dynamodbv2.online.index.TableWriter; 34 | import com.google.common.util.concurrent.RateLimiter; 35 | 36 | /** 37 | * 38 | * Unit test TableWriter. 39 | * 40 | */ 41 | @SuppressWarnings("unchecked") 42 | public class TableWriterTest { 43 | /** Mock objects */ 44 | private AmazonDynamoDBClient mockDynamoDBClient = Mockito.mock(AmazonDynamoDBClient.class); 45 | private RateLimiter mockRateLimiter = Mockito.mock(RateLimiter.class); 46 | private ArrayList mockBatchDeleteRequests = Mockito.mock(ArrayList.class); 47 | 48 | private final String tableName = "TestTableForDB"; 49 | private final String hashKeyName = "hashKey"; 50 | private final String rangeKeyName = "rangeKey"; 51 | 52 | private RandomDataGenerator randDataGenerator = new RandomDataGenerator(); 53 | private TableWriter tableWriter; 54 | 55 | @Before 56 | public void setup() { 57 | tableWriter = new TableWriter(tableName, hashKeyName, rangeKeyName, mockDynamoDBClient, mockRateLimiter, mockBatchDeleteRequests); 58 | } 59 | 60 | @Test 61 | public void testGenTablePrimaryKeyWithBothHashKeyAndRangeKey() { 62 | String hashKeyValue = randDataGenerator.nextRadomString(10); 63 | String rangekeyValue = String.valueOf(randDataGenerator.nextRandomInt()); 64 | AttributeValue tableHashkey = new AttributeValue().withS(hashKeyValue); 65 | AttributeValue tableRangeKey = new AttributeValue().withN(rangekeyValue); 66 | Map primaryKey = tableWriter.genTablePrimaryKey(tableHashkey, tableRangeKey); 67 | assertEquals("Should return the given table hash key", tableHashkey, primaryKey.get(hashKeyName)); 68 | assertEquals("Should return the given table range key", tableRangeKey, primaryKey.get(rangeKeyName)); 69 | } 70 | 71 | @Test 72 | public void testGenTablePrimaryKeyWithOnlyHashKey() { 73 | String hashKeyValue = randDataGenerator.nextRadomString(10); 74 | AttributeValue tableHashkey = new AttributeValue().withS(hashKeyValue); 75 | Map primaryKey = tableWriter.genTablePrimaryKey(tableHashkey, null); 76 | assertEquals("Should return the given table hash key", tableHashkey, primaryKey.get(hashKeyName)); 77 | assertNull(primaryKey.get(rangeKeyName)); 78 | } 79 | 80 | @Test 81 | public void testAddDeleteRequest() { 82 | String hashKeyValue = randDataGenerator.nextRadomString(10); 83 | String rangekeyValue = String.valueOf(randDataGenerator.nextRandomInt()); 84 | AttributeValue tableHashkey = new AttributeValue().withS(hashKeyValue); 85 | AttributeValue tableRangeKey = new AttributeValue().withN(rangekeyValue); 86 | Mockito.when(mockBatchDeleteRequests.size()).thenReturn(0); 87 | assertEquals("Should be no delete now ", 0, tableWriter.addDeleteRequest(tableHashkey, tableRangeKey)); 88 | } 89 | 90 | @Test 91 | public void testAddDeleteRequestWithItem() { 92 | String hashKeyValue = randDataGenerator.nextRadomString(10); 93 | String rangekeyValue = String.valueOf(randDataGenerator.nextRandomInt()); 94 | AttributeValue tableHashkey = new AttributeValue().withS(hashKeyValue); 95 | AttributeValue tableRangeKey = new AttributeValue().withN(rangekeyValue); 96 | Map mockItem = Mockito.mock(HashMap.class); 97 | Mockito.when(mockItem.get(hashKeyName)).thenReturn(tableHashkey); 98 | Mockito.when(mockItem.get(rangeKeyName)).thenReturn(tableRangeKey); 99 | Mockito.when(mockBatchDeleteRequests.size()).thenReturn(0); 100 | assertEquals("Should be no delete now", 0, tableWriter.addDeleteRequest(mockItem)); 101 | } 102 | 103 | @Test 104 | public void testSendDeleteRequestsWithListEmpty() { 105 | Mockito.when(mockBatchDeleteRequests.isEmpty()).thenReturn(true); 106 | assertEquals("Should not be any delete if empty", 0, tableWriter.sendDeleteRequests()); 107 | } 108 | 109 | @Test 110 | public void testCountAndPrintUndeletedItems() { 111 | BatchWriteItemResult mockBatchWriteResult = Mockito.mock(BatchWriteItemResult.class); 112 | HashMap> mockUnProcessedItems = Mockito.mock(HashMap.class); 113 | Mockito.when(mockBatchWriteResult.getUnprocessedItems()).thenReturn(mockUnProcessedItems); 114 | Mockito.when(mockUnProcessedItems.isEmpty()).thenReturn(true); 115 | tableWriter.countAndPrintUndeletedItems(mockBatchWriteResult); 116 | Mockito.verify(mockUnProcessedItems, Mockito.times(1)).isEmpty(); 117 | Mockito.verify(mockUnProcessedItems, Mockito.times(0)).get(Mockito.anyString()); 118 | } 119 | 120 | @Test 121 | public void testGenAttributeValueForKeyWithS() { 122 | String value = randDataGenerator.nextRadomString(10); 123 | AttributeValue keyValue = new AttributeValue().withS(value); 124 | assertEquals("Should return the right attribute value", keyValue, tableWriter.genAttributeValueForKey("S", value)); 125 | } 126 | 127 | @Test 128 | public void testGenAttributeValueForKeyWithN() { 129 | String value = String.valueOf(randDataGenerator.nextRandomInt()); 130 | AttributeValue keyValue = new AttributeValue().withN(value); 131 | assertEquals("Should return the right attribute value", keyValue, tableWriter.genAttributeValueForKey("N", value)); 132 | } 133 | 134 | @Test(expected = IllegalArgumentException.class) 135 | public void testGenAttribtueValueForKeyWithSS() { 136 | String value = String.valueOf(randDataGenerator.nextRandomInt()); 137 | tableWriter.genAttributeValueForKey("SS", value); 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/ViolationCheckerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertTrue; 18 | 19 | import java.nio.ByteBuffer; 20 | import java.util.List; 21 | 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | import org.mockito.Mockito; 25 | 26 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 27 | import com.amazonaws.services.dynamodbv2.model.KeyType; 28 | import com.amazonaws.services.dynamodbv2.online.index.ViolationChecker; 29 | import com.amazonaws.services.dynamodbv2.online.index.ViolationRecord; 30 | import com.amazonaws.services.dynamodbv2.online.index.ViolationWriter; 31 | 32 | /** 33 | * 34 | * Unit test violation checker. 35 | * 36 | */ 37 | public class ViolationCheckerTest { 38 | /** Mock objects */ 39 | private ViolationRecord mockViolationRecord = Mockito.mock(ViolationRecord.class); 40 | private ViolationWriter mockViolationWriter = Mockito.mock(ViolationWriter.class); 41 | 42 | private ViolationChecker violationChecker; 43 | 44 | private static RandomDataGenerator dataGenerator = new RandomDataGenerator(); 45 | 46 | private static String outputFilePath = "/tmp/testoutput.csv"; 47 | private static String hashKeyName = "hashKey"; 48 | private static String rangeKeyName = "rangeKey"; 49 | private static String attributeName1 = "attribute1"; 50 | private static String attributeName1Type = "S"; 51 | private static String attributeName2 = "attribute2"; 52 | private static String attributeName2Type = "N"; 53 | 54 | @Before 55 | public void setupBeforeTest() { 56 | violationChecker = new ViolationChecker(outputFilePath, hashKeyName, rangeKeyName, attributeName1, attributeName1Type, attributeName2, 57 | attributeName2Type, true, true, mockViolationRecord, mockViolationWriter); 58 | } 59 | 60 | @Test 61 | public void testCheckAttributeViolationWithHashKeySizeViolation() { 62 | String value = dataGenerator.nextRadomString(ViolationChecker.MAX_HASH_KEY_SIZE + 10); 63 | AttributeValue keyValue = new AttributeValue().withS(value); 64 | assertTrue(violationChecker.checkAttributeViolation(keyValue, "S", KeyType.HASH)); 65 | } 66 | 67 | @Test 68 | public void testCheckAttributeViolationWithRangeKeySizeViolation() { 69 | String value = dataGenerator.nextRadomString(ViolationChecker.MAX_RANGE_KEY_SIZE + 10); 70 | AttributeValue keyValue = new AttributeValue().withS(value); 71 | assertTrue(violationChecker.checkAttributeViolation(keyValue, "S", KeyType.RANGE)); 72 | } 73 | 74 | @Test 75 | public void testCheckAttributeViolationWithHashTypeWiolation() { 76 | String value = dataGenerator.nextRadomString(10); 77 | AttributeValue keyValue = new AttributeValue().withS(value); 78 | assertTrue(violationChecker.checkAttributeViolation(keyValue, "N", KeyType.HASH)); 79 | } 80 | 81 | @Test 82 | public void testCheckAttributeViolationWithRangeTypeViolation() { 83 | String value = dataGenerator.nextRadomString(10); 84 | AttributeValue keyValue = new AttributeValue().withS(value); 85 | assertTrue(violationChecker.checkAttributeViolation(keyValue, "N", KeyType.RANGE)); 86 | } 87 | 88 | @Test 89 | public void testCheckAttribtueViolationWithSSTypeViolation() { 90 | List value = dataGenerator.nextRandomStringArray(10, 1); 91 | AttributeValue keyValue = new AttributeValue().withSS(value); 92 | assertTrue(violationChecker.checkAttributeViolation(keyValue, "S", KeyType.RANGE)); 93 | } 94 | 95 | @Test 96 | public void testCheckAttributeViolationWithBSTypeViolation() { 97 | AttributeValue keyValue = new AttributeValue().withBS(dataGenerator.nextRandomBinaryArray(10)); 98 | assertTrue(violationChecker.checkAttributeViolation(keyValue, "B", KeyType.RANGE)); 99 | } 100 | 101 | @Test 102 | public void testCheckAttributViolationWithNSTypeViolation() { 103 | AttributeValue keyValue = new AttributeValue().withNS(dataGenerator.nextRandomIntArray()); 104 | assertTrue(violationChecker.checkAttributeViolation(keyValue, "N", KeyType.RANGE)); 105 | } 106 | 107 | 108 | @Test 109 | public void testRecordItemTableHashKeyWithS() { 110 | String value = dataGenerator.nextRadomString(10); 111 | AttributeValue keyValue = new AttributeValue().withS(value); 112 | violationChecker.recordItemTableHashKey(keyValue); 113 | Mockito.verify(mockViolationRecord).setTableHashKey(value); 114 | } 115 | 116 | @Test 117 | public void testRecordItemTableHashKeyWithN() { 118 | String value = String.valueOf(dataGenerator.nextRandomInt()); 119 | AttributeValue keyValue = new AttributeValue().withN(value); 120 | violationChecker.recordItemTableHashKey(keyValue); 121 | Mockito.verify(mockViolationRecord).setTableHashKey(value); 122 | } 123 | 124 | @Test 125 | public void testRecordItemTableHashKeyWithB() { 126 | ByteBuffer value = dataGenerator.nextRandomBinary(10); 127 | AttributeValue keyValue = new AttributeValue().withB(value); 128 | violationChecker.recordItemTableHashKey(keyValue); 129 | Mockito.verify(mockViolationRecord).setTableHashKey(new String(value.array())); 130 | } 131 | 132 | @Test(expected = IllegalArgumentException.class) 133 | public void testRecordItemTableHashKeyWithSS() { 134 | List value = dataGenerator.nextRandomStringArray(10); 135 | AttributeValue keyValue = new AttributeValue().withSS(value); 136 | violationChecker.recordItemTableHashKey(keyValue); 137 | } 138 | 139 | @Test(expected = IllegalArgumentException.class) 140 | public void testRecordItemTableHashKeyWithNS() { 141 | List value = dataGenerator.nextRandomIntArray(); 142 | AttributeValue keyValue = new AttributeValue().withSS(value); 143 | violationChecker.recordItemTableHashKey(keyValue); 144 | } 145 | 146 | @Test(expected = IllegalArgumentException.class) 147 | public void testRecordItemTableHashKeyWithBS() { 148 | List value = dataGenerator.nextRandomBinaryArray(10); 149 | AttributeValue keyValue = new AttributeValue().withBS(value); 150 | violationChecker.recordItemTableHashKey(keyValue); 151 | } 152 | 153 | @Test 154 | public void testRecordItemTableRangeKeyWithS() { 155 | String value = dataGenerator.nextRadomString(10); 156 | AttributeValue keyValue = new AttributeValue().withS(value); 157 | violationChecker.recordItemTableRangeKey(keyValue); 158 | Mockito.verify(mockViolationRecord).setTableRangeKey(value); 159 | } 160 | 161 | @Test 162 | public void testRecordItemTableRangeKeyWithB() { 163 | ByteBuffer value = dataGenerator.nextRandomBinary(10); 164 | AttributeValue keyValue = new AttributeValue().withB(value); 165 | violationChecker.recordItemTableRangeKey(keyValue); 166 | Mockito.verify(mockViolationRecord).setTableRangeKey(new String(value.array())); 167 | } 168 | 169 | @Test 170 | public void testRecordItemTableRangeKeyWithN() { 171 | int value = dataGenerator.nextRandomInt(); 172 | AttributeValue keyValue = new AttributeValue().withN(String.valueOf(value)); 173 | violationChecker.recordItemTableRangeKey(keyValue); 174 | Mockito.verify(mockViolationRecord).setTableRangeKey(String.valueOf(value)); 175 | } 176 | 177 | @Test(expected = IllegalArgumentException.class) 178 | public void testRecordItemTableRangeKeyWithSS() { 179 | AttributeValue keyValue = new AttributeValue().withSS(dataGenerator.nextRandomStringArray(10)); 180 | violationChecker.recordItemTableRangeKey(keyValue); 181 | } 182 | 183 | @Test(expected = IllegalArgumentException.class) 184 | public void testRecordItemTableRangeKeyWithNS() { 185 | AttributeValue keyValue = new AttributeValue().withNS(dataGenerator.nextRandomIntArray()); 186 | violationChecker.recordItemTableRangeKey(keyValue); 187 | } 188 | 189 | @Test(expected = IllegalArgumentException.class) 190 | public void testRecordItemTableRangeKeyWithBS() { 191 | AttributeValue keyValue = new AttributeValue().withBS(dataGenerator.nextRandomBinaryArray(10)); 192 | violationChecker.recordItemTableRangeKey(keyValue); 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/ViolationRecordTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index; 16 | 17 | import static org.junit.Assert.assertTrue; 18 | 19 | import java.util.List; 20 | 21 | import org.junit.Test; 22 | 23 | import com.amazonaws.services.dynamodbv2.online.index.ViolationRecord; 24 | 25 | /** 26 | * 27 | * Unit test ViolationRecords. 28 | * 29 | */ 30 | public class ViolationRecordTest { 31 | private boolean tableHasRangeKey; 32 | private boolean checkGSIHashkey; 33 | private boolean checkGSIRangeKey; 34 | private RandomDataGenerator randDataGenerator = new RandomDataGenerator(); 35 | 36 | @Test(expected = IllegalArgumentException.class) 37 | public void testInvalidSet() { 38 | tableHasRangeKey = false; 39 | checkGSIHashkey = true; 40 | checkGSIRangeKey = true; 41 | ViolationRecord record = new ViolationRecord(tableHasRangeKey, checkGSIHashkey, checkGSIRangeKey, true); 42 | record.setTableRangeKey(randDataGenerator.nextRadomString(10)); 43 | } 44 | 45 | @Test(expected = IllegalArgumentException.class) 46 | public void testCheckNoGSIKey() { 47 | tableHasRangeKey = true; 48 | checkGSIHashkey = false; 49 | checkGSIRangeKey = false; 50 | new ViolationRecord(tableHasRangeKey, checkGSIHashkey, checkGSIRangeKey, true); 51 | } 52 | 53 | @Test 54 | public void testToList() { 55 | tableHasRangeKey = true; 56 | checkGSIHashkey = true; 57 | checkGSIRangeKey = true; 58 | ViolationRecord record = new ViolationRecord(tableHasRangeKey, checkGSIHashkey, checkGSIRangeKey, true); 59 | List randomStrArray = randDataGenerator.nextRandomStringArray(10, 4); 60 | record.setTableHashKey(randomStrArray.get(0)); 61 | record.setTableRangeKey(randomStrArray.get(1)); 62 | record.setGSIHashKey(randomStrArray.get(2)); 63 | record.setGSIRangeKey(randomStrArray.get(3)); 64 | List returnedArray = record.toStringList(); 65 | for (String s : randomStrArray) { 66 | assertTrue(returnedArray.contains(s)); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/integration/tests/TableManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index.integration.tests; 16 | 17 | import java.util.ArrayList; 18 | import java.util.HashMap; 19 | import java.util.List; 20 | import java.util.Map; 21 | import java.util.Map.Entry; 22 | 23 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; 24 | import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; 25 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 26 | import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; 27 | import com.amazonaws.services.dynamodbv2.model.DescribeTableRequest; 28 | import com.amazonaws.services.dynamodbv2.model.DescribeTableResult; 29 | import com.amazonaws.services.dynamodbv2.model.GetItemRequest; 30 | import com.amazonaws.services.dynamodbv2.model.GetItemResult; 31 | import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; 32 | import com.amazonaws.services.dynamodbv2.model.KeyType; 33 | import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; 34 | import com.amazonaws.services.dynamodbv2.model.PutItemRequest; 35 | import com.amazonaws.services.dynamodbv2.model.ResourceInUseException; 36 | import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; 37 | import com.amazonaws.services.dynamodbv2.model.ScanRequest; 38 | import com.amazonaws.services.dynamodbv2.model.ScanResult; 39 | import com.amazonaws.services.dynamodbv2.model.TableDescription; 40 | import com.amazonaws.services.dynamodbv2.model.TableStatus; 41 | import com.amazonaws.services.dynamodbv2.online.index.PrintHelper; 42 | import com.amazonaws.services.dynamodbv2.online.index.RandomDataGenerator; 43 | 44 | /** 45 | * 46 | * Manage DynamoDB tables for functional tests. 47 | * 48 | */ 49 | public class TableManager { 50 | private AmazonDynamoDBClient client; 51 | RandomDataGenerator randDataGenerator; 52 | PrintHelper printHelper = new PrintHelper(); 53 | 54 | public TableManager(AmazonDynamoDBClient client) { 55 | this.client = client; 56 | this.randDataGenerator = new RandomDataGenerator(); 57 | } 58 | 59 | /** 60 | * Check if table has already existed Using describe table API. If table has 61 | * not been created, exception will be thrown. 62 | */ 63 | public String isTableExist(String tableName) { 64 | if (tableName == null) 65 | throw new IllegalArgumentException("tableName should not be null."); 66 | DescribeTableResult table = new DescribeTableResult(); 67 | 68 | try { 69 | table = client.describeTable(tableName); 70 | return table.getTable().getTableName(); 71 | } catch (ResourceNotFoundException rnfe) { 72 | return null; 73 | } 74 | } 75 | 76 | /** 77 | * 78 | * Create table with given name and schema 79 | * 80 | */ 81 | public boolean createNewTable(String tableName, String hashKeyName, String hashKeyType, String rangeKeyName, String rangeKeyType, long readCapacity, 82 | long writeCapacity) { 83 | /** Attribute definition */ 84 | ArrayList attributeDefinition = new ArrayList(); 85 | attributeDefinition.add(new AttributeDefinition().withAttributeName(hashKeyName).withAttributeType(hashKeyType)); 86 | if (rangeKeyName != null) 87 | attributeDefinition.add(new AttributeDefinition().withAttributeName(rangeKeyName).withAttributeType(rangeKeyType)); 88 | 89 | /** KeySchema */ 90 | ArrayList keySchema = new ArrayList(); 91 | keySchema.add(new KeySchemaElement().withAttributeName(hashKeyName).withKeyType(KeyType.HASH)); 92 | if (rangeKeyName != null) 93 | keySchema.add(new KeySchemaElement().withAttributeName(rangeKeyName).withKeyType(KeyType.RANGE)); 94 | 95 | /** Provisioned throughput */ 96 | ProvisionedThroughput provisionedThroughput = new ProvisionedThroughput().withReadCapacityUnits(readCapacity).withWriteCapacityUnits(writeCapacity); 97 | 98 | /** Create table request */ 99 | CreateTableRequest request = new CreateTableRequest().withTableName(tableName).withAttributeDefinitions(attributeDefinition).withKeySchema(keySchema) 100 | .withProvisionedThroughput(provisionedThroughput); 101 | 102 | /** Try to create table */ 103 | try { 104 | client.createTable(request); 105 | waitForTableToBecomeAvailable(tableName); 106 | System.out.println("Table " + tableName + " created."); 107 | } catch (ResourceInUseException riue) { 108 | System.out.println("Table " + tableName + " already existed."); 109 | return false; 110 | } catch (Exception e) { 111 | System.out.println(e.getMessage()); 112 | return false; 113 | } 114 | return true; 115 | } 116 | 117 | private void waitForTableToBecomeAvailable(String tableName) { 118 | System.out.println("Waiting for " + tableName + " to become ACTIVE..."); 119 | long startTime = System.currentTimeMillis(); 120 | long endTime = startTime + (10 * 60 * 1000); 121 | while (System.currentTimeMillis() < endTime) { 122 | DescribeTableRequest request = new DescribeTableRequest() 123 | .withTableName(tableName); 124 | TableDescription tableDescription = client.describeTable( 125 | request).getTable(); 126 | String tableStatus = tableDescription.getTableStatus(); 127 | System.out.println(" - current state: " + tableStatus); 128 | if (tableStatus.equals(TableStatus.ACTIVE.toString())) 129 | return; 130 | try { Thread.sleep(1000 * 20); } catch (Exception e) { } 131 | } 132 | throw new RuntimeException("Table " + tableName + " never went active"); 133 | } 134 | 135 | /** 136 | * 137 | * Load the specific kind of data into table. 138 | * 139 | */ 140 | public List> loadRandomData(String tableName, String hashKeyName, String hashKeyType, String rangeKeyName, String rangeKeyType, 141 | Map attributePairs, int randomStringLength, int loadItemCount) { 142 | List> list = new ArrayList>(); 143 | for (int i = 0; i < loadItemCount; i++) { 144 | Map item = new HashMap(); 145 | // hash-key 146 | item.put(hashKeyName, generateItem(hashKeyName, hashKeyType, 5)); 147 | // range-key 148 | item.put(rangeKeyName, generateItem(rangeKeyName, rangeKeyType, 5)); 149 | /** Adding random values for specific attribute type */ 150 | for (Entry attribute : attributePairs.entrySet()) { 151 | item.put(attribute.getKey(), generateItem(attribute.getKey(), attribute.getValue(), randomStringLength)); 152 | } 153 | /** Put data into table */ 154 | PutItemRequest putItemRequest = new PutItemRequest().withTableName(tableName).withItem(item); 155 | // printHelper.printItem(1, i + 1, item); 156 | client.putItem(putItemRequest); 157 | list.add(item); 158 | } 159 | return list; 160 | } 161 | 162 | protected AttributeValue generateItem(String attributeName, String attributeType, int randomStringLength) { 163 | /** String */ 164 | if ("S".equals(attributeType)) { 165 | return new AttributeValue().withS(randDataGenerator.nextRadomString(randomStringLength)); 166 | } 167 | /** Number */ 168 | if ("N".equals(attributeType)) { 169 | return new AttributeValue().withN(Integer.toString(randDataGenerator.nextRandomInt())); 170 | } 171 | /** Binary */ 172 | if ("B".equals(attributeType)) { 173 | return new AttributeValue().withB(randDataGenerator.nextRandomBinary(randomStringLength)); 174 | } 175 | /** String Set */ 176 | if ("SS".equals(attributeType)) { 177 | return new AttributeValue().withSS(randDataGenerator.nextRandomStringArray(randomStringLength)); 178 | } 179 | /** Integer set */ 180 | if ("NS".equals(attributeType)) { 181 | return new AttributeValue().withNS(randDataGenerator.nextRandomIntArray()); 182 | } 183 | /** Binary Set */ 184 | if ("BS".equals(attributeType)) { 185 | return new AttributeValue().withBS(randDataGenerator.nextRandomBinaryArray(randomStringLength)); 186 | } 187 | return null; 188 | } 189 | 190 | /** 191 | * 192 | * Get the item from table by their hash key and range key 193 | * 194 | */ 195 | public Map getItem(String tableName, String hashKeyName, AttributeValue hashkeyValue, String rangeKeyName, 196 | AttributeValue rangeKeyValue) { 197 | HashMap key = new HashMap(); 198 | key.put(hashKeyName, hashkeyValue); 199 | if (rangeKeyValue != null) 200 | key.put(rangeKeyName, rangeKeyValue); 201 | 202 | GetItemRequest getItemRequest = new GetItemRequest().withTableName(tableName).withKey(key); 203 | 204 | GetItemResult result = client.getItem(getItemRequest); 205 | return result.getItem(); 206 | } 207 | 208 | /** 209 | * 210 | * Put the item in table 211 | * 212 | */ 213 | public void putItem(String tableName, Map item) { 214 | 215 | PutItemRequest putItemRequest = new PutItemRequest().withTableName(tableName).withItem(item); 216 | client.putItem(putItemRequest); 217 | } 218 | 219 | public void deleteTable(String tableName) { 220 | client.deleteTable(tableName); 221 | } 222 | 223 | public List> getItems(String tableName) { 224 | List> items = new ArrayList>(); 225 | ScanRequest scanRequest = new ScanRequest().withTableName(tableName); 226 | ScanResult scanResult = client.scan(scanRequest); 227 | Map lastEvaluatedKey = null; 228 | do { 229 | scanRequest = new ScanRequest().withTableName(tableName).withExclusiveStartKey(lastEvaluatedKey); 230 | scanResult = client.scan(scanRequest); 231 | items.addAll(scanResult.getItems()); 232 | lastEvaluatedKey = scanResult.getLastEvaluatedKey(); 233 | } while(lastEvaluatedKey != null); 234 | return items; 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /src/test/java/com/amazonaws/services/dynamodbv2/online/index/integration/tests/TestUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazonaws.services.dynamodbv2.online.index.integration.tests; 16 | 17 | import java.io.File; 18 | import java.security.Permission; 19 | 20 | import org.apache.commons.csv.CSVFormat; 21 | 22 | /** 23 | * Utility methods for functional tests 24 | * 25 | */ 26 | public class TestUtils { 27 | 28 | // Set this to false if you do not wish to run tests on DynamoDB Local. 29 | // Make sure that the server is running on port 8000 before kicking off the tests. 30 | public static final boolean RUN_TESTS_ON_DYNAMODB_LOCAL = true; 31 | 32 | public static final String STRING_TYPE = "S"; 33 | public static final String NUMBER_TYPE = "N"; 34 | public static final String BINARY_TYPE = "B"; 35 | public static final String STRING_SET_TYPE = "SS"; 36 | public static final String NUMBER_SET_TYPE = "NS"; 37 | public static final String BINARY_SET_TYPE = "BS"; 38 | 39 | public static final int MAX_HASH_KEY_SIZE = 2048; 40 | public static final int MAX_RANGE_KEY_SIZE = 1024; 41 | 42 | public static final CSVFormat csvFormat = CSVFormat.RFC4180.withHeader().withDelimiter(',').withIgnoreEmptyLines(true); 43 | 44 | public static final int DETECTOR_ITEMS_SCANNED_INDEX = 0; 45 | public static final int DETECTOR_VIOLATIONS_FOUND_INDEX = 1; 46 | public static final int DETECTOR_VIOLATIONS_DELETED_INDEX = 2; 47 | 48 | public static final int CORRECTION_SUCCESSFUL_UPDATES_INDEX = 0; 49 | public static final int CORRECTION_UPDATE_REQUESTS_INDEX = 1; 50 | public static final int CORRECTION_CONDITIONAL_UPDATE_FAILURES_INDEX = 2; 51 | public static final int CORRECTION_UNEXPECTED_ERRORS_INDEX = 3; 52 | 53 | 54 | @SuppressWarnings("serial") 55 | protected static class ExitException extends SecurityException { 56 | public final int status; 57 | public ExitException(int status) { 58 | this.status = status; 59 | } 60 | } 61 | 62 | protected static class NoExitSecurityManager extends SecurityManager { 63 | @Override 64 | public void checkPermission(Permission perm) { 65 | // allow anything. 66 | } 67 | @Override 68 | public void checkPermission(Permission perm, Object context) { 69 | // allow anything. 70 | } 71 | @Override 72 | public void checkExit(int status) { 73 | super.checkExit(status); 74 | throw new ExitException(status); 75 | } 76 | } 77 | 78 | public static String returnDifferentAttributeType(String givenType) { 79 | if(STRING_TYPE.equals(givenType)) { 80 | return NUMBER_TYPE; 81 | } else if (NUMBER_TYPE.equals(givenType)) { 82 | return BINARY_TYPE; 83 | } else if (BINARY_TYPE.equals(givenType)) { 84 | return STRING_TYPE; 85 | } 86 | return null; 87 | } 88 | 89 | public static void deleteFiles(String[] fileNames) { 90 | for(String fileName : fileNames) { 91 | File file = new File(fileName); 92 | if(file.exists()) { 93 | file.delete(); 94 | } 95 | } 96 | } 97 | } 98 | --------------------------------------------------------------------------------