├── .gitignore ├── Demo_Instructions.pdf ├── Demo_Instructions_JP.pdf ├── LICENSE.txt ├── NOTICE.txt ├── NOTICE_JP.txt ├── README.md ├── README_JP.md ├── ingester ├── ImageIngester.properties ├── README.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── amazonaws │ │ └── services │ │ └── dynamodbv2 │ │ └── json │ │ ├── converter │ │ ├── JacksonConverter.java │ │ ├── JacksonConverterException.java │ │ ├── JacksonStreamReader.java │ │ ├── JacksonStreamReaderException.java │ │ ├── impl │ │ │ ├── JacksonConverterImpl.java │ │ │ ├── JacksonStreamReaderImpl.java │ │ │ └── package-info.java │ │ └── package-info.java │ │ └── demo │ │ └── mars │ │ ├── ExitException.java │ │ ├── HelpException.java │ │ ├── ImageIngester.java │ │ ├── ImageIngesterCLI.java │ │ ├── package-info.java │ │ ├── util │ │ ├── ConfigParser.java │ │ ├── DynamoDBManager.java │ │ ├── JSONParser.java │ │ ├── MarsDynamoDBManager.java │ │ ├── NetworkUtils.java │ │ └── package-info.java │ │ └── worker │ │ ├── DynamoDBImageWorker.java │ │ ├── DynamoDBJSONRootWorker.java │ │ ├── DynamoDBMissionWorker.java │ │ ├── DynamoDBSolWorker.java │ │ ├── DynamoDBWorkerUtils.java │ │ └── package-info.java │ └── test │ └── java │ ├── ParserTestFiles │ ├── IMAGE_MANIFEST.json │ ├── IMAGE_MANIFEST_invalid_version.json │ ├── IMAGE_MANIFEST_missing_sol_id.json │ ├── IMAGE_MANIFEST_missing_sol_list.json │ ├── IMAGE_MANIFEST_missing_sol_url.json │ ├── JSON_ROOT.json │ ├── JSON_ROOT_empty_mission_body.json │ ├── SOL_MER.json │ ├── SOL_MER_EXPECTED.json │ ├── SOL_MSL.json │ └── SOL_MSL_EXPECTED.json │ ├── com │ └── amazonaws │ │ └── services │ │ └── dynamodbv2 │ │ └── json │ │ ├── converter │ │ ├── TestJacksonConvertorImpl.java │ │ └── TestJacksonStreamReader.java │ │ └── demo │ │ └── mars │ │ ├── PhotoIngesterCLITest.java │ │ ├── util │ │ ├── ConfigParserTest.java │ │ ├── DynamoDBManagerTest.java │ │ ├── JSONParserTest.java │ │ └── MarsDynamoDBManagerTest.java │ │ └── worker │ │ ├── DynamoDBJSONRootWorkerTest.java │ │ ├── DynamoDBMissionWorkerTest.java │ │ ├── DynamoDBSolWorkerTest.java │ │ ├── DynamoDBWorkerUtilsTest.java │ │ └── WorkerTestUtils.java │ └── res │ ├── flickr.json │ ├── image_manifest_broken.json │ └── image_manifest_converter.json └── viewer ├── Gruntfile.js ├── README.md ├── app ├── .buildignore ├── .htaccess ├── 404.html ├── images │ ├── mars.jpg │ ├── rover.jpg │ └── rover_jumbotron.jpg ├── index.jade ├── robots.txt ├── scripts │ ├── app.js │ ├── controllers │ │ ├── dialog.js │ │ ├── favorites.js │ │ ├── sidemenu.js │ │ ├── timeline.js │ │ └── top-voted.js │ └── services │ │ ├── AWS.js │ │ ├── blueimp.js │ │ └── mars-photos.js ├── styles │ ├── main.scss │ └── timeline.scss └── views │ ├── includes │ └── head.jade │ ├── layout.jade │ └── partials │ ├── dialog.jade │ ├── image-gallery.jade │ ├── rover-detail.jade │ ├── sidemenu.jade │ └── timeline.jade ├── bower.json ├── lib ├── mynconf.coffee ├── prepare_tables.coffee ├── random_votes.coffee └── util.coffee ├── package.json └── test ├── .jshintrc ├── karma.conf.js └── spec ├── controllers ├── favorites.js ├── sidemenu.js ├── timeline.js └── top-voted.js └── views ├── image-gallery.js ├── sidemenu.js └── timeline.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | dist-launcher 4 | .tmp 5 | .dependencies 6 | .sass-cache 7 | bower_components 8 | .DS_Store 9 | target 10 | -------------------------------------------------------------------------------- /Demo_Instructions.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-dynamodb-mars-json-demo/81c2536951bdfea5fbf3852b943d87c0e4a4cb32/Demo_Instructions.pdf -------------------------------------------------------------------------------- /Demo_Instructions_JP.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-dynamodb-mars-json-demo/81c2536951bdfea5fbf3852b943d87c0e4a4cb32/Demo_Instructions_JP.pdf -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Amazon DynamoDB Mars JSON Demo 2 | Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /NOTICE_JP.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-dynamodb-mars-json-demo/81c2536951bdfea5fbf3852b943d87c0e4a4cb32/NOTICE_JP.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Note 2 | This repository has been archived and may be deleted at any time. You have been warned. 3 | 4 | ## Features 5 | Easily store and index the images of Mars published by NASA Jet Propulsion Laboratory. This project includes a Java application for storing the data in DynamoDB or DynamoDB Local and a frontend web application for interacting with and display the images. Provides an example of how to store JSON data in DynamoDB using the low-level Java SDK and query data from DynamoDB using the DynamoDB Document SDK for JavaScript. 6 | 7 | ## Getting Started 8 | To run the demo locally with DynamoDB Local with a small subset of metadata, please run the following commands 9 | ``` 10 | > cd viewer 11 | > npm install 12 | > bower install 13 | > grunt serve 14 | ``` 15 | 16 | ## Minimum Requirements 17 | - Java 1.7+ 18 | - NodeJS 19 | - npm 20 | - bower 21 | - grunt 22 | - coffee-script 23 | - Ruby 24 | - compass 25 | - Maven 26 | 27 | ## Building from Source 28 | ### Image Ingester 29 | You can build the Java application using Maven. Go to the directory `ingester` and run the following command 30 | ``` 31 | > mvn clean install 32 | ``` 33 | 34 | ### Image Viewer 35 | You can build and run the frontend web application with the following commands. 36 | ``` 37 | > npm install 38 | > bower install 39 | > grunt build 40 | ``` 41 | 42 | ## Data Source / Data Ingestion Scheme 43 | 44 | JSON image data is from [http://json.jpl.nasa.gov/data.json http://json.jpl.nasa.gov/data.json]. The image ingester is included and found under directory `photo_ingester`. 45 | 46 | ## Release Notes 47 | -------------------------------------------------------------------------------- /README_JP.md: -------------------------------------------------------------------------------- 1 | ## 機能 2 | NASA ジェット推進研究所が発行した火星の画像を簡単に保存してインデックスを作成することができます。このプロジェクトには、DynamoDB または DynamoDB ローカルにデータを格納するための Java アプリケーションと、画像のやり取りや表示を行うフロントエンド Web アプリケーションが含まれています。低レベルの Java SDK を使用して DynamoDB に JSON データを格納する方法と、DynamoDB ドキュメントの SDK for JavaScript を使用して DynamoDB からデータのクエリを行う方法の例を示します。 3 | 4 | ## 使用開始 5 | メタデータの一部を使用して DynamoDB ローカルのデモをローカルで実行するには、次のコマンドを実行してください 6 | ``` 7 | > cd viewer 8 | > npm install 9 | > bower install 10 | > grunt serve 11 | ``` 12 | 13 | ## 最小要件 14 | - Java 1.7+ 15 | - NodeJS 16 | - npm 17 | - bower 18 | - grunt 19 | - coffee-script 20 | - Ruby 21 | - compass 22 | - Maven 23 | 24 | ## ソースからの作成 25 | ### イメージインジェスター 26 | Maven を使用して Java アプリケーションをビルドすることができます。ディレクトリ 'ingester' に移動して、次のコマンドを実行してください 27 | ``` 28 | > mvn clean install 29 | ``` 30 | 31 | ### イメージビューアー 32 | 次のコマンドを使用して、フロントエンド Web アプリケーションをビルドおよび実行できます。 33 | ``` 34 | > npm install 35 | > bower install 36 | > grunt build 37 | ``` 38 | 39 | ## データソース/データインジェスションスキーマ 40 | 41 | JSON イメージ データは [http://json.jpl.nasa.gov/data.json http://json.jpl.nasa.gov/data.json] からのものです。イメージインジェスターは、ディレクトリ 'photo_ingester' にあります。 42 | 43 | ## リリースノート -------------------------------------------------------------------------------- /ingester/ImageIngester.properties: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Ingester settings 3 | # Image thumbnail (stored in DynamoDB image table) size in pixels 4 | ingester.image.thumbnail.width=300 5 | ingester.image.thumbnail.height=300 6 | # Wait time between checking for completed asynchronous tasks 7 | ingester.waitTime=300 8 | # Timeout for requesting http resources 9 | # 60 seconds 10 | ingester.timeout=60000 11 | # Thread pool sizes 12 | # Number of threads for processing the root and mission manifests 13 | ingester.manifest.threads=1 14 | # Number of threads for processing sols 15 | ingester.sol.threads=4 16 | # Number of threads for processing images 17 | ingester.image.threads=12 18 | # Should the ingester track resources by ETag? 19 | ingester.track-resources=false 20 | # Should the ingester store thumbnail data in the image table? 21 | ingester.store-thumbnails=false 22 | ############################################################################### 23 | # Root of the trimmed JPL manifests 24 | JSON.root=https://s3.amazonaws.com/dynamodb-mars-json/root.json 25 | # Root of the full JPL manifests 26 | #JSON.root=http://json.jpl.nasa.gov/data.json 27 | ############################################################################### 28 | 29 | 30 | 31 | ############################################################################### 32 | # DynamoDB table names and endpoint 33 | dynamodb.resource=marsDemoResources 34 | dynamodb.image=marsDemoImages 35 | dynamodb.endpoint=http://localhost:8000 36 | ############################################################################### 37 | 38 | ############################################################################### 39 | # Parameters for creating DynamoDB tables 40 | # Resource table 41 | dynamodb.resource.create=true 42 | dynamodb.resource.readCapacityUnits=10 43 | dynamodb.resource.writeCapacityUnits=10 44 | # Image table 45 | dynamodb.image.create=true 46 | dynamodb.image.readCapacityUnits=10 47 | dynamodb.image.writeCapacityUnits=10 48 | dynamodb.image.globalSecondaryIndex.time.readCapacityUnits=10 49 | dynamodb.image.globalSecondaryIndex.time.writeCapacityUnits=10 50 | dynamodb.image.globalSecondaryIndex.vote.readCapacityUnits=10 51 | dynamodb.image.globalSecondaryIndex.vote.writeCapacityUnits=10 52 | ############################################################################### 53 | -------------------------------------------------------------------------------- /ingester/README.md: -------------------------------------------------------------------------------- 1 | ## Features 2 | Easily store and index the images of Mars published by NASA Jet Propulsion Laboratory. This project includes a Java application for storing the data in DynamoDB or DynamoDB Local. Provides an example of how to store JSON data in DynamoDB using the low-level Java SDK. 3 | 4 | ## Getting Started 5 | - Option 1: Sign up for Amazon Web Services 6 | - Option 2: Get DynamoDB Local 7 | 8 | ## Minimum Requirements 9 | - Java 1.7+ 10 | - Maven 11 | 12 | ## Building the Image Ingester from Source 13 | You can build the Java application using Maven. Use this command: mvn clean install 14 | ``` 15 | > mvn clean install 16 | ``` 17 | 18 | ## Running the Application 19 | 1. Set up DynamoDB or DynamoDB Local 20 | - Option 1: Using DynamoDB 21 | Get IAM credentials that allow for DynamoDB operations. (See [IAM Introduction](http://docs.aws.amazon.com/IAM/latest/UserGuide/IAM_Introduction.html)) 22 | - Option 2: Using DynamoDB Local 23 | Run the DynamoDB Local server with the following command (See [DynamoDB Local](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html for more options)): 24 | 25 | ``` 26 | java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar 27 | ``` 28 | 29 | 2. Configure Image Ingester 30 | Options for the image ingester are contained in the ImageIngester.properties file found in the root directory. The included file has some default settings that run the demo with DynamoDB Local. 31 | 3. Run the Image Ingester 32 | mvn exec:exec 33 | 34 | ## Release Notes 35 | -------------------------------------------------------------------------------- /ingester/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.amazonaws 5 | aws-dynamodb-mars-json-demo 6 | jar 7 | AWS DynamoDB Mars JSON Demo 8 | 1.0 9 | The AWS DynamoDB Mars JSON Demo provides Java developers 10 | with an example of how to store JSON data in DynamoDB. It crawls the 11 | JSON manifests published by NASA Jet Propulsion Laboratory and stores 12 | image metadata and thumbnails in DynamoDB or DynamoDB Local. 13 | 14 | https://aws.amazon.com/dynamodb 15 | 16 | https://github.com/awslabs/aws-dynamodb-mars-json-demo.git 17 | 18 | 19 | 20 | Amazon Software License 21 | https://aws.amazon.com/asl 22 | repo 23 | 24 | 25 | 26 | 1.2 27 | 1.9.0 28 | 4.13.1 29 | 3.2 30 | 1.5 31 | 32 | 33 | 34 | com.amazonaws 35 | aws-java-sdk 36 | ${aws-java-sdk.version} 37 | 38 | 39 | commons-cli 40 | commons-cli 41 | ${commons-cli.version} 42 | 43 | 44 | junit 45 | junit 46 | ${junit.version} 47 | test 48 | 49 | 50 | org.easymock 51 | easymock 52 | ${easymock.version} 53 | test 54 | 55 | 56 | org.powermock 57 | powermock-module-junit4 58 | ${powermock.version} 59 | test 60 | 61 | 62 | org.powermock 63 | powermock-api-easymock 64 | ${powermock.version} 65 | test 66 | 67 | 68 | 69 | 70 | amazonwebservices 71 | Amazon Web Services 72 | https://aws.amazon.com 73 | 74 | developer 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | org.apache.maven.plugins 83 | maven-compiler-plugin 84 | 85 | 1.7 86 | 1.7 87 | UTF-8 88 | 89 | 90 | 91 | 92 | 93 | 94 | org.apache.maven.plugins 95 | maven-surefire-plugin 96 | 2.17 97 | 98 | 99 | ./ 100 | ./src/test/java/ParserTestFiles/ 101 | ./src/test/java/res/ 102 | 103 | 104 | 105 | 106 | org.codehaus.mojo 107 | exec-maven-plugin 108 | 1.3.2 109 | 110 | 111 | 112 | exec 113 | 114 | 115 | 116 | 117 | java 118 | 119 | -classpath 120 | 121 | com.amazonaws.services.dynamodbv2.json.demo.mars.ImageIngester 122 | -f 123 | ./ImageIngester.properties 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/JacksonConverter.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 7 | import com.fasterxml.jackson.databind.JsonNode; 8 | 9 | /** 10 | * Utility for transforming between Jackson JSON and DynamoDB representations. 11 | */ 12 | public interface JacksonConverter { 13 | 14 | /** 15 | * Converts a list of maps of AttributeValues to a JsonNode instance that represents the list of maps. 16 | * 17 | * @param items 18 | * A list of maps of AttributeValues 19 | * @return A JsonNode instance that represents the converted JSON array. 20 | * @throws JacksonConverterException 21 | * Error converting DynamoDB item to JSON 22 | */ 23 | JsonNode itemListToJsonArray(List> items) throws JacksonConverterException; 24 | 25 | /** 26 | * Converts a JSON array to a list of AttributeValues. 27 | * 28 | * @param array 29 | * A JsonNode instance that represents the target JSON array. 30 | * @return A list of AttributeValues that represents the JSON array. 31 | * @throws JacksonConverterException 32 | * if JsonNode is not an array 33 | */ 34 | List jsonArrayToList(JsonNode array) throws JacksonConverterException; 35 | 36 | /** 37 | * Converts a JSON object to a map of AttributeValues. 38 | * 39 | * @param object 40 | * A JsonNode instance that represents the target JSON object. 41 | * @return A map of AttributeValues that represents the JSON object. 42 | * @throws JacksonConverterException 43 | * if JsonNode is not an object. 44 | */ 45 | Map jsonObjectToMap(JsonNode object) throws JacksonConverterException; 46 | 47 | /** 48 | * Converts a list of AttributeValues to a JsonNode instance that represents the list. 49 | * 50 | * @param list 51 | * A list of AttributeValues 52 | * @return A JsonNode instance that represents the converted JSON array. 53 | * @throws JacksonConverterException 54 | * Error converting DynamoDB item to JSON 55 | */ 56 | JsonNode listToJsonArray(List list) throws JacksonConverterException; 57 | 58 | /** 59 | * Converts a map of AttributeValues to a JsonNode instance that represents the map. 60 | * 61 | * @param map 62 | * A map of AttributeValues 63 | * @return A JsonNode instance that represents the converted JSON object. 64 | * @throws JacksonConverterException 65 | * Error converting DynamoDB item to JSON 66 | */ 67 | JsonNode mapToJsonObject(Map map) throws JacksonConverterException; 68 | } 69 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/JacksonConverterException.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter; 2 | 3 | /** 4 | * Exception occurred while transforming between representations. 5 | */ 6 | public class JacksonConverterException extends Exception { 7 | 8 | /** 9 | * Serial Version. 10 | */ 11 | private static final long serialVersionUID = 8457313895307710705L; 12 | 13 | /** 14 | * Constructs a new {@link JacksonConverterException} with the provided message. 15 | * 16 | * @param message 17 | * Error message detailing exception 18 | */ 19 | public JacksonConverterException(final String message) { 20 | super(message); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/JacksonStreamReader.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | 6 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 7 | 8 | /** 9 | * Utility for transforming between Jackson JSON streaming representation and DynamoDB format. 10 | */ 11 | public interface JacksonStreamReader { 12 | 13 | /** 14 | * Looks for the beginning of the next JSON object in the JsonParser and generates a map of AttributeValues that 15 | * represents the JSON object. If the internal JsonParser's current token is the beginning of a JSON array, it 16 | * advances to the next token and tries to find an object in the array. 17 | * 18 | * @return A map of AttributeValues that represents the JSON object. Null if the end of stream is reached. 19 | * @throws IOException 20 | * Error reading stream 21 | */ 22 | Map getNextItem() throws IOException; 23 | 24 | /** 25 | * Seeks for a field specified with the argument and advances the JsonParser to the value of the field. 26 | * 27 | * @param fieldName 28 | * The key of the field to seek 29 | * @return True if a field with the specified name is found. False, otherwise. 30 | * @throws IOException 31 | * Error reading stream 32 | */ 33 | boolean seek(String fieldName) throws IOException; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/JacksonStreamReaderException.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter; 2 | 3 | import com.fasterxml.jackson.core.JsonLocation; 4 | import com.fasterxml.jackson.core.JsonParseException; 5 | 6 | /** 7 | * Exception occurred while transforming between representations. 8 | */ 9 | public class JacksonStreamReaderException extends JsonParseException { 10 | 11 | /** 12 | * Serial Version. 13 | */ 14 | private static final long serialVersionUID = -210292231601427891L; 15 | 16 | /** 17 | * Constructs a {@link JacksonStreamReaderException} with the provided message at the specified location. 18 | * 19 | * @param message 20 | * Error message 21 | * @param location 22 | * Location of exception 23 | */ 24 | public JacksonStreamReaderException(final String message, final JsonLocation location) { 25 | super(message, location); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/impl/JacksonConverterImpl.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter.impl; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.Iterator; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.Map.Entry; 9 | 10 | import com.amazonaws.services.dynamodbv2.json.converter.JacksonConverter; 11 | import com.amazonaws.services.dynamodbv2.json.converter.JacksonConverterException; 12 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 13 | import com.fasterxml.jackson.databind.JsonNode; 14 | import com.fasterxml.jackson.databind.node.ArrayNode; 15 | import com.fasterxml.jackson.databind.node.JsonNodeFactory; 16 | import com.fasterxml.jackson.databind.node.ObjectNode; 17 | 18 | /** 19 | * Implementation of the {@link JacksonConverter}. 20 | */ 21 | public class JacksonConverterImpl implements JacksonConverter { 22 | /** 23 | * Maximum JSON depth. 24 | */ 25 | private static final int MAX_DEPTH = 50; 26 | 27 | /** 28 | * Constructs a {@link JacksonConverterImpl}. 29 | */ 30 | public JacksonConverterImpl() { 31 | } 32 | 33 | /** 34 | * Asserts the depth is not greater than {@link #MAX_DEPTH}. 35 | * 36 | * @param depth 37 | * Current JSON depth 38 | * @throws JacksonConverterException 39 | * Depth is greater than {@link #MAX_DEPTH} 40 | */ 41 | private void assertDepth(final int depth) throws JacksonConverterException { 42 | if (depth > MAX_DEPTH) { 43 | throw new JacksonConverterException("Max depth reached. The object/array has too much depth."); 44 | } 45 | } 46 | 47 | /** 48 | * Gets an DynamoDB representation of a JsonNode. 49 | * 50 | * @param node 51 | * The JSON to convert 52 | * @param depth 53 | * Current JSON depth 54 | * @return DynamoDB representation of the JsonNode 55 | * @throws JacksonConverterException 56 | * Unknown JsonNode type or JSON is too deep 57 | */ 58 | private AttributeValue getAttributeValue(final JsonNode node, final int depth) throws JacksonConverterException { 59 | assertDepth(depth); 60 | switch (node.asToken()) { 61 | case VALUE_STRING: 62 | return new AttributeValue().withS(node.textValue()); 63 | case VALUE_NUMBER_INT: 64 | case VALUE_NUMBER_FLOAT: 65 | return new AttributeValue().withN(node.numberValue().toString()); 66 | case VALUE_TRUE: 67 | case VALUE_FALSE: 68 | return new AttributeValue().withBOOL(node.booleanValue()); 69 | case VALUE_NULL: 70 | return new AttributeValue().withNULL(true); 71 | case START_OBJECT: 72 | return new AttributeValue().withM(jsonObjectToMap(node, depth)); 73 | case START_ARRAY: 74 | return new AttributeValue().withL(jsonArrayToList(node, depth)); 75 | default: 76 | throw new JacksonConverterException("Unknown node type: " + node); 77 | } 78 | } 79 | 80 | /** 81 | * Converts a DynamoDB attribute to a JSON representation. 82 | * 83 | * @param av 84 | * DynamoDB attribute 85 | * @param depth 86 | * Current JSON depth 87 | * @return JSON representation of the DynamoDB attribute 88 | * @throws JacksonConverterException 89 | * Unknown DynamoDB type or JSON is too deep 90 | */ 91 | private JsonNode getJsonNode(final AttributeValue av, final int depth) throws JacksonConverterException { 92 | assertDepth(depth); 93 | if (av.getS() != null) { 94 | return JsonNodeFactory.instance.textNode(av.getS()); 95 | } else if (av.getN() != null) { 96 | try { 97 | return JsonNodeFactory.instance.numberNode(Integer.parseInt(av.getN())); 98 | } catch (final NumberFormatException e) { 99 | // Not an integer 100 | try { 101 | return JsonNodeFactory.instance.numberNode(Float.parseFloat(av.getN())); 102 | } catch (final NumberFormatException e2) { 103 | // Not a number 104 | throw new JacksonConverterException(e.getMessage()); 105 | } 106 | } 107 | } else if (av.getBOOL() != null) { 108 | return JsonNodeFactory.instance.booleanNode(av.getBOOL()); 109 | } else if (av.getNULL() != null) { 110 | return JsonNodeFactory.instance.nullNode(); 111 | } else if (av.getL() != null) { 112 | return listToJsonArray(av.getL(), depth); 113 | } else if (av.getM() != null) { 114 | return mapToJsonObject(av.getM(), depth); 115 | } else { 116 | throw new JacksonConverterException("Unknown type value " + av); 117 | } 118 | } 119 | 120 | /** 121 | * {@inheritDoc} 122 | */ 123 | @Override 124 | public JsonNode itemListToJsonArray(final List> items) throws JacksonConverterException { 125 | if (items != null) { 126 | final ArrayNode array = JsonNodeFactory.instance.arrayNode(); 127 | for (final Map item : items) { 128 | array.add(mapToJsonObject(item, 0)); 129 | } 130 | return array; 131 | } 132 | throw new JacksonConverterException("Items cannnot be null"); 133 | } 134 | 135 | /** 136 | * {@inheritDoc} 137 | */ 138 | @Override 139 | public List jsonArrayToList(final JsonNode node) throws JacksonConverterException { 140 | return jsonArrayToList(node, 0); 141 | } 142 | 143 | /** 144 | * Helper method to convert a JsonArrayNode to a DynamoDB list. 145 | * 146 | * @param node 147 | * Array node to convert 148 | * @param depth 149 | * Current JSON depth 150 | * @return DynamoDB list representation of the array node 151 | * @throws JacksonConverterException 152 | * JsonNode is not an array or depth is too great 153 | */ 154 | private List jsonArrayToList(final JsonNode node, final int depth) throws JacksonConverterException { 155 | assertDepth(depth); 156 | if (node != null && node.isArray()) { 157 | final List result = new ArrayList(); 158 | final Iterator children = node.elements(); 159 | while (children.hasNext()) { 160 | final JsonNode child = children.next(); 161 | result.add(getAttributeValue(child, depth)); 162 | } 163 | return result; 164 | } 165 | throw new JacksonConverterException("Expected JSON array, but received " + node); 166 | } 167 | 168 | /** 169 | * {@inheritDoc} 170 | */ 171 | @Override 172 | public Map jsonObjectToMap(final JsonNode node) throws JacksonConverterException { 173 | return jsonObjectToMap(node, 0); 174 | } 175 | 176 | /** 177 | * Transforms a JSON object to a DynamoDB object. 178 | * 179 | * @param node 180 | * JSON object 181 | * @param depth 182 | * Current JSON depth 183 | * @return DynamoDB object representation of JSON 184 | * @throws JacksonConverterException 185 | * JSON is not an object or depth is too great 186 | */ 187 | private Map jsonObjectToMap(final JsonNode node, final int depth) 188 | throws JacksonConverterException { 189 | assertDepth(depth); 190 | if (node != null && node.isObject()) { 191 | final Map result = new HashMap(); 192 | final Iterator keys = node.fieldNames(); 193 | while (keys.hasNext()) { 194 | final String key = keys.next(); 195 | result.put(key, getAttributeValue(node.get(key), depth + 1)); 196 | } 197 | return result; 198 | } 199 | throw new JacksonConverterException("Expected JSON Object, but received " + node); 200 | } 201 | 202 | /** 203 | * {@inheritDoc} 204 | */ 205 | @Override 206 | public JsonNode listToJsonArray(final List item) throws JacksonConverterException { 207 | return listToJsonArray(item, 0); 208 | } 209 | 210 | /** 211 | * Converts a DynamoDB list to a JSON list. 212 | * 213 | * @param item 214 | * DynamoDB list 215 | * @param depth 216 | * Current JSON depth 217 | * @return JSON array node representation of DynamoDB list 218 | * @throws JacksonConverterException 219 | * Null DynamoDB list or JSON too deep 220 | */ 221 | private JsonNode listToJsonArray(final List item, final int depth) throws JacksonConverterException { 222 | assertDepth(depth); 223 | if (item != null) { 224 | final ArrayNode node = JsonNodeFactory.instance.arrayNode(); 225 | for (final AttributeValue value : item) { 226 | node.add(getJsonNode(value, depth + 1)); 227 | } 228 | return node; 229 | } 230 | throw new JacksonConverterException("Item cannot be null"); 231 | } 232 | 233 | /** 234 | * {@inheritDoc} 235 | */ 236 | @Override 237 | public JsonNode mapToJsonObject(final Map item) throws JacksonConverterException { 238 | return mapToJsonObject(item, 0); 239 | } 240 | 241 | /** 242 | * Converts a DynamoDB object to a JSON map. 243 | * 244 | * @param item 245 | * DynamoDB object 246 | * @param depth 247 | * Current JSON depth 248 | * @return JSON map representation of the DynamoDB object 249 | * @throws JacksonConverterException 250 | * Null DynamoDB object or JSON too deep 251 | */ 252 | private JsonNode mapToJsonObject(final Map item, final int depth) 253 | throws JacksonConverterException { 254 | assertDepth(depth); 255 | if (item != null) { 256 | final ObjectNode node = JsonNodeFactory.instance.objectNode(); 257 | 258 | for (final Entry entry : item.entrySet()) { 259 | node.put(entry.getKey(), getJsonNode(entry.getValue(), depth + 1)); 260 | } 261 | return node; 262 | } 263 | throw new JacksonConverterException("Item cannot be null"); 264 | } 265 | 266 | } 267 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/impl/JacksonStreamReaderImpl.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter.impl; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import com.amazonaws.services.dynamodbv2.json.converter.JacksonStreamReader; 10 | import com.amazonaws.services.dynamodbv2.json.converter.JacksonStreamReaderException; 11 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 12 | import com.fasterxml.jackson.core.JsonLocation; 13 | import com.fasterxml.jackson.core.JsonParser; 14 | import com.fasterxml.jackson.core.JsonToken; 15 | 16 | /** 17 | * Implementation of JacksonStreamReader transformer. 18 | */ 19 | public class JacksonStreamReaderImpl implements JacksonStreamReader { 20 | /** 21 | * JsonParser for getting tokens. 22 | */ 23 | private final JsonParser jp; 24 | 25 | /** 26 | * Constructs a {@link JacksonStreamReaderImpl} with the provided {@link JsonParser}. 27 | * 28 | * @param jp 29 | * JsonParser from which to get tokens 30 | * @throws IOException 31 | * Null JsonParser or error getting token 32 | */ 33 | public JacksonStreamReaderImpl(final JsonParser jp) throws IOException { 34 | if (jp == null) { 35 | throw new JacksonStreamReaderException("JsonParser cannot be null", JsonLocation.NA); 36 | } 37 | this.jp = jp; 38 | if (jp.getCurrentToken() == null) { 39 | jp.nextToken(); 40 | } 41 | } 42 | 43 | /** 44 | * {@inheritDoc} 45 | */ 46 | @Override 47 | public Map getNextItem() throws IOException { 48 | if (isEndReached()) { 49 | return null; 50 | } 51 | 52 | if (isObject()) { 53 | return getNextMap(); 54 | } else if (isArray() || isEndOfArray() || isEndOfObject()) { 55 | jp.nextToken(); 56 | return getNextItem(); 57 | } else { 58 | throw new JacksonStreamReaderException("The start of next item needs to be an object, but was " 59 | + jp.getCurrentToken(), jp.getCurrentLocation()); 60 | } 61 | } 62 | 63 | /** 64 | * Gets the next list from the JsonParser in a DynamoDB representation. 65 | * 66 | * @return DynamoDB representation of the next list from the JsonParser 67 | * @throws IOException 68 | * Error getting token or unknown value type. 69 | */ 70 | private List getNextList() throws IOException { 71 | final List list = new ArrayList(); 72 | while (JsonToken.END_ARRAY != jp.nextToken()) { 73 | switch (jp.getCurrentToken()) { 74 | case VALUE_STRING: 75 | list.add(new AttributeValue().withS(jp.getText())); 76 | break; 77 | case VALUE_NUMBER_INT: 78 | case VALUE_NUMBER_FLOAT: 79 | list.add(new AttributeValue().withN(jp.getValueAsString())); 80 | break; 81 | case VALUE_TRUE: 82 | case VALUE_FALSE: 83 | list.add(new AttributeValue().withBOOL(jp.getBooleanValue())); 84 | break; 85 | case VALUE_NULL: 86 | list.add(new AttributeValue().withNULL(true)); 87 | break; 88 | case START_OBJECT: 89 | list.add(new AttributeValue().withM(getNextMap())); 90 | break; 91 | case START_ARRAY: 92 | list.add(new AttributeValue().withL(getNextList())); 93 | break; 94 | default: 95 | throw new JacksonStreamReaderException("Unknown value type: " + jp.getCurrentToken().name(), 96 | jp.getCurrentLocation()); 97 | } 98 | } 99 | return list; 100 | } 101 | 102 | /** 103 | * Gets the next map from the JsonParser in a DynamoDB representation. 104 | * 105 | * @return DynamoDB representation of the next map from the JsonParser 106 | * @throws IOException 107 | * Error getting token or unknown value type 108 | */ 109 | private Map getNextMap() throws IOException { 110 | final Map map = new HashMap(); 111 | while (JsonToken.END_OBJECT != jp.nextToken()) { 112 | switch (jp.getCurrentToken()) { 113 | case FIELD_NAME: 114 | continue; 115 | case VALUE_STRING: 116 | map.put(jp.getCurrentName(), new AttributeValue().withS(jp.getText())); 117 | break; 118 | case VALUE_NUMBER_INT: 119 | case VALUE_NUMBER_FLOAT: 120 | map.put(jp.getCurrentName(), new AttributeValue().withN(jp.getValueAsString())); 121 | break; 122 | case VALUE_TRUE: 123 | case VALUE_FALSE: 124 | map.put(jp.getCurrentName(), new AttributeValue().withBOOL(jp.getBooleanValue())); 125 | break; 126 | case VALUE_NULL: 127 | map.put(jp.getCurrentName(), new AttributeValue().withNULL(true)); 128 | break; 129 | case START_OBJECT: 130 | map.put(jp.getCurrentName(), new AttributeValue().withM(getNextMap())); 131 | break; 132 | case START_ARRAY: 133 | map.put(jp.getCurrentName(), new AttributeValue().withL(getNextList())); 134 | break; 135 | default: 136 | throw new JacksonStreamReaderException("Unknown value type: " + jp.getCurrentToken().name(), 137 | jp.getCurrentLocation()); 138 | } 139 | } 140 | return map; 141 | } 142 | 143 | /** 144 | * Checks if current token is the start of an array. 145 | * 146 | * @return True if current token is the start of an array. 147 | */ 148 | private boolean isArray() { 149 | return jp.getCurrentToken() == JsonToken.START_ARRAY; 150 | } 151 | 152 | /** 153 | * Checks if the current token is the end of an array. 154 | * 155 | * @return True if the current token is the end of an array 156 | */ 157 | private boolean isEndOfArray() { 158 | return jp.getCurrentToken() == JsonToken.END_ARRAY; 159 | } 160 | 161 | /** 162 | * Checks if the current token is the end of an object. 163 | * 164 | * @return True if the current token is the end of an object 165 | */ 166 | private boolean isEndOfObject() { 167 | return jp.getCurrentToken() == JsonToken.END_OBJECT; 168 | } 169 | 170 | /** 171 | * Checks if the current token is null. 172 | * 173 | * @return True if the current token is null 174 | */ 175 | private boolean isEndReached() { 176 | return jp.getCurrentToken() == null; 177 | } 178 | 179 | /** 180 | * Checks if the current token is the start of an object. 181 | * 182 | * @return True if the current token is the start of an object 183 | */ 184 | private boolean isObject() { 185 | return jp.getCurrentToken() == JsonToken.START_OBJECT; 186 | } 187 | 188 | /** 189 | * {@inheritDoc} 190 | */ 191 | @Override 192 | public boolean seek(final String fieldName) throws IOException { 193 | if (fieldName == null) { 194 | return false; 195 | } 196 | do { 197 | if (fieldName.equals(jp.getCurrentName())) { 198 | jp.nextValue(); 199 | return true; 200 | } 201 | jp.nextToken(); 202 | } while (!isEndReached()); 203 | return false; 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/impl/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Implementations of the Jackson JSON - DynamoDB item transformers. 3 | */ 4 | package com.amazonaws.services.dynamodbv2.json.converter.impl; 5 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/converter/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Defines interfaces for transformers that convert between Jackson JSON data and DynamoDB items. 3 | */ 4 | package com.amazonaws.services.dynamodbv2.json.converter; 5 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/ExitException.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars; 2 | 3 | /** 4 | * A fatal exception for the {@link ImageIngester}. 5 | */ 6 | public class ExitException extends Exception { 7 | /** 8 | * Serial Version. 9 | */ 10 | private static final long serialVersionUID = 67891134344302080L; 11 | 12 | /** 13 | * Constructs an {@link ExitException} with the supplied error message. 14 | * 15 | * @param message 16 | * The error message for the exception. 17 | */ 18 | public ExitException(final String message) { 19 | super(message); 20 | } 21 | 22 | /** 23 | * Constructs an {@link ExitException} with the supplied error message and cause {@link Throwable}. 24 | * 25 | * @param message 26 | * The error message for the exception 27 | * @param cause 28 | * The {@link Throwable} that caused the exception 29 | */ 30 | public ExitException(final String message, final Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/HelpException.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars; 2 | 3 | /** 4 | * Exception for quitting when help is displayed. 5 | */ 6 | public class HelpException extends RuntimeException { 7 | 8 | /** 9 | * Serial Version. 10 | */ 11 | private static final long serialVersionUID = -7766042082065548522L; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/ImageIngesterCLI.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.IOException; 6 | import java.io.InputStreamReader; 7 | import java.nio.charset.Charset; 8 | import java.util.Properties; 9 | 10 | import org.apache.commons.cli.BasicParser; 11 | import org.apache.commons.cli.CommandLine; 12 | import org.apache.commons.cli.HelpFormatter; 13 | import org.apache.commons.cli.Option; 14 | import org.apache.commons.cli.Options; 15 | import org.apache.commons.cli.ParseException; 16 | 17 | /** 18 | * Processes command line arguments and loads properties file containing configuration. 19 | *

20 | * The configuration file can be loaded using one of the following methods: 21 | *

    22 | *
  • Default location: {@value #DEFAULT_CONFIG_FILE_LOCATION}
    • 23 | *
  • Filepath: specify using CLI option {@value #OPTION_FILE_SHORT}
    • 24 | *
  • Classpath: specify using CLI option {@value #OPTION_CLASSPATH_SHORT}
    • 25 | *
26 | */ 27 | public class ImageIngesterCLI { 28 | /** 29 | * Classpath separator for the OS. 30 | */ 31 | public static final String CLASSPATH_SEPARATOR = System.getProperty("path.separator"); 32 | /** 33 | * Classpath. 34 | */ 35 | public static final String CLASSPATH = System.getProperty("java.class.path"); 36 | /** 37 | * File encoding. 38 | */ 39 | public static final String FILE_ENCODING = "UTF-8"; 40 | 41 | // Constants for CLI 42 | /** 43 | * Application name for displaying help. 44 | */ 45 | private static final String APP_NAME = "DynamoDB Mars Image Ingester"; 46 | /** 47 | * Options for CLI. 48 | */ 49 | private static final Options OPTIONS; 50 | /** 51 | * Help option description. 52 | */ 53 | private static final String OPTION_HELP_DESC = "Shows help"; 54 | /** 55 | * Help short option. 56 | */ 57 | private static final String OPTION_HELP_SHORT = "h"; 58 | /** 59 | * Help long option. 60 | */ 61 | private static final String OPTION_HELP_LONG = "help"; 62 | /** 63 | * Help has no argument. 64 | */ 65 | private static final boolean OPTION_HELP_HASARG = false; 66 | /** 67 | * Help is not a required flag. 68 | */ 69 | private static final boolean OPTION_HELP_REQ = false; 70 | /** 71 | * Configuration filepath description. 72 | */ 73 | private static final String OPTION_FILE_DESC = "Load configuration file from filepath"; 74 | /** 75 | * Configuration filepath short option. 76 | */ 77 | private static final String OPTION_FILE_SHORT = "f"; 78 | /** 79 | * Configuration filepath long option. 80 | */ 81 | private static final String OPTION_FILE_LONG = "filepath-configuration"; 82 | /** 83 | * Configuration filepath has an argument. 84 | */ 85 | private static final boolean OPTION_FILE_HASARG = true; 86 | /** 87 | * Configuration file is not a required option. 88 | */ 89 | private static final boolean OPTION_FILE_REQ = false; 90 | /** 91 | * Configuration file name in classpath description. 92 | */ 93 | private static final String OPTION_CLASSPATH_DESC = "Load configuration file by name from classpath"; 94 | /** 95 | * Configuration file name in classpath short option. 96 | */ 97 | private static final String OPTION_CLASSPATH_SHORT = "n"; 98 | /** 99 | * Configuration file name in classpath long option. 100 | */ 101 | private static final String OPTION_CLASSPATH_LONG = "name-from-classpath"; 102 | /** 103 | * Configuration file name in classpath has an argument. 104 | */ 105 | private static final boolean OPTION_CLASSPATH_HASARG = true; 106 | /** 107 | * Configuration file name in classpath is not a required option. 108 | */ 109 | private static final boolean OPTION_CLASSPATH_REQ = false; 110 | /** 111 | * Default configuration file name. 112 | */ 113 | public static final String DEFAULT_CONFIG_FILE_NAME = "ImageIngester.properties"; 114 | 115 | static { 116 | OPTIONS = new Options(); 117 | final Option helpOpt = new Option(OPTION_HELP_SHORT, OPTION_HELP_LONG, OPTION_HELP_HASARG, OPTION_HELP_DESC); 118 | helpOpt.setRequired(OPTION_HELP_REQ); 119 | OPTIONS.addOption(helpOpt); 120 | final Option configOpt = new Option(OPTION_FILE_SHORT, OPTION_FILE_LONG, OPTION_FILE_HASARG, OPTION_FILE_DESC); 121 | configOpt.setRequired(OPTION_FILE_REQ); 122 | OPTIONS.addOption(configOpt); 123 | final Option classpathOpt = new Option(OPTION_CLASSPATH_SHORT, OPTION_CLASSPATH_LONG, OPTION_CLASSPATH_HASARG, 124 | OPTION_CLASSPATH_DESC); 125 | classpathOpt.setRequired(OPTION_CLASSPATH_REQ); 126 | OPTIONS.addOption(classpathOpt); 127 | } 128 | 129 | // State 130 | /** 131 | * CLI arguments. 132 | */ 133 | private final String[] args; 134 | /** 135 | * Loaded configuration. 136 | */ 137 | private Properties config = null; 138 | 139 | /** 140 | * Constructs a {@link ImageIngesterCLI} for parsing the supplied CLI arguments. 141 | * 142 | * @param args 143 | * CLI arguments 144 | */ 145 | public ImageIngesterCLI(final String[] args) { 146 | this.args = args.clone(); 147 | } 148 | 149 | /** 150 | * Called by main program to get the configuration. 151 | * 152 | * @return Properties 153 | * @throws ExitException 154 | * Help option specified, syntax error, or error loading configuration file 155 | */ 156 | public Properties getConfig() throws ExitException { 157 | parse(); 158 | return config; 159 | } 160 | 161 | /** 162 | * Prints help message when help option {@value #OPTION_HELP_SHORT} or {@value #OPTION_HELP_LONG} is specified or 163 | * illegal syntax is used. 164 | */ 165 | private void help() { 166 | final HelpFormatter helpFormatter = new HelpFormatter(); 167 | helpFormatter.printHelp(APP_NAME, OPTIONS); 168 | throw new HelpException(); 169 | } 170 | 171 | /** 172 | * Loads the configuration file by name from the classpath. 173 | * 174 | * @param file 175 | * The name of the file 176 | * @return Configuration properties 177 | * @throws ExitException 178 | * Error loading configuration file from classpath 179 | */ 180 | private Properties loadConfigFromClasspath(final String file) throws ExitException { 181 | final Properties p = new Properties(); 182 | InputStreamReader reader = null; 183 | try { 184 | reader = new InputStreamReader(getClass().getClassLoader().getResourceAsStream(file), 185 | Charset.forName(FILE_ENCODING)); 186 | p.load(reader); 187 | } catch (final IOException | NullPointerException e) { 188 | throw new ExitException("Could not load configuration file from classpath. File= " + file + ", Classpath=" 189 | + CLASSPATH, e); 190 | } finally { 191 | if (reader != null) { 192 | try { 193 | reader.close(); 194 | } catch (final IOException e) { 195 | throw new ExitException("Could not close configuration classpath"); 196 | } 197 | } 198 | } 199 | return p; 200 | } 201 | 202 | /** 203 | * Loads the configuration file from a filepath. 204 | * 205 | * @param filepath 206 | * The path to the file 207 | * @return Configuration properties 208 | * @throws ExitException 209 | * Error loading configuration file from filepath 210 | */ 211 | private Properties loadConfigFromFilepath(final String filepath) throws ExitException { 212 | final Properties p = new Properties(); 213 | InputStreamReader reader = null; 214 | try { 215 | reader = new InputStreamReader(new FileInputStream(new File(filepath)), Charset.forName(FILE_ENCODING)); 216 | p.load(reader); 217 | } catch (final IOException e) { 218 | throw new ExitException("Could not load configuration file from filepath", e); 219 | } finally { 220 | if (reader != null) { 221 | try { 222 | reader.close(); 223 | } catch (final IOException e) { 224 | throw new ExitException("Could not close configuration file"); 225 | } 226 | } 227 | } 228 | return p; 229 | } 230 | 231 | /** 232 | * Parses command line arguments to load the configuration file. 233 | * 234 | * @throws ExitException 235 | * Help option specified, syntax error, or error loading configuration file 236 | */ 237 | private void parse() throws ExitException { 238 | final BasicParser parser = new BasicParser(); 239 | CommandLine cl = null; 240 | try { 241 | cl = parser.parse(OPTIONS, args); 242 | } catch (final ParseException e) { 243 | help(); 244 | 245 | } 246 | if (cl.hasOption(OPTION_HELP_SHORT)) { 247 | help(); 248 | } 249 | if (cl.hasOption(OPTION_FILE_SHORT) && cl.hasOption(OPTION_CLASSPATH_SHORT)) { 250 | throw new ExitException("May not specify both of the command line options " + OPTION_FILE_SHORT + " and " 251 | + OPTION_CLASSPATH_SHORT); 252 | } 253 | if (cl.hasOption(OPTION_FILE_SHORT)) { 254 | final String filePath = cl.getOptionValue(OPTION_FILE_SHORT); 255 | config = loadConfigFromFilepath(filePath); 256 | } else { 257 | config = loadConfigFromClasspath(DEFAULT_CONFIG_FILE_NAME); 258 | } 259 | if (cl.hasOption(OPTION_CLASSPATH_SHORT)) { 260 | final String file = cl.getOptionValue(OPTION_CLASSPATH_SHORT); 261 | config = loadConfigFromClasspath(file); 262 | } 263 | } 264 | 265 | } 266 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Provides an application that ingests the JSON image data from NASA JPL's Mars rovers into DynamoDB. 3 | */ 4 | package com.amazonaws.services.dynamodbv2.json.demo.mars; 5 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/util/DynamoDBManager.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.util; 2 | 3 | import java.util.logging.Logger; 4 | 5 | import com.amazonaws.AmazonClientException; 6 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 7 | import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; 8 | import com.amazonaws.services.dynamodbv2.model.DescribeTableResult; 9 | import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; 10 | import com.amazonaws.services.dynamodbv2.model.TableDescription; 11 | import com.amazonaws.services.dynamodbv2.model.TableStatus; 12 | 13 | /** 14 | * Provides static utility methods for managing DynamoDB tables. 15 | */ 16 | public final class DynamoDBManager { 17 | /** 18 | * Logger for {@link DynamoDBManager}. 19 | */ 20 | private static final Logger LOGGER = Logger.getLogger(DynamoDBManager.class.getName()); 21 | /** 22 | * Amount of time to wait between checking if a table has become ACTIVE. 23 | */ 24 | private static final long RETRY_DELAY = 20 * 1000; // 20 seconds 25 | /** 26 | * Number of times to to check if a table has become ACTIVE before failing. 27 | */ 28 | private static final int RETRY_COUNT = 3; 29 | 30 | /** 31 | * Creates DynamoDB table. If the table already exists, it validates the key schema. If the key schemas match, a 32 | * warning is logged, otherwise an exception is raised. 33 | * 34 | * @param dynamoDB 35 | * {@link AmazonDynamoDB} used to create the table specified in the request. 36 | * @param request 37 | * Request for creating a table. 38 | * @return TableDescription of the existing table or newly created table 39 | */ 40 | public static TableDescription createTable(final AmazonDynamoDB dynamoDB, final CreateTableRequest request) { 41 | try { 42 | final DescribeTableResult result = dynamoDB.describeTable(request.getTableName()); 43 | if (!request.getKeySchema().equals(result.getTable().getKeySchema())) { 44 | throw new IllegalStateException("Table " + request.getTableName() 45 | + " already exists and has an invalid schema"); 46 | } 47 | LOGGER.warning("Table " + request.getTableName() + " already exists"); 48 | return result.getTable(); 49 | } catch (final ResourceNotFoundException e) { 50 | return dynamoDB.createTable(request).getTableDescription(); 51 | } 52 | } 53 | 54 | /** 55 | * Checks if a table exists in DynamoDB. 56 | * 57 | * @param dynamoDB 58 | * A dynamoDB client configured for the proper region and credentials 59 | * @param tableName 60 | * The table to check for existence 61 | * @return True if the table exists, false if the table does not exist or an error occurs 62 | */ 63 | public static boolean doesTableExist(final AmazonDynamoDB dynamoDB, final String tableName) { 64 | try { 65 | dynamoDB.describeTable(tableName); 66 | return true; 67 | } catch (final ResourceNotFoundException e) { 68 | return false; 69 | } catch (final AmazonClientException e) { 70 | LOGGER.severe(e.getMessage()); 71 | return false; 72 | } 73 | 74 | } 75 | 76 | /** 77 | * Gets the table status. 78 | * 79 | * @param dynamoDB 80 | * The {@link AmazonDynamoDB} to use to get the table status 81 | * @param tableName 82 | * The table to get the status of 83 | * @return The table status 85 | */ 86 | public static TableStatus getTableStatus(final AmazonDynamoDB dynamoDB, final String tableName) { 87 | return TableStatus.fromValue(dynamoDB.describeTable(tableName).getTable().getTableStatus()); 88 | } 89 | 90 | /** 91 | * Blocks until the specified table becomes active or {@link #RETRY_COUNT} checks. There is a delay of 92 | * {@link #RETRY_DELAY} milliseconds between checks. 93 | * 94 | * @param dynamoDB 95 | * The {@link AmazonDynamoDB} to use to get the table status 96 | * @param tableName 97 | * The table to wait for an ACTIVE status 98 | */ 99 | public static void waitForTableToBecomeActive(final AmazonDynamoDB dynamoDB, final String tableName) { 100 | int numTries = 0; 101 | TableStatus currentState; 102 | try { 103 | currentState = getTableStatus(dynamoDB, tableName); 104 | } catch (final ResourceNotFoundException e) { 105 | throw new IllegalStateException("Table " + tableName + " does not exist"); 106 | } 107 | while (numTries < RETRY_COUNT) { 108 | try { 109 | Thread.sleep(RETRY_DELAY); 110 | } catch (final InterruptedException e) { 111 | LOGGER.severe(e.getMessage()); 112 | } 113 | currentState = getTableStatus(dynamoDB, tableName); 114 | numTries++; 115 | LOGGER.info("Table " + tableName + " is in " + currentState + " state"); 116 | switch (currentState) { 117 | case ACTIVE: 118 | return; 119 | case DELETING: 120 | throw new IllegalStateException("Table " + tableName 121 | + " has DELETING status and will never become active"); 122 | case UPDATING: 123 | break; 124 | case CREATING: 125 | break; 126 | default: 127 | throw new IllegalStateException("Unknown table status: " + currentState); 128 | } 129 | } 130 | 131 | final DescribeTableResult result = dynamoDB.describeTable(tableName); 132 | throw new IllegalStateException("Table " + tableName + " never went ACTIVE" + result); 133 | } 134 | 135 | /** 136 | * Private constructor for a static class. 137 | */ 138 | private DynamoDBManager() { 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/util/JSONParser.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.util; 2 | 3 | import java.io.IOException; 4 | import java.net.URL; 5 | 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import com.fasterxml.jackson.databind.ObjectMapper; 8 | 9 | /** 10 | * Provides static utility methods for retrieving JSON data from a URL. 11 | */ 12 | public final class JSONParser { 13 | /** 14 | * {@link ObjectMapper} used to parse data to a Jackson JSON tree representation. 15 | */ 16 | public static final ObjectMapper MAPPER = new ObjectMapper(); 17 | 18 | /** 19 | * Retrieves JSON from a URL. 20 | * 21 | * @param url 22 | * The URL to retrieve JSON data from 23 | * @param connectTimeout 24 | * Timeout for retrieving JSON data 25 | * @return {@link JsonNode} pointing to the head of the Jackson tree representation of the JSON data 26 | * @throws IOException 27 | * Invalid URL, JSON data, or connection error 28 | */ 29 | public static JsonNode getJSONFromURL(final URL url, final int connectTimeout) throws IOException { 30 | return getJSONFromURL(url, null, connectTimeout); 31 | } 32 | 33 | /** 34 | * Retrieves JSON from a URL that supports ETag headers if the current ETag is equal to the specified expected 35 | * value. 36 | * 37 | * @param url 38 | * The URL to retrieve JSON data from 39 | * @param expectedETag 40 | * Expected value for the ETag field when requesting the resource 41 | * @param connectTimeout 42 | * Timeout for retrieving JSON 43 | * @return {@link JsonNode} containing the head of the Jackson tree representation of the JSON data and the new ETag 44 | * @throws IOException 45 | * Invalid URL, JSON data, or connection error 46 | */ 47 | public static JsonNode getJSONFromURL(final URL url, final String expectedETag, final int connectTimeout) 48 | throws IOException { 49 | final byte[] data = NetworkUtils.getDataFromURL(url, expectedETag, connectTimeout); 50 | if (data != null) { 51 | return MAPPER.readTree(data); 52 | } else { 53 | return null; 54 | } 55 | } 56 | 57 | /** 58 | * Private constructor for a static class. 59 | */ 60 | private JSONParser() { 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/util/NetworkUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.util; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.net.HttpURLConnection; 7 | import java.net.URL; 8 | import java.util.logging.Level; 9 | import java.util.logging.Logger; 10 | 11 | /** 12 | * Utilities for retrieving data from http URLs. 13 | */ 14 | public final class NetworkUtils { 15 | /** 16 | * Logger for {@link NetworkUtils}. 17 | */ 18 | private static final Logger LOGGER = Logger.getLogger(NetworkUtils.class.getName()); 19 | /** 20 | * Header field key for ETag. 21 | */ 22 | public static final String ETAG_HEADER = "ETag"; 23 | /** 24 | * HTTP method HEAD. 25 | */ 26 | private static final String HEAD = "HEAD"; 27 | 28 | /** 29 | * Retrieves data from a URL. 30 | * 31 | * @param url 32 | * The URL to retrieve data from 33 | * @param connectTimeout 34 | * Connection timeout for retrieving data 35 | * @return byte array containing data 36 | * @throws IOException 37 | * Invalid URL or connection error 38 | */ 39 | public static byte[] getDataFromURL(final URL url, final int connectTimeout) throws IOException { 40 | return getDataFromURL(url, null, connectTimeout); 41 | } 42 | 43 | /** 44 | * Retrieves data from a URL that supports ETag headers if the current ETag is matches the expected value. 45 | * 46 | * @param url 47 | * The URL to retrieve data from 48 | * @param expectedETag 49 | * Previously recorded ETag for the resource 50 | * @param connectTimeout 51 | * Connection timeout for retrieving data 52 | * @return byte array containing retrieved data 53 | * @throws IOException 54 | * Invalid URL or connection error 55 | */ 56 | public static byte[] getDataFromURL(final URL url, final String expectedETag, final int connectTimeout) 57 | throws IOException { 58 | HttpURLConnection conn = null; 59 | InputStream in; 60 | try { 61 | conn = (HttpURLConnection) url.openConnection(); 62 | conn.setConnectTimeout(connectTimeout); 63 | conn.connect(); 64 | if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { 65 | in = conn.getInputStream(); 66 | if (expectedETag != null) { 67 | final String eTag = conn.getHeaderField(ETAG_HEADER); 68 | if (eTag == null || !eTag.equals(expectedETag)) { 69 | throw new IllegalStateException("Expected ETag: " + expectedETag + ". Actual ETag: " + eTag); 70 | } 71 | } 72 | final byte[] data = readStream(in); 73 | LOGGER.finer("Successfully retreived data from " + url.toExternalForm()); 74 | return data; 75 | } else { 76 | LOGGER.log(Level.WARNING, 77 | "Could not retrieve data from " + url.toExternalForm() + ": " + conn.getResponseCode() + " - " + conn.getResponseMessage()); 78 | return null; 79 | } 80 | } catch (final ClassCastException e) { 81 | throw new UnsupportedOperationException("URL is not a valid HTTP URL"); 82 | } finally { 83 | if (conn != null) { 84 | conn.disconnect(); 85 | } 86 | } 87 | 88 | } 89 | 90 | /** 91 | * Gets the ETag header String for a URL. 92 | * 93 | * @param url 94 | * URL to use to open an HTTPURLConnection 95 | * @return ETag header String 96 | * @throws IOException 97 | * connection error 98 | */ 99 | public static String getETag(final URL url) throws IOException { 100 | 101 | HttpURLConnection conn = null; 102 | try { 103 | conn = (HttpURLConnection) url.openConnection(); 104 | conn.setRequestMethod(HEAD); 105 | if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { 106 | if (conn.getHeaderField(ETAG_HEADER) != null) { 107 | return conn.getHeaderField(ETAG_HEADER); 108 | } else { 109 | throw new IOException("No ETag header present in " + url + ". " + conn.getResponseCode() + ": " 110 | + conn.getResponseMessage()); 111 | } 112 | } else { 113 | throw new IOException("Could not retrieve ETag header from " + url + ". " + conn.getResponseCode() 114 | + ": " + conn.getResponseMessage()); 115 | } 116 | } catch (final ClassCastException e) { 117 | throw new UnsupportedOperationException("URL is not a valid HTTP URL"); 118 | } finally { 119 | if (conn != null) { 120 | conn.disconnect(); 121 | } 122 | } 123 | } 124 | 125 | /** 126 | * Helper method to read an {@link InputStream} to a data source. 127 | * 128 | * @param in 129 | * Input stream pointing to the data source 130 | * @return byte array containing data 131 | * @throws IOException 132 | * Connection error 133 | */ 134 | private static byte[] readStream(final InputStream in) throws IOException { 135 | final ByteArrayOutputStream baos = new ByteArrayOutputStream(); 136 | final int bufSize = 1024 * 1024; // 1MB 137 | final byte[] buf = new byte[bufSize]; 138 | int n; 139 | while ((n = in.read(buf, 0, buf.length)) != -1) { 140 | baos.write(buf, 0, n); 141 | } 142 | return baos.toByteArray(); 143 | } 144 | 145 | /** 146 | * Private constructor for static class. 147 | */ 148 | private NetworkUtils() { 149 | 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/util/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Static utility classes for parsing configurations, and interacting with HTTP URLs and DynamoDB. 3 | */ 4 | package com.amazonaws.services.dynamodbv2.json.demo.mars.util; 5 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBImageWorker.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import java.awt.Graphics; 4 | import java.awt.image.BufferedImage; 5 | import java.io.ByteArrayInputStream; 6 | import java.io.ByteArrayOutputStream; 7 | import java.io.IOException; 8 | import java.net.URL; 9 | import java.util.Map; 10 | import java.util.logging.Level; 11 | import java.util.logging.Logger; 12 | 13 | import javax.imageio.ImageIO; 14 | 15 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 16 | import com.amazonaws.services.dynamodbv2.json.converter.JacksonConverter; 17 | import com.amazonaws.services.dynamodbv2.json.converter.impl.JacksonConverterImpl; 18 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.JSONParser; 19 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.MarsDynamoDBManager; 20 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.NetworkUtils; 21 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 22 | import com.fasterxml.jackson.databind.node.ObjectNode; 23 | 24 | /** 25 | * Takes a JSON representation of an image and puts it into the DynamoDB image table. Skips the image and reports a 26 | * warning if there is an error. 27 | */ 28 | public class DynamoDBImageWorker implements Runnable { 29 | /** 30 | * Logger for the {@link DynamoDBImageWorker}. 31 | */ 32 | private static final Logger LOGGER = Logger.getLogger(DynamoDBImageWorker.class.getName()); 33 | /** 34 | * Transformer for converting JSON to a DynamoDB item. 35 | */ 36 | private static final JacksonConverter CONVERTER = new JacksonConverterImpl(); 37 | 38 | /** 39 | * Helper method to process image from URL to thumbnail as base-64-encoded String. 40 | * 41 | * @param imageURL 42 | * URL to retrieve image from 43 | * @param expectedETag 44 | * The ETag to expect when retrieving the image 45 | * @param connectTimeout 46 | * Timeout for retrieving the image 47 | * @param thumbnailWidth 48 | * Width for resulting thumbnail 49 | * @param thumbnailHeight 50 | * Height for resulting thumbnail 51 | * @return Base-64-encoded String representation of the image thumbnail 52 | * @throws IOException 53 | * Error retrieving image or corrupt image data. 54 | */ 55 | protected static String getBase64EncodedImageFromURL(final URL imageURL, final String expectedETag, 56 | final int connectTimeout, final int thumbnailWidth, final int thumbnailHeight) throws IOException { 57 | final byte[] image = NetworkUtils.getDataFromURL(imageURL, expectedETag, connectTimeout); 58 | // Scale image down to thumbnail as byte array 59 | final byte[] thumbnail = makeThumbnail(image, thumbnailWidth, thumbnailHeight); 60 | // Base-64-encode the image 61 | final String encodedThumbnail = JSONParser.MAPPER.convertValue(thumbnail, String.class); 62 | return encodedThumbnail; 63 | } 64 | 65 | /** 66 | * Uses the java.awt library to scale the image. 67 | * 68 | * @param byteArray 69 | * byte array representation of the original image 70 | * @param thumbnailWidth 71 | * Width for resulting thumbnail 72 | * @param thumbnailHeight 73 | * Height for resulting thumbnail 74 | * @return byte array representation of the scaled thumbnail 75 | * @throws IOException 76 | * error reading or writing image 77 | */ 78 | protected static byte[] makeThumbnail(final byte[] byteArray, final int thumbnailWidth, final int thumbnailHeight) 79 | throws IOException { 80 | final ByteArrayInputStream bais = new ByteArrayInputStream(byteArray); 81 | final ByteArrayOutputStream baos = new ByteArrayOutputStream(); 82 | final BufferedImage thumb = new BufferedImage(thumbnailWidth, thumbnailHeight, BufferedImage.TYPE_INT_RGB); 83 | final BufferedImage im = ImageIO.read(bais); 84 | final Graphics g = thumb.createGraphics(); 85 | g.drawImage(im, 0, 0, thumbnailWidth, thumbnailHeight, null); 86 | g.dispose(); 87 | ImageIO.write(thumb, "jpg", baos); 88 | return baos.toByteArray(); 89 | } 90 | 91 | /** 92 | * {@link AmazonDynamoDB} used to persist image to DynamoDB. 93 | */ 94 | private final AmazonDynamoDB dynamoDB; 95 | /** 96 | * Raw JSON of the image from the sol. 97 | */ 98 | private final ObjectNode image; 99 | /** 100 | * DynamoDB table for persisting images. 101 | */ 102 | private final String imageTable; 103 | /** 104 | * DynamoDB table to read and write image ETAGs. 105 | */ 106 | private final String resourceTable; 107 | /** 108 | * Timeout for retrieving image data. 109 | */ 110 | private final int connectTimeout; 111 | /** 112 | * Width to process image to for thumbnail. 113 | */ 114 | private final int thumbnailWidth; 115 | /** 116 | * Height to process image to for thumbnail. 117 | */ 118 | private final int thumbnailHeight; 119 | /** 120 | * Flag for tracking resources by ETag in a resource table. 121 | */ 122 | private final boolean trackResources; 123 | /** 124 | * Flag for storing thumbnail data in the image table. 125 | */ 126 | private final boolean storeThumbnail; 127 | 128 | /** 129 | * Constructs A {@link DynamoDBImageWorker} to retrieve binary image and persist to DynamoDB. 130 | * 131 | * @param dynamoDB 132 | * Used to persist image to DynamoDB 133 | * @param imageTable 134 | * DynamoDB table for persisting images 135 | * @param resourceTable 136 | * DynamoDB table to read and write image ETAGs 137 | * @param image 138 | * Raw JSON of the image from the sol 139 | * @param connectTimeout 140 | * Timeout for retrieving image data 141 | * @param thumbnailWidth 142 | * Width to process image to for thumbnail 143 | * @param thumbnailHeight 144 | * Height to process image to for thumbnail 145 | * @param trackResources 146 | * Flag for tracking resources by ETag in a resource table 147 | * @param storeThumbnail 148 | * Flag for storing thumbnail data in the image table 149 | */ 150 | public DynamoDBImageWorker(final AmazonDynamoDB dynamoDB, final String imageTable, final String resourceTable, 151 | final ObjectNode image, final int connectTimeout, final int thumbnailWidth, final int thumbnailHeight, 152 | final boolean trackResources, final boolean storeThumbnail) { 153 | this.dynamoDB = dynamoDB; 154 | this.image = image; 155 | this.imageTable = imageTable; 156 | this.resourceTable = resourceTable; 157 | this.connectTimeout = connectTimeout; 158 | this.thumbnailWidth = thumbnailWidth; 159 | this.thumbnailHeight = thumbnailHeight; 160 | this.trackResources = trackResources; 161 | this.storeThumbnail = storeThumbnail; 162 | } 163 | 164 | /** 165 | * {@inheritDoc} 166 | */ 167 | @Override 168 | public void run() { 169 | 170 | try { 171 | // Retrieve image as thumbnail 172 | final String imageURL = image.get(MarsDynamoDBManager.IMAGE_TABLE_URL_ATTRIBUTE).asText(); 173 | String expectedETag = null; 174 | if (storeThumbnail) { 175 | String data = null; 176 | 177 | while (data == null) { 178 | if (trackResources) { 179 | final String oldETag = DynamoDBWorkerUtils.getStoredETag(dynamoDB, resourceTable, imageURL); 180 | final String newETag = NetworkUtils.getETag(new URL(imageURL)); 181 | if (newETag.equals(oldETag)) { 182 | LOGGER.fine("No change in image: " 183 | + image.get(MarsDynamoDBManager.IMAGE_TABLE_HASH_KEY).asText()); 184 | return; 185 | } else { 186 | expectedETag = newETag; 187 | } 188 | } 189 | data = getBase64EncodedImageFromURL(new URL(imageURL), expectedETag, connectTimeout, 190 | thumbnailWidth, thumbnailHeight); 191 | } 192 | // Store thumbnail data in the JSON 193 | image.put(MarsDynamoDBManager.IMAGE_TABLE_THUMBNAIL_ATTRIBUTE, data); 194 | } 195 | // Build the item 196 | final Map item = CONVERTER.jsonObjectToMap(image); 197 | // Put item into DynamoDB 198 | dynamoDB.putItem(imageTable, item); 199 | LOGGER.fine("Updated image: " + image.get(MarsDynamoDBManager.IMAGE_TABLE_HASH_KEY).asText()); 200 | if (trackResources) { 201 | DynamoDBWorkerUtils.updateETag(dynamoDB, resourceTable, imageURL, expectedETag); 202 | } 203 | } catch (final Exception e) { 204 | LOGGER.log(Level.WARNING, "Could not update image: " + image.get(MarsDynamoDBManager.IMAGE_TABLE_HASH_KEY), 205 | e); 206 | } 207 | 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBJSONRootWorker.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import java.io.IOException; 4 | import java.net.URL; 5 | import java.util.HashMap; 6 | import java.util.Iterator; 7 | import java.util.Map; 8 | import java.util.Map.Entry; 9 | import java.util.concurrent.Callable; 10 | import java.util.logging.Logger; 11 | 12 | import com.amazonaws.services.dynamodbv2.json.demo.mars.ExitException; 13 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.JSONParser; 14 | import com.fasterxml.jackson.databind.JsonNode; 15 | 16 | /** 17 | * Retrieves JSON from root URL and parses mission manifest URLs. Provides a map of mission name to mission URL. 18 | */ 19 | public class DynamoDBJSONRootWorker implements Callable> { 20 | /** 21 | * Logger for the {@link DynamoDBJSONRootWorker}. 22 | */ 23 | private static final Logger LOGGER = Logger.getLogger(DynamoDBJSONRootWorker.class.getName()); 24 | /** 25 | * JSON key for mission manifests. 26 | */ 27 | private static final String IMAGE_RESOURCE_KEY = "image_manifest"; 28 | 29 | /** 30 | * Processes the JSON root for Mars missions into a map of Mars mission names (i.e. MSL, MERA, MERB) to the JSON 31 | * Image Manifest URL. 32 | * 33 | * @param rootJsonUrl 34 | * The URL of the JSON root 35 | * @param connectTimeout 36 | * Timeout for retrieving the root JSON 37 | * @return Map of mission names to mission JSON URL 38 | * @throws IOException 39 | * Invalid URL, invalid JSON, or connection error 40 | */ 41 | public static Map getMissionToManifestMap(final URL rootJsonUrl, final int connectTimeout) 42 | throws IOException { 43 | final Map map = new HashMap(); 44 | // Retrieve JSON 45 | final JsonNode root = JSONParser.getJSONFromURL(rootJsonUrl, connectTimeout); 46 | final Iterator> it = root.fields(); 47 | // Top level is array of mission names 48 | while (it.hasNext()) { 49 | final Entry pair = it.next(); 50 | final String mission = pair.getKey(); 51 | if (pair.getKey() == null) { 52 | // Log error and skip mission 53 | LOGGER.warning("Null mission name"); 54 | continue; 55 | } 56 | final JsonNode missionObject = pair.getValue(); 57 | if (missionObject == null) { 58 | // Log error and skip mission 59 | LOGGER.warning("Null mission object for: " + mission); 60 | continue; 61 | } 62 | if (!missionObject.has(IMAGE_RESOURCE_KEY)) { 63 | // Log error and skip mission 64 | LOGGER.warning("Missing mission manifest for " + mission + ": " + missionObject.toString()); 65 | continue; 66 | } 67 | // Image manifest key has URL 68 | final String manifestValue = missionObject.get(IMAGE_RESOURCE_KEY).asText(); 69 | map.put(mission, manifestValue); 70 | } 71 | return map; 72 | } 73 | 74 | // state 75 | /** 76 | * URL of the root JSON. 77 | */ 78 | private final String rootURL; 79 | /** 80 | * Connection timeout for retrieving the root JSON. 81 | */ 82 | private final int connectTimeout; 83 | 84 | /** 85 | * Constructs a {@link DynamoDBJSONRootWorker} that retrieves and parses the root JSON at the provided URL using the 86 | * specified timeout. 87 | * 88 | * @param rootURL 89 | * URL of the root JSON 90 | * @param connectTimeout 91 | * Connection timeout for retrieving the root JSON 92 | */ 93 | public DynamoDBJSONRootWorker(final String rootURL, final int connectTimeout) { 94 | this.rootURL = rootURL; 95 | this.connectTimeout = connectTimeout; 96 | } 97 | 98 | /** 99 | * {@inheritDoc} 100 | */ 101 | @Override 102 | public Map call() throws Exception { 103 | Map topLevelManifests = null; 104 | try { 105 | // Get JSON root and process to map 106 | topLevelManifests = DynamoDBJSONRootWorker.getMissionToManifestMap(new URL(rootURL), connectTimeout); 107 | } catch (final IOException e) { 108 | throw new ExitException("Error connecting to " + rootURL, e); 109 | } 110 | return topLevelManifests; 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBMissionWorker.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import java.io.IOException; 4 | import java.net.URL; 5 | import java.util.Arrays; 6 | import java.util.Collections; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.concurrent.Callable; 11 | import java.util.logging.Level; 12 | import java.util.logging.Logger; 13 | 14 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.JSONParser; 15 | import com.fasterxml.jackson.databind.JsonNode; 16 | import com.fasterxml.jackson.databind.node.ArrayNode; 17 | 18 | /** 19 | * Retrieves a mission manifest and processes it into a map of sol number to sol URL. 20 | */ 21 | public class DynamoDBMissionWorker implements Callable> { 22 | /** 23 | * Logger for DynamoDBMissionWorker. 24 | */ 25 | private static final Logger LOGGER = Logger.getLogger(DynamoDBMissionWorker.class.getName()); 26 | // Parsing constants 27 | /** 28 | * JSON key for resource type. 29 | */ 30 | private static final String RESOURCE_TYPE_KEY = "type"; 31 | /** 32 | * Supported resource types. Used to fail gracefully if NASA changes data model. 33 | */ 34 | private static final List SUPPORTED_TYPES = Arrays.asList("mer-images-manifest-1.0", 35 | "msl-images-manifest-2.0"); 36 | /** 37 | * JSON key for sols array. 38 | */ 39 | private static final String SOLS_LIST_KEY = "sols"; 40 | /** 41 | * JSON key for sol number. 42 | */ 43 | private static final String SOL_ID_KEY = "sol"; 44 | /** 45 | * JSON key for sol URL. 46 | */ 47 | private static final String SOL_URL_KEY = "url"; 48 | 49 | /** 50 | * Retrieves and parses a mission manifest to a map of sol numbers to sol URLs. 51 | * 52 | * @param url 53 | * Location of the mission manifest 54 | * @param connectTimeout 55 | * Timeout for retrieving the mission manifest 56 | * @return Map of sol number to sol URL contained in the mission manifest 57 | * @throws IOException 58 | * Invalid URL, invalid JSON data, or connection error 59 | */ 60 | public static Map getSolJSON(final URL url, final int connectTimeout) throws IOException { 61 | final Map map = new HashMap(); 62 | // Retrieve the JSON data 63 | final JsonNode manifest = JSONParser.getJSONFromURL(url, connectTimeout); 64 | // Validate the JSON data version 65 | if (!manifest.has(RESOURCE_TYPE_KEY) || !SUPPORTED_TYPES.contains(manifest.get(RESOURCE_TYPE_KEY).asText())) { 66 | throw new IllegalArgumentException("Manifest version verification failed"); 67 | } 68 | // Validate that the JSON data contains a sol list 69 | if (!manifest.has(SOLS_LIST_KEY)) { 70 | throw new IllegalArgumentException("Manifest does not contain a sol list"); 71 | } 72 | final ArrayNode sols = (ArrayNode) manifest.get(SOLS_LIST_KEY); 73 | // Process each sol in the sol list 74 | for (int i = 0; i < sols.size(); i++) { 75 | final JsonNode sol = sols.path(i); 76 | if (sol.has(SOL_ID_KEY) && sol.has(SOL_URL_KEY)) { 77 | final Integer solID = sol.get(SOL_ID_KEY).asInt(); 78 | final String solURL = sol.get(SOL_URL_KEY).asText(); 79 | if (solID != null && solURL != null) { 80 | // Add valid sol to the map 81 | map.put(solID, solURL); 82 | } else { 83 | LOGGER.warning("Sol contains unexpected values: " + sol); 84 | } 85 | } else { 86 | LOGGER.warning("Sol missing required keys: "); 87 | } 88 | } 89 | return map; 90 | } 91 | 92 | // State 93 | /** 94 | * URL of the mission manifest. 95 | */ 96 | private final String manifestURL; 97 | /** 98 | * Timeout for retreiving mission manifest. 99 | */ 100 | private final int connectTimeout; 101 | 102 | /** 103 | * Constructs new {@link DynamoDBMissionWorker} to retrieve manifest at the specified URL. Will use specified 104 | * timeout when connecting. 105 | * 106 | * @param manifestURL 107 | * URL of the mission manifest to retrieve 108 | * @param connectTimeout 109 | * Amount of time in milliseconds to timeout while retrieving manifest 110 | */ 111 | public DynamoDBMissionWorker(final String manifestURL, final int connectTimeout) { 112 | this.manifestURL = manifestURL; 113 | this.connectTimeout = connectTimeout; 114 | } 115 | 116 | /** 117 | * {@inheritDoc} 118 | */ 119 | @Override 120 | public Map call() throws Exception { 121 | try { 122 | // Always check manifest - sol could have updated 123 | final Map sols = getSolJSON(new URL(manifestURL), connectTimeout); 124 | LOGGER.info("Processed Manifest (" + sols.size() + " sols): " + manifestURL); 125 | return sols; 126 | 127 | } catch (final Exception e) { 128 | LOGGER.log(Level.WARNING, "Skipping manifest: " + manifestURL, e); 129 | return Collections.emptyMap(); 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBWorkerUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 7 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.MarsDynamoDBManager; 8 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 9 | import com.amazonaws.services.dynamodbv2.model.GetItemResult; 10 | 11 | /** 12 | * Provides a static method for retrieving an ETag stored in DynamoDB. 13 | */ 14 | public final class DynamoDBWorkerUtils { 15 | /** 16 | * DynamoDB item key for ETAG. 17 | */ 18 | public static final String ETAG_KEY = "ETag"; 19 | 20 | /** 21 | * Retrieves the stored ETag, if one exists, from DynamoDB. 22 | * 23 | * @param dynamoDB 24 | * DynamoDB client configured with a region and credentials 25 | * @param table 26 | * The resource table name 27 | * @param resource 28 | * The URL String of the resource 29 | * @return The ETag String of the last copy processed or null if the resource has never been processed 30 | */ 31 | public static String getStoredETag(final AmazonDynamoDB dynamoDB, final String table, final String resource) { 32 | String oldETag; 33 | // Build key to retrieve item 34 | final Map resourceKey = new HashMap(); 35 | resourceKey.put(MarsDynamoDBManager.RESOURCE_TABLE_HASH_KEY, new AttributeValue(resource)); 36 | // Get item 37 | final GetItemResult result = dynamoDB.getItem(table, resourceKey); 38 | final Map item = result.getItem(); 39 | if (item != null && item.containsKey(ETAG_KEY)) { 40 | // Item was found and contains ETag 41 | oldETag = item.get(ETAG_KEY).getS(); 42 | } else { 43 | // Item was not found or did not contain ETag 44 | oldETag = null; 45 | } 46 | return oldETag; 47 | } 48 | 49 | /** 50 | * Updates the resource table for the specified resource with the specified ETag. 51 | * 52 | * @param dynamoDB 53 | * DynamoDB client configured with a region and credentials 54 | * @param table 55 | * The DynamoDB resource table 56 | * @param resource 57 | * The resource URL 58 | * @param eTag 59 | * The new ETag for the resource 60 | */ 61 | public static void updateETag(final AmazonDynamoDB dynamoDB, final String table, final String resource, 62 | final String eTag) { 63 | // Build item 64 | final Map newResource = new HashMap<>(); 65 | newResource.put(MarsDynamoDBManager.RESOURCE_TABLE_HASH_KEY, new AttributeValue(resource)); 66 | newResource.put(DynamoDBWorkerUtils.ETAG_KEY, new AttributeValue(eTag)); 67 | dynamoDB.putItem(table, newResource); 68 | } 69 | 70 | /** 71 | * Private constructor for static class. 72 | */ 73 | private DynamoDBWorkerUtils() { 74 | 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /ingester/src/main/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Asynchronous workers for parsing and persisting the multiple layers of JSON published by NASA JPL. 3 | */ 4 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 5 | -------------------------------------------------------------------------------- /ingester/src/test/java/ParserTestFiles/IMAGE_MANIFEST_missing_sol_list.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "msl-images-manifest-2.0", 3 | "latest_sol": 753, 4 | "num_images": 100152, 5 | "most_recent_image": "2014-09-19T10:59:18.912Z", 6 | "last_manifest_update": "2014-09-19T21:37:56.000Z" 7 | } -------------------------------------------------------------------------------- /ingester/src/test/java/ParserTestFiles/JSON_ROOT.json: -------------------------------------------------------------------------------- 1 | { 2 | "MERA": { 3 | "image_manifest": "https://merpublic.s3.amazonaws.com/oss/mera/images/image_manifest.json" 4 | }, 5 | "MERB": { 6 | "image_manifest": "https://merpublic.s3.amazonaws.com/oss/merb/images/image_manifest.json" 7 | }, 8 | "MSL": 9 | {"image_manifest": "https://msl-raws.s3.amazonaws.com/images/image_manifest.json" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /ingester/src/test/java/ParserTestFiles/JSON_ROOT_empty_mission_body.json: -------------------------------------------------------------------------------- 1 | { 2 | "MERA": { 3 | "image_manifest": "https://merpublic.s3.amazonaws.com/oss/mera/images/image_manifest.json" 4 | }, 5 | "MERB": { 6 | 7 | }, 8 | "MSL": 9 | {"image_manifest": "https://msl-raws.s3.amazonaws.com/images/image_manifest.json" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/converter/TestJacksonConvertorImpl.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertTrue; 5 | 6 | import java.io.File; 7 | import java.util.ArrayList; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | 15 | import com.amazonaws.services.dynamodbv2.json.converter.impl.JacksonConverterImpl; 16 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 17 | import com.fasterxml.jackson.databind.JsonNode; 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import com.fasterxml.jackson.databind.node.JsonNodeFactory; 20 | import com.fasterxml.jackson.databind.node.ObjectNode; 21 | 22 | public class TestJacksonConvertorImpl { 23 | private static JacksonConverter convertor; 24 | private static final String testFile = ClassLoader.getSystemResource("flickr.json").getFile(); 25 | 26 | @Test 27 | public void giveLoopedJsonNode() throws Exception { 28 | final ObjectNode node = JsonNodeFactory.instance.objectNode(); 29 | 30 | node.put("child", node); 31 | 32 | try { 33 | convertor.jsonObjectToMap(node); 34 | } catch (final JacksonConverterException e){ 35 | // expected behavior 36 | assert(e.getMessage().startsWith("Max depth reached.")); 37 | } 38 | } 39 | 40 | 41 | @Test 42 | public void giveLoopedMap() throws Exception { 43 | final Map item = new HashMap(); 44 | item.put("child", new AttributeValue().withM(item)); 45 | 46 | 47 | try { 48 | convertor.mapToJsonObject(item); 49 | } catch (final JacksonConverterException e){ 50 | // expected behavior 51 | assert(e.getMessage().startsWith("Max depth reached.")); 52 | } 53 | } 54 | 55 | @Test 56 | public void giveWrongJsonNode() throws Exception { 57 | final ObjectMapper mapper = new ObjectMapper(); 58 | final JsonNode jsonArray = mapper.readValue("[]", JsonNode.class); 59 | try { 60 | convertor.jsonObjectToMap(jsonArray); 61 | } catch (final JacksonConverterException e){ 62 | // Correct behavior 63 | } 64 | } 65 | 66 | @Test 67 | public void itemToJsonObject() throws Exception { 68 | final ObjectMapper mapper = new ObjectMapper(); 69 | final JsonNode json = mapper.readValue(new File(testFile), JsonNode.class); 70 | final Map item = convertor.jsonObjectToMap(json); 71 | 72 | final JsonNode node = convertor.mapToJsonObject(item); 73 | assertEquals("14911691861", node.get("id").textValue()); 74 | assertEquals(6, node.get("farm").intValue()); 75 | assertTrue(node.get("views-index").isDouble()); 76 | assertTrue(node.get("fans").isArray()); 77 | assertEquals(JsonNodeFactory.instance.nullNode(), node.get("video")); 78 | } 79 | 80 | @Test 81 | public void loadEmptyArray() throws Exception { 82 | final ObjectMapper mapper = new ObjectMapper(); 83 | final JsonNode json = mapper.readValue("[]", JsonNode.class); 84 | 85 | final List item = convertor.jsonArrayToList(json); 86 | assertEquals(0, item.size()); 87 | } 88 | 89 | @Test 90 | public void loadEmptyObject() throws Exception { 91 | final ObjectMapper mapper = new ObjectMapper(); 92 | final JsonNode json = mapper.readValue("{}", JsonNode.class); 93 | 94 | final Map item = convertor.jsonObjectToMap(json); 95 | assertEquals(0, item.size()); 96 | } 97 | 98 | @Before 99 | public void setup(){ 100 | convertor = new JacksonConverterImpl(); 101 | } 102 | 103 | @Test 104 | public void simpleJsonObjectToItem() throws Exception { 105 | final ObjectMapper mapper = new ObjectMapper(); 106 | final JsonNode json = mapper.readValue(new File(testFile), JsonNode.class); 107 | 108 | final Map item = convertor.jsonObjectToMap(json); 109 | final ArrayList fans = new ArrayList(); 110 | fans.add(new AttributeValue().withS("kentay")); 111 | 112 | assertEquals(new AttributeValue().withS("14911691861"), item.get("id")); 113 | assertEquals(new AttributeValue().withN("6"), item.get("farm")); 114 | assertEquals(new AttributeValue().withL(fans), item.get("fans")); 115 | assertEquals(new AttributeValue().withN("2.14735356869"), item.get("views-index")); 116 | assertEquals(new AttributeValue().withNULL(true), item.get("video")); 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/converter/TestJacksonStreamReader.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.converter; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertTrue; 5 | 6 | import java.io.File; 7 | import java.util.ArrayList; 8 | import java.util.Map; 9 | 10 | import org.junit.Before; 11 | import org.junit.Test; 12 | 13 | import com.amazonaws.services.dynamodbv2.json.converter.impl.JacksonStreamReaderImpl; 14 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 15 | import com.fasterxml.jackson.core.JsonFactory; 16 | import com.fasterxml.jackson.core.JsonParseException; 17 | import com.fasterxml.jackson.core.JsonParser; 18 | 19 | public class TestJacksonStreamReader { 20 | private static final String imageManifest = ClassLoader.getSystemResource("image_manifest_converter.json").getFile(); 21 | private static final String imageManifestBroken = ClassLoader.getSystemResource("image_manifest_broken.json") 22 | .getFile(); 23 | private static final String flickrPhoto = ClassLoader.getSystemResource("flickr.json").getFile(); 24 | private JsonFactory jsonFactory; 25 | 26 | @Test 27 | public void getFirstSolMetadata() throws Exception { 28 | final JsonParser jp = jsonFactory.createJsonParser(new File(imageManifest)); 29 | final JacksonStreamReaderImpl reader = new JacksonStreamReaderImpl(jp); 30 | // Seek for sols array 31 | assertTrue(reader.seek("sols")); 32 | 33 | final Map item = reader.getNextItem(); 34 | assertEquals("1", item.get("sol").getN()); 35 | assertEquals("150", item.get("num_images").getN()); 36 | assertEquals("2004-01-30T08:11:15.222Z", item.get("most_recent_image").getS()); 37 | assertEquals("2013-12-05T20:37:00.000Z", item.get("last_manifest_update").getS()); 38 | assertEquals("http://merpublic.s3.amazonaws.com/oss/merb/images/images_sol1.json", item.get("url").getS()); 39 | } 40 | 41 | @Test 42 | public void getLastSolMetadata() throws Exception { 43 | final JsonParser jp = jsonFactory.createJsonParser(new File(imageManifest)); 44 | final JacksonStreamReaderImpl reader = new JacksonStreamReaderImpl(jp); 45 | // Seek for sols array 46 | assertTrue(reader.seek("sols")); 47 | 48 | Map item = null; 49 | Map next; 50 | while ((next = reader.getNextItem()) != null) { 51 | item = next; 52 | } 53 | assertEquals("3760", item.get("sol").getN()); 54 | assertEquals("0", item.get("num_images").getN()); 55 | assertEquals("1900", item.get("most_recent_image").getS()); 56 | assertEquals("2014-08-22T19:06:50.000Z", item.get("last_manifest_update").getS()); 57 | assertEquals("http://merpublic.s3.amazonaws.com/oss/merb/images/images_sol3760.json", item.get("url").getS()); 58 | assertEquals(null, reader.getNextItem()); 59 | } 60 | 61 | @Test 62 | public void loadBrokenJsonFile() throws Exception { 63 | final JsonParser jp = jsonFactory.createJsonParser(new File(imageManifestBroken)); 64 | final JacksonStreamReaderImpl reader = new JacksonStreamReaderImpl(jp); 65 | // Seek for sols array 66 | assertTrue(reader.seek("sols")); 67 | 68 | final Map item = reader.getNextItem(); 69 | // First item can be read correctly 70 | assertEquals("1", item.get("sol").getN()); 71 | 72 | try { 73 | // Second one should fail because the second item is corrupted 74 | reader.getNextItem(); 75 | } catch (final JsonParseException e) { 76 | // Expected behavior 77 | } 78 | } 79 | 80 | @Test 81 | public void loadEmptyArray() throws Exception { 82 | final JsonParser jp = jsonFactory.createJsonParser("[]"); 83 | 84 | final JacksonStreamReaderImpl reader = new JacksonStreamReaderImpl(jp); 85 | final Map item = reader.getNextItem(); 86 | 87 | assert (item == null); 88 | } 89 | 90 | @Test 91 | public void loadEmptyObject() throws Exception { 92 | final JsonParser jp = jsonFactory.createJsonParser("{}"); 93 | 94 | final JacksonStreamReaderImpl reader = new JacksonStreamReaderImpl(jp); 95 | final Map item = reader.getNextItem(); 96 | 97 | assertEquals(0, item.size()); 98 | } 99 | 100 | @Test 101 | public void loadFlickrMetadata() throws Exception { 102 | final JsonParser jp = jsonFactory.createJsonParser(new File(flickrPhoto)); 103 | 104 | final JacksonStreamReaderImpl reader = new JacksonStreamReaderImpl(jp); 105 | final Map item = reader.getNextItem(); 106 | 107 | final ArrayList fans = new ArrayList(); 108 | fans.add(new AttributeValue().withS("kentay")); 109 | 110 | assertEquals(new AttributeValue().withS("14911691861"), item.get("id")); 111 | assertEquals(new AttributeValue().withN("6"), item.get("farm")); 112 | assertEquals(new AttributeValue().withL(fans), item.get("fans")); 113 | assertEquals(new AttributeValue().withN("2.14735356869"), item.get("views-index")); 114 | assertEquals(new AttributeValue().withNULL(true), item.get("video")); 115 | } 116 | 117 | @Test 118 | public void loadImageManifestAsASingleItem() throws Exception { 119 | final JsonParser jp = jsonFactory.createJsonParser(new File(imageManifest)); 120 | 121 | final JacksonStreamReaderImpl reader = new JacksonStreamReaderImpl(jp); 122 | final Map item = reader.getNextItem(); 123 | assertEquals("mer-images-manifest-1.0", item.get("type").getS()); 124 | assertEquals(3286, item.get("sols").getL().size()); 125 | } 126 | 127 | @Before 128 | public void setup() { 129 | jsonFactory = new JsonFactory(); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/PhotoIngesterCLITest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars; 2 | 3 | import static org.junit.Assert.assertTrue; 4 | 5 | import java.io.IOException; 6 | import java.io.OutputStream; 7 | import java.io.PrintStream; 8 | 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | 12 | public class PhotoIngesterCLITest { 13 | 14 | private static final String JSON_ROOT = "JSON.root"; 15 | private static final String JSON_ROOT_VALUE = "http://json.jpl.nasa.gov/data.json"; 16 | private static final String JSON_ROOT_VALUE2 = "https://s3.amazonaws.com/dynamodb-mars-json/root.json"; 17 | private static final PrintStream nullStream = new PrintStream(new OutputStream() { 18 | @Override 19 | public void write(final int b) throws IOException { 20 | } 21 | }); 22 | 23 | @Before 24 | public void setup() { 25 | System.setOut(nullStream); 26 | System.setErr(nullStream); 27 | } 28 | 29 | @Test 30 | public void testEmpty() throws ExitException { 31 | final String args[] = { "" }; 32 | final ImageIngesterCLI cli = new ImageIngesterCLI(args); 33 | assertTrue(cli.getConfig() != null); 34 | assertTrue(cli.getConfig().get(JSON_ROOT).equals(JSON_ROOT_VALUE) || cli.getConfig().get(JSON_ROOT).equals(JSON_ROOT_VALUE2)); 35 | } 36 | 37 | @Test 38 | public void testFile() throws ExitException { 39 | final String args[] = { "-f", "ImageIngester.properties" }; 40 | final ImageIngesterCLI cli = new ImageIngesterCLI(args); 41 | assertTrue(cli.getConfig() != null); 42 | assertTrue(cli.getConfig().get(JSON_ROOT).equals(JSON_ROOT_VALUE) || cli.getConfig().get(JSON_ROOT).equals(JSON_ROOT_VALUE2)); 43 | } 44 | 45 | @Test(expected = HelpException.class) 46 | public void testHelp() throws ExitException { 47 | final String args[] = { "-f", "file.properties", "-h" }; 48 | new ImageIngesterCLI(args).getConfig(); 49 | 50 | } 51 | 52 | @Test(expected = HelpException.class) 53 | public void testHelp2() throws ExitException { 54 | final String args[] = { "-f", "file.properties", "--help" }; 55 | new ImageIngesterCLI(args).getConfig(); 56 | } 57 | 58 | @Test(expected = HelpException.class) 59 | public void testInvalidOption() throws ExitException { 60 | final String args[] = { "-f", "file.properties", "--invalid-option" }; 61 | new ImageIngesterCLI(args).getConfig(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/util/JSONParserTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.util; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.fail; 5 | import static org.powermock.api.easymock.PowerMock.replayAll; 6 | 7 | import java.io.ByteArrayInputStream; 8 | import java.io.IOException; 9 | import java.io.InputStream; 10 | import java.net.URL; 11 | 12 | import org.junit.Test; 13 | import org.junit.runner.RunWith; 14 | import org.powermock.api.easymock.PowerMock; 15 | import org.powermock.core.classloader.annotations.PrepareForTest; 16 | import org.powermock.modules.junit4.PowerMockRunner; 17 | 18 | import com.amazonaws.services.dynamodbv2.json.demo.mars.ImageIngester; 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | 22 | @RunWith(PowerMockRunner.class) 23 | @PrepareForTest({ URL.class, JSONParser.class, NetworkUtils.class }) 24 | public class JSONParserTest { 25 | private static final String ROOT_JSON = "{\"MERA\":{\"image_manifest\":\"https://merpublic.s3.amazonaws.com/oss/mera/images/image_manifest.json\"},\"MSL\":{\"image_manifest\":\"https://msl-raws.s3.amazonaws.com/images/image_manifest.json\"},\"MERB\":{\"image_manifest\":\"https://merpublic.s3.amazonaws.com/oss/merb/images/image_manifest.json\"}}"; 26 | 27 | @Test(expected = IOException.class) 28 | public void testBadURL() throws IOException { 29 | final URL url = PowerMock.createMock(URL.class); 30 | url.openConnection(); 31 | PowerMock.expectLastCall().andThrow(new IOException()); 32 | replayAll(); 33 | JSONParser.getJSONFromURL(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 34 | 35 | } 36 | 37 | @Test 38 | public void testgetJSONFromURL() { 39 | PowerMock.mockStatic(NetworkUtils.class); 40 | try { 41 | final URL url = PowerMock.createMock(URL.class); 42 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 43 | PowerMock.expectLastCall().andReturn(ROOT_JSON.getBytes()); 44 | replayAll(); 45 | final ObjectMapper mapper = new ObjectMapper(); 46 | final JsonNode json = mapper.readTree(ROOT_JSON); 47 | final JsonNode result = JSONParser.getJSONFromURL(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 48 | assertEquals(json, result); 49 | } catch (final IOException e) { 50 | fail(); 51 | } 52 | 53 | } 54 | 55 | @Test(expected = IOException.class) 56 | public void testInvalidURL() throws IOException { 57 | JSONParser.getJSONFromURL(new URL("invalidURL"), ImageIngester.DEFAULT_CONNECT_TIMEOUT); 58 | fail("URL should be invalid"); 59 | 60 | } 61 | 62 | @Test(expected = IOException.class) 63 | public void testNotJSON() throws IOException { 64 | PowerMock.mockStatic(NetworkUtils.class); 65 | final String notJSON = "I am not json"; 66 | final URL url = PowerMock.createMock(URL.class); 67 | InputStream is = null; 68 | try { 69 | is = new ByteArrayInputStream(notJSON.getBytes()); 70 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 71 | PowerMock.expectLastCall().andReturn(notJSON.getBytes()); 72 | replayAll(); 73 | JSONParser.getJSONFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 74 | } finally { 75 | if (is != null) { 76 | is.close(); 77 | } 78 | } 79 | fail("Source is invalid json"); 80 | } 81 | 82 | @Test(expected = NullPointerException.class) 83 | public void testNull() { 84 | try { 85 | JSONParser.getJSONFromURL(null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 86 | } catch (final IOException e) { 87 | fail(e.getMessage()); 88 | } 89 | } 90 | 91 | @Test(expected = NullPointerException.class) 92 | public void testNullURL() { 93 | try { 94 | JSONParser.getJSONFromURL(null, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 95 | } catch (final IOException e) { 96 | fail("URL is null"); 97 | } 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/util/MarsDynamoDBManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.util; 2 | 3 | import java.util.Arrays; 4 | 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.powermock.api.easymock.PowerMock; 8 | import org.powermock.core.classloader.annotations.PrepareForTest; 9 | import org.powermock.modules.junit4.PowerMockRunner; 10 | 11 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 12 | import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; 13 | import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndex; 14 | import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; 15 | 16 | @RunWith(PowerMockRunner.class) 17 | @PrepareForTest({ AmazonDynamoDB.class, DynamoDBManager.class }) 18 | public class MarsDynamoDBManagerTest { 19 | 20 | private static final ProvisionedThroughput PROVISIONED_THROUGHPUT = new ProvisionedThroughput(); 21 | private static final String TABLE_NAME = "table"; 22 | 23 | @Test 24 | public void testCreateImageTable() { 25 | final AmazonDynamoDB dynamoDB = PowerMock.createMock(AmazonDynamoDB.class); 26 | PowerMock.mockStatic(DynamoDBManager.class); 27 | final CreateTableRequest request = new CreateTableRequest(); 28 | request.setAttributeDefinitions(MarsDynamoDBManager.IMAGE_TABLE_ATTRIBUTE_DEFINITIONS); 29 | request.setKeySchema(MarsDynamoDBManager.IMAGE_TABLE_KEY_SCHEMA); 30 | final GlobalSecondaryIndex timeGSI = new GlobalSecondaryIndex(); 31 | timeGSI.setIndexName(MarsDynamoDBManager.IMAGE_TABLE_TIME_GSI_NAME); 32 | timeGSI.setKeySchema(Arrays.asList(MarsDynamoDBManager.IMAGE_TABLE_TIME_GSI_HASH_KSE, 33 | MarsDynamoDBManager.IMAGE_TABLE_TIME_GSI_RANGE_KSE)); 34 | timeGSI.setProjection(MarsDynamoDBManager.IMAGE_TABLE_TIME_GSI_PROJECTION); 35 | timeGSI.setProvisionedThroughput(PROVISIONED_THROUGHPUT); 36 | final GlobalSecondaryIndex voteGSI = new GlobalSecondaryIndex(); 37 | voteGSI.setIndexName(MarsDynamoDBManager.IMAGE_TABLE_VOTE_GSI_NAME); 38 | voteGSI.setKeySchema(Arrays.asList(MarsDynamoDBManager.IMAGE_TABLE_VOTE_GSI_HASH_KSE, 39 | MarsDynamoDBManager.IMAGE_TABLE_VOTE_GSI_RANGE_KSE)); 40 | voteGSI.setProjection(MarsDynamoDBManager.IMAGE_TABLE_VOTE_GSI_PROJECTION); 41 | voteGSI.setProvisionedThroughput(PROVISIONED_THROUGHPUT); 42 | request.setGlobalSecondaryIndexes(Arrays.asList(timeGSI, voteGSI)); 43 | request.setProvisionedThroughput(PROVISIONED_THROUGHPUT); 44 | request.setTableName(TABLE_NAME); 45 | 46 | DynamoDBManager.createTable(dynamoDB, request); 47 | PowerMock.expectLastCall().andReturn(null); 48 | PowerMock.replayAll(); 49 | MarsDynamoDBManager.createImageTable(dynamoDB, TABLE_NAME, PROVISIONED_THROUGHPUT, PROVISIONED_THROUGHPUT, 50 | PROVISIONED_THROUGHPUT); 51 | PowerMock.verifyAll(); 52 | 53 | } 54 | 55 | @Test 56 | public void testCreateResourceTable() { 57 | final AmazonDynamoDB dynamoDB = PowerMock.createMock(AmazonDynamoDB.class); 58 | PowerMock.mockStatic(DynamoDBManager.class); 59 | final CreateTableRequest request = new CreateTableRequest(); 60 | request.setAttributeDefinitions(MarsDynamoDBManager.RESOURCE_TABLE_ATTRIBUTE_DEFINITIONS); 61 | request.setKeySchema(MarsDynamoDBManager.RESOURCE_TABLE_KEY_SCHEMA); 62 | request.setProvisionedThroughput(PROVISIONED_THROUGHPUT); 63 | request.setTableName(TABLE_NAME); 64 | 65 | DynamoDBManager.createTable(dynamoDB, request); 66 | PowerMock.expectLastCall().andReturn(null); 67 | PowerMock.replayAll(); 68 | MarsDynamoDBManager.createResourceTable(dynamoDB, TABLE_NAME, PROVISIONED_THROUGHPUT); 69 | PowerMock.verifyAll(); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBJSONRootWorkerTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertTrue; 5 | import static org.junit.Assert.fail; 6 | 7 | import java.io.ByteArrayOutputStream; 8 | import java.io.IOException; 9 | import java.net.URL; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | import java.util.logging.Handler; 13 | import java.util.logging.Logger; 14 | import java.util.logging.SimpleFormatter; 15 | import java.util.logging.StreamHandler; 16 | 17 | import org.junit.Test; 18 | import org.junit.runner.RunWith; 19 | import org.powermock.api.easymock.PowerMock; 20 | import org.powermock.core.classloader.annotations.PrepareForTest; 21 | import org.powermock.modules.junit4.PowerMockRunner; 22 | 23 | import com.amazonaws.services.dynamodbv2.json.demo.mars.ImageIngester; 24 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.JSONParser; 25 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.NetworkUtils; 26 | import com.amazonaws.util.json.JSONUtils; 27 | 28 | @RunWith(PowerMockRunner.class) 29 | @PrepareForTest({ URL.class, JSONParser.class, NetworkUtils.class }) 30 | public class DynamoDBJSONRootWorkerTest { 31 | private static final String ROOT_JSON_FILE = WorkerTestUtils.getPath("JSON_ROOT.json"); 32 | private static final String EMPTY_MISSION_BODY_FILE = WorkerTestUtils.getPath("JSON_ROOT_empty_mission_body.json"); 33 | 34 | private static final Map EXPECTED_MAP; 35 | static { 36 | EXPECTED_MAP = new HashMap(); 37 | EXPECTED_MAP.put("MERA", "https://merpublic.s3.amazonaws.com/oss/mera/images/image_manifest.json"); 38 | EXPECTED_MAP.put("MERB", "https://merpublic.s3.amazonaws.com/oss/merb/images/image_manifest.json"); 39 | EXPECTED_MAP.put("MSL", "https://msl-raws.s3.amazonaws.com/images/image_manifest.json"); 40 | } 41 | 42 | @Test 43 | public void testEmptyMissionBody() throws IOException { 44 | final Logger logger = Logger.getLogger(DynamoDBJSONRootWorker.class.getName()); 45 | Handler handler; 46 | final ByteArrayOutputStream os = new ByteArrayOutputStream(); 47 | final URL url = PowerMock.createMock(URL.class); 48 | PowerMock.mockStatic(JSONUtils.class); 49 | PowerMock.mockStatic(NetworkUtils.class); 50 | String manifest = null; 51 | try { 52 | manifest = WorkerTestUtils.readFile(EMPTY_MISSION_BODY_FILE); 53 | } catch (final IOException e1) { 54 | fail("Could not read file: " + EMPTY_MISSION_BODY_FILE); 55 | } 56 | try { 57 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 58 | PowerMock.expectLastCall().andReturn(manifest.getBytes()); 59 | PowerMock.replayAll(); 60 | handler = new StreamHandler(os, new SimpleFormatter()); 61 | logger.setUseParentHandlers(false); 62 | logger.addHandler(handler); 63 | try { 64 | DynamoDBJSONRootWorker.getMissionToManifestMap(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 65 | } catch (final IOException e) { 66 | fail(e.getMessage()); 67 | } 68 | handler.flush(); 69 | assertTrue(os.toString().contains("WARNING: Missing mission manifest for MERB: {}")); 70 | } finally { 71 | try { 72 | os.close(); 73 | } catch (final IOException e) { 74 | fail(e.getMessage()); 75 | } 76 | } 77 | } 78 | 79 | @Test 80 | public void testGetMissionToManifestMap() { 81 | final URL url = PowerMock.createMock(URL.class); 82 | PowerMock.mockStatic(JSONUtils.class); 83 | PowerMock.mockStatic(NetworkUtils.class); 84 | String manifest = null; 85 | try { 86 | manifest = WorkerTestUtils.readFile(ROOT_JSON_FILE); 87 | } catch (final IOException e1) { 88 | fail("Could not read file: " + ROOT_JSON_FILE); 89 | } 90 | try { 91 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 92 | PowerMock.expectLastCall().andReturn(manifest.getBytes()); 93 | PowerMock.replayAll(); 94 | final Map missionMap = DynamoDBJSONRootWorker.getMissionToManifestMap(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 95 | assertEquals(EXPECTED_MAP, missionMap); 96 | } catch (final IOException e) { 97 | fail(e.getMessage()); 98 | } 99 | 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBMissionWorkerTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.assertTrue; 5 | import static org.junit.Assert.fail; 6 | 7 | import java.io.ByteArrayOutputStream; 8 | import java.io.IOException; 9 | import java.net.URL; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | import java.util.logging.Handler; 13 | import java.util.logging.Logger; 14 | import java.util.logging.SimpleFormatter; 15 | import java.util.logging.StreamHandler; 16 | 17 | import org.junit.Rule; 18 | import org.junit.Test; 19 | import org.junit.rules.ExpectedException; 20 | import org.junit.runner.RunWith; 21 | import org.powermock.api.easymock.PowerMock; 22 | import org.powermock.core.classloader.annotations.PrepareForTest; 23 | import org.powermock.modules.junit4.PowerMockRunner; 24 | 25 | import com.amazonaws.services.dynamodbv2.json.demo.mars.ImageIngester; 26 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.JSONParser; 27 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.NetworkUtils; 28 | import com.amazonaws.util.json.JSONUtils; 29 | 30 | @RunWith(PowerMockRunner.class) 31 | @PrepareForTest({ URL.class, JSONParser.class, NetworkUtils.class }) 32 | public class DynamoDBMissionWorkerTest { 33 | 34 | private static final String IMAGE_RESOURCE_FILE = WorkerTestUtils.getPath("IMAGE_MANIFEST.json"); 35 | 36 | private static final String INVALID_VERSION_FILE = WorkerTestUtils.getPath("IMAGE_MANIFEST_invalid_version.json"); 37 | private static final String MISSING_SOL_LIST_FILE = WorkerTestUtils.getPath("IMAGE_MANIFEST_missing_sol_list.json"); 38 | private static final String MISSING_SOL_ID_FILE = WorkerTestUtils.getPath("IMAGE_MANIFEST_missing_sol_id.json"); 39 | private static final String MISSING_SOL_URL_FILE = WorkerTestUtils.getPath("IMAGE_MANIFEST_missing_sol_url.json"); 40 | 41 | private static final Map EXPECTED_MAP; 42 | static { 43 | EXPECTED_MAP = new HashMap(); 44 | for (int i = 0; i <= 753; i++) { 45 | EXPECTED_MAP.put(i, "http://msl-raws.s3.amazonaws.com/images/images_sol" + i + ".json"); 46 | } 47 | final int[] missingSols = { 557, 570, 577, 596, 598, 599, 600, 616, 625, 693 }; 48 | for (final Integer missingSol : missingSols) { 49 | EXPECTED_MAP.remove(missingSol); 50 | } 51 | } 52 | 53 | @Rule 54 | public ExpectedException expectedEx = ExpectedException.none(); 55 | 56 | @Test 57 | public void testGetSolJSON() { 58 | final URL url = PowerMock.createMock(URL.class); 59 | PowerMock.mockStatic(JSONUtils.class); 60 | PowerMock.mockStatic(NetworkUtils.class); 61 | String manifest = null; 62 | try { 63 | manifest = WorkerTestUtils.readFile(IMAGE_RESOURCE_FILE); 64 | } catch (final IOException e1) { 65 | fail("Could not read file: " + IMAGE_RESOURCE_FILE); 66 | } 67 | try { 68 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 69 | PowerMock.expectLastCall().andReturn(manifest.getBytes()); 70 | PowerMock.replayAll(); 71 | final Map actual = DynamoDBMissionWorker.getSolJSON(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 72 | assertEquals(EXPECTED_MAP, actual); 73 | } catch (final IOException e) { 74 | fail(e.getMessage()); 75 | } 76 | } 77 | 78 | @Test 79 | public void testInvalidVersion() { 80 | expectedEx.expect(IllegalArgumentException.class); 81 | expectedEx.expectMessage("version verification failed"); 82 | final URL url = PowerMock.createMock(URL.class); 83 | PowerMock.mockStatic(JSONUtils.class); 84 | PowerMock.mockStatic(NetworkUtils.class); 85 | String manifest = null; 86 | try { 87 | manifest = WorkerTestUtils.readFile(INVALID_VERSION_FILE); 88 | } catch (final IOException e) { 89 | fail("Could not read file: " + INVALID_VERSION_FILE); 90 | } 91 | try { 92 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 93 | PowerMock.expectLastCall().andReturn(manifest.getBytes()); 94 | PowerMock.replayAll(); 95 | DynamoDBMissionWorker.getSolJSON(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 96 | } catch (final IOException e) { 97 | fail(e.getMessage()); 98 | } 99 | fail("Version is incorrect"); 100 | } 101 | 102 | @Test 103 | public void testMissingSolID() { 104 | final Logger logger = Logger.getLogger(DynamoDBMissionWorker.class.getName()); 105 | Handler handler; 106 | final ByteArrayOutputStream os = new ByteArrayOutputStream(); 107 | final URL url = PowerMock.createMock(URL.class); 108 | PowerMock.mockStatic(JSONUtils.class); 109 | PowerMock.mockStatic(NetworkUtils.class); 110 | String manifest = null; 111 | try { 112 | manifest = WorkerTestUtils.readFile(MISSING_SOL_ID_FILE); 113 | } catch (final IOException e) { 114 | fail("Could not read file: " + MISSING_SOL_ID_FILE); 115 | } 116 | try { 117 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 118 | PowerMock.expectLastCall().andReturn(manifest.getBytes()); 119 | PowerMock.replayAll(); 120 | handler = new StreamHandler(os, new SimpleFormatter()); 121 | logger.setUseParentHandlers(false); 122 | logger.addHandler(handler); 123 | DynamoDBMissionWorker.getSolJSON(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 124 | handler.flush(); 125 | assertTrue(os.toString().contains("WARNING: Sol missing required keys")); 126 | } catch (final IOException e) { 127 | fail(e.getMessage()); 128 | } finally { 129 | try { 130 | os.close(); 131 | } catch (final IOException e) { 132 | fail(e.getMessage()); 133 | } 134 | } 135 | } 136 | 137 | @Test 138 | public void testMissingSolList() { 139 | expectedEx.expect(IllegalArgumentException.class); 140 | expectedEx.expectMessage("does not contain a sol list"); 141 | final URL url = PowerMock.createMock(URL.class); 142 | PowerMock.mockStatic(JSONUtils.class); 143 | PowerMock.mockStatic(NetworkUtils.class); 144 | String manifest = null; 145 | try { 146 | manifest = WorkerTestUtils.readFile(MISSING_SOL_LIST_FILE); 147 | } catch (final IOException e) { 148 | fail("Could not read file: " + MISSING_SOL_LIST_FILE); 149 | } 150 | try { 151 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 152 | PowerMock.expectLastCall().andReturn(manifest.getBytes()); 153 | PowerMock.replayAll(); 154 | DynamoDBMissionWorker.getSolJSON(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 155 | } catch (final IOException e) { 156 | fail(e.getMessage()); 157 | } 158 | fail("Missing sol list"); 159 | } 160 | 161 | @Test 162 | public void testMissingSolURL() { 163 | final Logger logger = Logger.getLogger(DynamoDBMissionWorker.class.getName()); 164 | Handler handler; 165 | final ByteArrayOutputStream os = new ByteArrayOutputStream(); 166 | final URL url = PowerMock.createMock(URL.class); 167 | PowerMock.mockStatic(JSONUtils.class); 168 | PowerMock.mockStatic(NetworkUtils.class); 169 | String manifest = null; 170 | try { 171 | manifest = WorkerTestUtils.readFile(MISSING_SOL_URL_FILE); 172 | } catch (final IOException e) { 173 | fail("Could not read file: " + MISSING_SOL_URL_FILE); 174 | } 175 | try { 176 | NetworkUtils.getDataFromURL(url, null, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 177 | PowerMock.expectLastCall().andReturn(manifest.getBytes()); 178 | PowerMock.replayAll(); 179 | handler = new StreamHandler(os, new SimpleFormatter()); 180 | logger.setUseParentHandlers(false); 181 | logger.addHandler(handler); 182 | DynamoDBMissionWorker.getSolJSON(url, ImageIngester.DEFAULT_CONNECT_TIMEOUT); 183 | handler.flush(); 184 | assertTrue(os.toString().contains("WARNING: Sol missing required keys")); 185 | } catch (final IOException e) { 186 | fail(e.getMessage()); 187 | } finally { 188 | try { 189 | os.close(); 190 | } catch (final IOException e) { 191 | fail(e.getMessage()); 192 | } 193 | } 194 | } 195 | 196 | } 197 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBSolWorkerTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | import static org.junit.Assert.fail; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.powermock.core.classloader.annotations.PrepareForTest; 12 | import org.powermock.modules.junit4.PowerMockRunner; 13 | 14 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 15 | import com.fasterxml.jackson.databind.JsonNode; 16 | import com.fasterxml.jackson.databind.ObjectMapper; 17 | import com.fasterxml.jackson.databind.node.ArrayNode; 18 | 19 | @RunWith(PowerMockRunner.class) 20 | @PrepareForTest({AmazonDynamoDB.class}) 21 | public class DynamoDBSolWorkerTest { 22 | private static final String MSL_SOL_FILE = WorkerTestUtils.getPath("SOL_MSL.json"); 23 | private static final String MSL_SOL_EXPECTED_FILE = WorkerTestUtils.getPath("SOL_MSL_EXPECTED.json"); 24 | private static final String MER_SOL_FILE = WorkerTestUtils.getPath("SOL_MER.json"); 25 | private static final String MER_SOL_EXPECTED_FILE = WorkerTestUtils.getPath("SOL_MER_EXPECTED.json"); 26 | private final ObjectMapper mapper = new ObjectMapper(); 27 | 28 | @Test 29 | public void testParseMERSol() { 30 | testParseSol(MER_SOL_FILE, MER_SOL_EXPECTED_FILE); 31 | } 32 | 33 | @Test 34 | public void testParseMSLSol() { 35 | testParseSol(MSL_SOL_FILE, MSL_SOL_EXPECTED_FILE); 36 | } 37 | 38 | private void testParseSol(final String file, final String expectedFile) { 39 | try { 40 | final JsonNode sol = mapper.readTree(new File(file)); 41 | final ArrayNode expected = (ArrayNode) mapper.readTree(new File(expectedFile)); 42 | 43 | final ArrayNode result = DynamoDBSolWorker.getImages(sol); 44 | assertEquals(expected, result); 45 | } catch (final IOException e) { 46 | fail(e.getMessage()); 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/DynamoDBWorkerUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.powermock.api.easymock.PowerMock; 11 | import org.powermock.core.classloader.annotations.PrepareForTest; 12 | import org.powermock.modules.junit4.PowerMockRunner; 13 | 14 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 15 | import com.amazonaws.services.dynamodbv2.json.demo.mars.util.MarsDynamoDBManager; 16 | import com.amazonaws.services.dynamodbv2.json.demo.mars.worker.DynamoDBWorkerUtils; 17 | import com.amazonaws.services.dynamodbv2.model.AttributeValue; 18 | import com.amazonaws.services.dynamodbv2.model.GetItemResult; 19 | 20 | @RunWith(PowerMockRunner.class) 21 | @PrepareForTest({ AmazonDynamoDB.class }) 22 | public class DynamoDBWorkerUtilsTest { 23 | 24 | private static final String table = "table"; 25 | private static final String resource = "resource"; 26 | private static final String eTag = "eTag"; 27 | 28 | @Test 29 | public void testGetStoredETagExists() { 30 | AmazonDynamoDB dynamoDB = PowerMock.createMock(AmazonDynamoDB.class); 31 | Map resourceKey = new HashMap(); 32 | resourceKey.put(MarsDynamoDBManager.RESOURCE_TABLE_HASH_KEY, new AttributeValue(resource)); 33 | // Get item 34 | dynamoDB.getItem(table, resourceKey); 35 | Map resourceResult = new HashMap(); 36 | resourceResult.put(MarsDynamoDBManager.RESOURCE_TABLE_HASH_KEY, new AttributeValue(resource)); 37 | resourceResult.put(DynamoDBWorkerUtils.ETAG_KEY, new AttributeValue(eTag)); 38 | GetItemResult result = new GetItemResult().withItem(resourceResult); 39 | PowerMock.expectLastCall().andReturn(result); 40 | PowerMock.replayAll(); 41 | String resultETag = DynamoDBWorkerUtils.getStoredETag(dynamoDB, table, resource); 42 | assertEquals(eTag, resultETag); 43 | PowerMock.verifyAll(); 44 | } 45 | 46 | @Test 47 | public void testGetStoredETagDoesNotExist() { 48 | AmazonDynamoDB dynamoDB = PowerMock.createMock(AmazonDynamoDB.class); 49 | Map resourceKey = new HashMap(); 50 | resourceKey.put(MarsDynamoDBManager.RESOURCE_TABLE_HASH_KEY, new AttributeValue(resource)); 51 | // Get item 52 | dynamoDB.getItem(table, resourceKey); 53 | GetItemResult result = new GetItemResult(); 54 | PowerMock.expectLastCall().andReturn(result); 55 | PowerMock.replayAll(); 56 | String resultETag = DynamoDBWorkerUtils.getStoredETag(dynamoDB, table, resource); 57 | assertEquals(null, resultETag); 58 | PowerMock.verifyAll(); 59 | } 60 | 61 | @Test 62 | public void testUpdateETag() { 63 | AmazonDynamoDB dynamoDB = PowerMock.createMock(AmazonDynamoDB.class); 64 | Map newItem = new HashMap(); 65 | newItem.put(MarsDynamoDBManager.RESOURCE_TABLE_HASH_KEY, new AttributeValue(resource)); 66 | newItem.put(DynamoDBWorkerUtils.ETAG_KEY, new AttributeValue(eTag)); 67 | dynamoDB.putItem(table, newItem); 68 | PowerMock.expectLastCall().andReturn(null); 69 | PowerMock.replayAll(); 70 | DynamoDBWorkerUtils.updateETag(dynamoDB, table, resource, eTag); 71 | PowerMock.verifyAll(); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /ingester/src/test/java/com/amazonaws/services/dynamodbv2/json/demo/mars/worker/WorkerTestUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazonaws.services.dynamodbv2.json.demo.mars.worker; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileReader; 6 | import java.io.IOException; 7 | import java.net.URL; 8 | 9 | public class WorkerTestUtils { 10 | 11 | public static final String CLASSPATH = System.getProperty("java.class.path"); 12 | public static final String FILE_ENCODING = "UTF-8"; 13 | 14 | public static String getPath(final String fileName) { 15 | final URL url = ClassLoader.getSystemResource(fileName); 16 | if (url != null) { 17 | return url.getFile(); 18 | } else { 19 | throw new IllegalArgumentException("File " + fileName + " does not exist on the classpath: " + CLASSPATH); 20 | } 21 | } 22 | 23 | static String readFile(final String file) throws IOException { 24 | BufferedReader reader = null; 25 | try { 26 | reader = new BufferedReader(new FileReader(new File(file))); 27 | final StringBuilder sb = new StringBuilder(); 28 | String line; 29 | while ((line = reader.readLine()) != null) { 30 | sb.append(line); 31 | sb.append(System.getProperty("line.separator")); 32 | } 33 | return sb.toString(); 34 | } finally { 35 | if (reader != null) { 36 | reader.close(); 37 | } 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /ingester/src/test/java/res/flickr.json: -------------------------------------------------------------------------------- 1 | { "id": "14911691861", 2 | "owner": "124077012@N04", 3 | "secret": "d876c32954", 4 | "server": "5576", 5 | "farm": 6, 6 | "title": "", 7 | "ispublic": 1, 8 | "isfriend": 0, 9 | "isfamily": 0, 10 | "video": null, 11 | "views-index": 2.14735356869, 12 | "fans": [ 13 | "kentay" 14 | ] 15 | } -------------------------------------------------------------------------------- /ingester/src/test/java/res/image_manifest_broken.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "mer-images-manifest-1.0", 3 | "latest_sol": 3760, 4 | "num_images": 139326, 5 | "most_recent_image": "2014-08-22T18:10:49.793Z", 6 | "last_manifest_update": "2014-08-22T19:06:50.000Z", 7 | "sols": [ 8 | { 9 | "sol": 1, 10 | "num_images": 150, 11 | "most_recent_image": "2004-01-30T08:11:15.222Z", 12 | "last_manifest_update": "2013-12-05T20:37:00.000Z", 13 | "url": "http://merpublic.s3.amazonaws.com/oss/merb/images/images_sol1.json" 14 | }, 15 | { 16 | "sol": 2, 17 | "num_images": 280, 18 | "most_recent_image": "2004-02-01T22:51:30.251Z", 19 | -------------------------------------------------------------------------------- /viewer/README.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | The Mars Science Laboratory Image Explorer is a data exploration application that allows a user to view Mars Rover images. This is a sample application to demonstrate the following features of DynamoDB: 3 | - JSON data support - image data is from http://json.jpl.nasa.gov and packaged in json format 4 | - Indexes on JSON data - indexes are built on top of the json data for faster querying 5 | - Large Items - thumbnail data can be pre-generated and stored in DynamoDB as binary 6 | - DynamoDB Document SDK - hides low level data structure in Amazon DynamoDB and provides document level API with developers 7 | 8 | It has timeline view and the user can look into specific point in time. The user can also switch from one equipment to another with the control panel on the side menu. Users can also vote on images and show images based on popularity rather than time. Each use case is implemented with Amazon DynamoDB. 9 | 10 | ## Getting started 11 | 1. Install node.js and ruby for your platform if not installed 12 | - node.js: http://nodejs.org/download/ 13 | - ruby: https://www.ruby-lang.org/en/installation/ 14 | 2. Install necessary tools, i.e. grunt, bower, coffee-script and compass 15 | ``` 16 | sudo npm -g install grunt-cli bower coffee-script 17 | sudo gem install compass 18 | ``` 19 | 3. Install dependencies 20 | ``` 21 | npm install 22 | bower install 23 | ``` 24 | 4. Run the following command to start the demo app. 25 | ``` 26 | grunt serve 27 | ``` 28 | This will download DynamoDB Local, ingest a small subset of images from NASA ,launches a local HTTP server and opens the demo app. 29 | 30 | ## Data Model 31 | ### 32 | - HashKey: imageid 33 | - RangeKey: ''none'' 34 | - GSI 1 (date-gsi) 35 | - HashKey: Mission+InstrumentID (concatenated) 36 | - RangeKey: TimeStamp 37 | - Projection (time, votes, mission, instrument) 38 | - GSI 2 (vote-gsi) 39 | - HashKey: Mission+InstrumentID (concatenated) 40 | - RangeKey: votes 41 | 42 | ### 43 | - HashKey: userid 44 | - RangeKey: imageid 45 | 46 | ### Queries used to retrieve data 47 | - Fetching timeline photos: Query marsDemoImages with date-gsi + getItem on each imageid to get thumbnails 48 | - Fetching top voted photos: Query marsDemoImages with vote-gsi + getItem on each imageid to get thumbnails 49 | - Fetching user voted photos: Query userVotes table + getItem on each imageid to get thumbnails 50 | - Voting on a photo: Conditional Update on userVotes table with (userid, imageid) and, if it successes, add 1 to votes attribute on marsDemoImages table 51 | 52 | ## Building distributable package 53 | 1. Edit the configuration in `lib/mynconf.coffee` as appropriate or overwride the parameters with environment variables with the same name. The default configuration is as follows. 54 | ``` 55 | nconf.defaults 56 | DYNAMODB_ENDPOINT_DEV: 'http://localhost:9000/dynamodb/' 57 | DYNAMODB_REGION_DEV: 'us-east-1' 58 | DYNAMODB_ENDPOINT_TEST: 'http://localhost:8080/dynamodb/' 59 | DYNAMODB_REGION_TEST: 'us-east-1' 60 | DYNAMODB_ENDPOINT_PROD: 'http://dynamodb.us-east-1.amazonaws.com/' 61 | DYNAMODB_REGION_PROD: 'us-east-1' 62 | USE_COGNITO_DEV: false 63 | USE_COGNITO_TEST: false 64 | USE_COGNITO_PROD: true 65 | AWS_ACCOUNT_ID: 'DummyAWSAccountID' 66 | COGNITO_IDENTITY_POOL_ID: 'DummyCognitoIdenityPoolID' 67 | COGNITO_UNAUTH_ROLE_ARN 'DummyCognitoUnauthRoleARN' 68 | TABLE_PHOTOS: 'marsDemoImages' 69 | TABLE_USER_VOTES: 'userVotes' 70 | TABLE_RESOURCES: 'marsDemoResources' 71 | READ_CAPACITY_PHOTOS: 1 72 | WRITE_CAPACITY_PHOTOS: 1 73 | ``` 74 | 75 | 2. Run the following command 76 | ``` 77 | grunt build 78 | ``` 79 | 80 | The above step will create two distribution directories. 81 | - dist -- contains the web app that can be deployed on an HTTP server, e.g. Amazon S3 82 | - dist-launcher -- contains the web app and a grunt script to launch the web app locally 83 | 84 | ## Automated Test 85 | The following command performs unit tests with DynamoDB Local. 86 | ``` 87 | grunt test 88 | ``` -------------------------------------------------------------------------------- /viewer/app/.buildignore: -------------------------------------------------------------------------------- 1 | *.coffee -------------------------------------------------------------------------------- /viewer/app/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Page Not Found :( 6 | 141 | 142 | 143 |
144 |

Not found :(

145 |

Sorry, but the page you were trying to view does not exist.

146 |

It looks like this was the result of either:

147 |
    148 |
  • a mistyped address
    • 149 |
  • an out-of-date link
    • 150 |
151 | 154 | 155 |
156 | 157 | 158 | -------------------------------------------------------------------------------- /viewer/app/images/mars.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-dynamodb-mars-json-demo/81c2536951bdfea5fbf3852b943d87c0e4a4cb32/viewer/app/images/mars.jpg -------------------------------------------------------------------------------- /viewer/app/images/rover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-dynamodb-mars-json-demo/81c2536951bdfea5fbf3852b943d87c0e4a4cb32/viewer/app/images/rover.jpg -------------------------------------------------------------------------------- /viewer/app/images/rover_jumbotron.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/aws-dynamodb-mars-json-demo/81c2536951bdfea5fbf3852b943d87c0e4a4cb32/viewer/app/images/rover_jumbotron.jpg -------------------------------------------------------------------------------- /viewer/app/index.jade: -------------------------------------------------------------------------------- 1 | extends views/layout 2 | 3 | block content 4 | 5 | 6 | .jumbotron 7 | h1 MSL Image Explorer 8 | p Let's explore Mars with the rover! 9 | 10 | 11 | div(ng-view="") 12 | 13 | .copyright 14 | p Images Courtesy NASA/JPL-Caltech. 15 | 16 | -------------------------------------------------------------------------------- /viewer/app/robots.txt: -------------------------------------------------------------------------------- 1 | # robotstxt.org 2 | 3 | User-agent: * 4 | -------------------------------------------------------------------------------- /viewer/app/scripts/app.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | angular.module('MSLImageExplorerApp', 3 | [ 4 | 'ngRoute', 5 | 'ngSanitize', 6 | 'ngTouch', 7 | 'ui.bootstrap.modal', 8 | 'config' 9 | ]). 10 | config(['$routeProvider', function($routeProvider) { 11 | $routeProvider. 12 | when('/', { 13 | templateUrl: 'views/partials/timeline.html', 14 | controller: 'TimelineCtrl' 15 | }). 16 | when('/timeline', { 17 | templateUrl: 'views/partials/timeline.html', 18 | controller: 'TimelineCtrl' 19 | }). 20 | when('/timeline/:instrument', { 21 | templateUrl: 'views/partials/timeline.html', 22 | controller: 'TimelineCtrl' 23 | }). 24 | when('/timeline/:instrument/:time', { 25 | templateUrl: 'views/partials/timeline.html', 26 | controller: 'TimelineCtrl' 27 | }). 28 | when('/topVoted', { 29 | templateUrl: 'views/partials/image-gallery.html', 30 | controller: 'TopVotedCtrl' 31 | }). 32 | when('/topVoted/:instrument', { 33 | templateUrl: 'views/partials/image-gallery.html', 34 | controller: 'TopVotedCtrl' 35 | }). 36 | when('/favorites', { 37 | templateUrl: 'views/partials/image-gallery.html', 38 | controller: 'FavoritesCtrl' 39 | }). 40 | otherwise({ 41 | redirectTo: '/' 42 | }); 43 | }]); 44 | 45 | 46 | -------------------------------------------------------------------------------- /viewer/app/scripts/controllers/dialog.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /** 3 | * @ngdoc function 4 | * @name MSLImageExplorerApp.controller:DialogCtrl 5 | * @description 6 | * # DialogCtrl 7 | * Controller of the MSLImageExplorerApp that handles the error dialog view. 8 | */ 9 | angular.module('MSLImageExplorerApp'). 10 | controller('DialogCtrl', function ($scope, $modalInstance, title, message) { 11 | /** 12 | * Sets the title. 13 | */ 14 | $scope.title = title; 15 | 16 | /** 17 | * Sets the message. 18 | */ 19 | $scope.message = message; 20 | 21 | /** 22 | * Closes the dialog. 23 | */ 24 | $scope.close = function(){ 25 | $modalInstance.close(); 26 | }; 27 | }); 28 | 29 | -------------------------------------------------------------------------------- /viewer/app/scripts/controllers/favorites.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | global $ 4 | */ 5 | 6 | /** 7 | * @ngdoc function 8 | * @name MSLImageExplorerApp.controller:FavoritesCtrl 9 | * @description 10 | * # FavoritesCtrl 11 | * Controller of the MSLImageExplorerApp that takes care of favorite photos view. 12 | */ 13 | angular.module('MSLImageExplorerApp'). 14 | controller('FavoritesCtrl', function ($scope, $log, MarsPhotosDBAccess, Blueimp) { 15 | 16 | $scope.title = 'Mars Images You Liked'; 17 | $scope.description = 'Photos that you liked.'; 18 | $scope.photos = []; 19 | 20 | /** 21 | * Fetches photos that the user has voted 22 | * via MarsPhotos service and replaces $scope.photos with 23 | * the results. It also triggers Blueimp 24 | * image gallery to prepare for the slideshow. 25 | */ 26 | $scope.updatePhotos = function(){ 27 | MarsPhotosDBAccess.queryUserVotedPhotos({ 28 | hashKey: localStorage.getItem('userid'), 29 | lastEvaluatedKey: $scope.lastEvaluatedKey 30 | }, function(error, data){ 31 | if (!error) { 32 | var photos = data.Items; 33 | $scope.lastEvaluatedKey = data.LastEvaluatedKey; 34 | $scope.$apply(function(){ 35 | $scope.photos = photos; 36 | }); 37 | } else { 38 | $log.error(error); 39 | } 40 | }); 41 | }; 42 | 43 | $scope.updatePhotos(); 44 | Blueimp.Gallery($('#links a'), $('#blueimp-gallery').data()); 45 | }); 46 | 47 | -------------------------------------------------------------------------------- /viewer/app/scripts/controllers/sidemenu.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | global $ 4 | */ 5 | 6 | /** 7 | * @ngdoc function 8 | * @name MSLImageExplorerApp.controller:SideMenuCtrl 9 | * @description 10 | * # SideMenuCtrl 11 | * Controller of the MSLImageExplorerApp that takes care of sidemenu a.k.a Mission Control. 12 | */ 13 | angular.module('MSLImageExplorerApp'). 14 | controller('SideMenuCtrl', function ($scope, $modal, $log, MarsPhotosDBAccess) { 15 | /** 16 | * List of instruments to select from. 17 | */ 18 | $scope.instrumentList = MarsPhotosDBAccess.instrumentList; 19 | 20 | $scope.active = true; 21 | 22 | /** 23 | * Flag that indicates if the sidebar is initialized 24 | */ 25 | $scope.isSidrInitialized = false; 26 | 27 | /** 28 | * Flag that indicates if the instrument list should be shown or not. 29 | */ 30 | $scope.showInstrumentList = false; 31 | 32 | /** 33 | * Opens the side menu. It initializes the component 34 | * on the first time it is called. 35 | */ 36 | $scope.openSidr = function(){ 37 | if (!$scope.isSidrInitialized) { 38 | initSidr(); 39 | $scope.isSidrInitialized = true; 40 | } 41 | $.sidr('open'); 42 | }; 43 | 44 | /** 45 | * Closes the side menu. 46 | */ 47 | $scope.closeSidr = function(){ 48 | $.sidr('close'); 49 | if ($scope.modalInstance) { 50 | $scope.modalInstance.dismiss('cancel'); 51 | } 52 | }; 53 | 54 | /** 55 | * Shows a modal window that shows the rover detail figure and 56 | * the list of instruments on the side menu. 57 | */ 58 | $scope.showRoverDetails = function(){ 59 | $scope.modalInstance = $modal.open({ 60 | templateUrl: 'views/partials/rover-detail.html', 61 | controller: 'DialogCtrl', 62 | resolve: { 63 | title: 'Instrument Selection', 64 | message: '' 65 | } 66 | }); 67 | $scope.showInstrumentList = true; 68 | }; 69 | 70 | /** 71 | * Sets the instrument to the one the user selected and reloads 72 | * the parent view, e.g. the timeline view and top-voted photos view. 73 | * It also closes the modal window that shows the rover details. 74 | */ 75 | $scope.setInstrument = function(instrument, $event) { 76 | $log.debug('Instrument is set to ' + instrument); 77 | $($event.currentTarget).addClass('active').siblings().removeClass('active'); 78 | $scope.$parent.instrument = instrument; 79 | $scope.showInstrumentList = false; 80 | $scope.modalInstance.dismiss('cancel'); 81 | 82 | reloadParentView(); 83 | }; 84 | 85 | /** 86 | * Called when the view is destroyed and closes the side menu. 87 | */ 88 | $scope.$on('$destroy', function(){ 89 | $scope.closeSidr(); 90 | }); 91 | 92 | /** 93 | * Calls reload function in the parent controller, e.g. timeline or 94 | * top-voted photos view, if exists. It triggers reloading of the 95 | * parent view with the specified time and instrument. 96 | */ 97 | var reloadParentView = function(){ 98 | if (typeof($scope.$parent.reload) === 'function') { 99 | $log.debug('Reloading parent view'); 100 | $scope.$parent.reload(); 101 | } else { 102 | $log.warn('Parent view does not have reload()'); 103 | } 104 | }; 105 | 106 | /** 107 | * Initializes the side menu and the date picker in it. 108 | */ 109 | var initSidr = function(){ 110 | $('#sidemenu').sidr({ 111 | side: 'right' 112 | }); 113 | $('.datepicker').datepicker('update', new Date()); 114 | if ($scope.time) { 115 | try { 116 | $('.datepicker').datepicker('update', new Date(parseInt($scope.time))); 117 | } catch (e) { 118 | $log.error('Invalid time specified: ' + $scope.time); 119 | } 120 | } 121 | 122 | $('.datepicker').datepicker().on('changeDate', function(ev){ 123 | $(ev.target).datepicker('hide'); 124 | $scope.$parent.time = ev.date.valueOf(); 125 | reloadParentView(); 126 | }); 127 | $('.datepicker').datepicker().on('show', function(){ 128 | // Workaround to remove a triangle at left top 129 | $('.datepicker-dropdown').removeClass('datepicker-orient-top'); 130 | }); 131 | }; 132 | }); 133 | 134 | -------------------------------------------------------------------------------- /viewer/app/scripts/controllers/timeline.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | global $, moment 4 | */ 5 | 6 | /** 7 | * @ngdoc function 8 | * @name MSLImageExplorerApp.controller:TimelineCtrl 9 | * @description 10 | * # TimelineCtrl 11 | * Controller of the MSLImageExplorerApp that takes care of the main timeline view. 12 | */ 13 | angular.module('MSLImageExplorerApp'). 14 | controller('TimelineCtrl', function ($scope, $log, $location, $routeParams, $timeout, $modal, MarsPhotosDBAccess, Blueimp) { 15 | var timelinePosition = $('.timeline').position(); 16 | $scope.isUpdatingPhotos = false; 17 | $scope.showDatePicker = true; 18 | $scope.photos = []; 19 | 20 | /* 21 | * Initializes instrument in the scope according to $routeParams. It 22 | * uses the default instrument if not specified. 23 | */ 24 | if ($routeParams.instrument) { 25 | $scope.instrument = $routeParams.instrument; 26 | } else { 27 | $scope.instrument = MarsPhotosDBAccess.defaultInstrument; 28 | } 29 | $scope.mission = MarsPhotosDBAccess.defaultMission; 30 | $scope.missionInstrument = $scope.mission + '+' + $scope.instrument; 31 | $scope.instrumentList = MarsPhotosDBAccess.instrumentList; 32 | 33 | /* 34 | * Parses time specified in $routeParams and sets it in $scope. 35 | */ 36 | if ($routeParams.time) { 37 | var time = parseInt($routeParams.time); 38 | if (typeof(time) === 'number' && time > 0) { 39 | $scope.time = time; 40 | } else { 41 | $log.error('Failed to parse time parameter:' + $routeParams.time); 42 | } 43 | } 44 | 45 | /** 46 | * Fetches more photos through MarsPhotos service and concatinates results 47 | * to $scope.photos. Called when the page is loaded and when the user 48 | * scrolls to the bottom of window. 49 | */ 50 | $scope.updatePhotos = function(){ 51 | $scope.isUpdatingPhotos = true; 52 | MarsPhotosDBAccess.queryWithDateIndex({ 53 | hashKey: $scope.missionInstrument, 54 | rangeKey: $scope.time, 55 | lastEvaluatedKey: $scope.lastEvaluatedKey 56 | }, function(error, data) { 57 | if (!error) { 58 | $scope.lastEvaluatedKey = data.LastEvaluatedKey; 59 | $scope.$apply(function(){ 60 | for (var index in data.Items) { 61 | var photo = data.Items[index]; 62 | /*jshint camelcase: false */ 63 | photo.time.creation_time = moment(photo.time.creation_timestamp_utc).format('MMMM Do YYYY, h:mm:ss a zz'); 64 | photo.time.received_in = moment(photo.time.received_timestamp_utc).from(photo.time.creation_timestamp_utc, true); 65 | $scope.photos.push(photo); 66 | } 67 | }); 68 | $scope.isUpdatingPhotos = false; 69 | } else { 70 | $log.error(error); 71 | } 72 | }); 73 | }; 74 | 75 | /** 76 | * Updates the location path and reloads the page. Called from the child 77 | * controller, SideMenuCtrl. 78 | */ 79 | $scope.reload = function(){ 80 | var path = '/timeline/' + $scope.instrument; 81 | if ($scope.time) { 82 | path += '/' + $scope.time; 83 | } 84 | $log.debug('Updating path to ' + path); 85 | $timeout(function(){ // We use $timeout() to avoid calling $scope.$apply() during another $scope.$apply() call 86 | $scope.$apply(function(){ 87 | $location.path(path); 88 | }); 89 | }); 90 | }; 91 | 92 | /** 93 | * Votes on the specified photo via MarsPhotos service. It updates the voting 94 | * cound of the photo on the successful voting. It shows an error dialog if 95 | * the user has already voted on the photo. 96 | */ 97 | $scope.vote = function(photo) { 98 | $log.debug('Voting on ' + photo.imageid); 99 | MarsPhotosDBAccess.voteOnPhoto( 100 | localStorage.getItem('userid'), 101 | photo, 102 | function(error, data) { 103 | if (!error) { 104 | $scope.$apply(function(){ 105 | photo.votes = data.Attributes.votes; 106 | }); 107 | } else { 108 | $modal.open({ 109 | templateUrl: 'views/partials/dialog.html', 110 | controller: 'DialogCtrl', 111 | resolve: { 112 | title: function() { 113 | return 'Error'; 114 | }, 115 | message: function() { 116 | return error; 117 | } 118 | } 119 | }); 120 | } 121 | }); 122 | }; 123 | 124 | /* 125 | * Handler called when the view content is loaded. The timelineAnimate function 126 | * and scroll handler need to be called after the view content is ready. 127 | */ 128 | $scope.$on('$viewContentLoaded', function() { 129 | $log.debug('Content loaded'); 130 | $scope.updatePhotos(); 131 | 132 | // Activates image gallery feature 133 | Blueimp.Gallery($('.timeline a.mars-photos'), $('#blueimp-gallery').data()); 134 | 135 | $(window).scroll(scrollHandler); 136 | }); 137 | 138 | /* 139 | * Handler called when the view is destroyed. The scroll handler needs to 140 | * be deregistered when moving to another view. 141 | */ 142 | $scope.$on('$destroy', function(){ 143 | $(window).off('scroll', scrollHandler); 144 | }); 145 | 146 | /** 147 | * Handler called when a scrolling event is fired. It checks if the bottom of the window 148 | * is reaching and calls updatePhotos() if so. 149 | */ 150 | var scrollHandler = function() { 151 | if (isBottomReaching() && !$scope.isUpdatingPhotos) { 152 | $log.debug('Fetching more photos'); 153 | $scope.updatePhotos(); 154 | } 155 | activateTimelineItemOnceWindowReached(); 156 | }; 157 | 158 | /** 159 | * Judges if the bottom of window is reaching. Used to decide if more photos 160 | * should be fetched upon a scrolling event or not. 161 | */ 162 | var isBottomReaching = function(){ 163 | return $(window).scrollTop() >= ($(document).height() - $(window).height() - timelinePosition.top); 164 | }; 165 | 166 | 167 | /** 168 | * Private function which triggers to display a time line item if the window 169 | * scrolls to the item. 170 | */ 171 | var activateTimelineItemOnceWindowReached = function() { 172 | var inactiveItems = $('.timeline-item:not(.active)'); 173 | if(inactiveItems.length > 0){ 174 | var item = inactiveItems.first(); 175 | var itemHead = timelinePosition.top + item.position().top + item.outerHeight() / 3; 176 | var windowBottom = $(window).scrollTop() + $(window).height(); 177 | if (windowBottom > itemHead) { 178 | item.addClass('active'); 179 | } 180 | } 181 | }; 182 | }); 183 | -------------------------------------------------------------------------------- /viewer/app/scripts/controllers/top-voted.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | global $ 4 | */ 5 | 6 | /** 7 | * @ngdoc function 8 | * @name MSLImageExplorerApp.controller:TopVotedCtrl 9 | * @description 10 | * # TopVotedCtrl 11 | * Controller of the MSLImageExplorerApp 12 | */ 13 | angular.module('MSLImageExplorerApp'). 14 | controller('TopVotedCtrl', function ($scope, $routeParams, $timeout, $location, $log, MarsPhotosDBAccess, Blueimp) { 15 | 16 | $scope.title = 'Top Voted Mars Images'; 17 | $scope.description = 'Photos taken by sorted by # of votes by viewers.'; 18 | $scope.photos = []; 19 | 20 | /* 21 | * Initializes instrument in the scope according to $routeParams. It 22 | * uses the default instrument if none specified. 23 | */ 24 | if ($routeParams.instrument) { 25 | $scope.instrument = $routeParams.instrument; 26 | } else { 27 | $scope.instrument = MarsPhotosDBAccess.defaultInstrument; 28 | } 29 | $scope.mission = MarsPhotosDBAccess.defaultMission; 30 | $scope.missionInstrument = $scope.mission + '+' + $scope.instrument; 31 | $scope.instrumentList = MarsPhotosDBAccess.instrumentList; 32 | 33 | /** 34 | * Updates the location path and reloads the page. Called from the child 35 | * controller, SideMenuCtrl. 36 | */ 37 | $scope.reload = function(){ 38 | var path = '/topVoted/' + $scope.instrument; 39 | $log.debug('Updating path to ' + path); 40 | $timeout(function(){ 41 | $scope.$apply(function(){ 42 | $location.path(path); 43 | }); 44 | }); 45 | }; 46 | 47 | /** 48 | * Fetches top voted photos via MarsPhotosDBAccess service and replaces 49 | * $scope.photos with the results. It also triggers Blueimp 50 | * image gallery to prepare for the slideshow. 51 | */ 52 | $scope.updatePhotos = function(){ 53 | MarsPhotosDBAccess.queryWithVoteIndex({ 54 | hashKey: $scope.missionInstrument, 55 | lastEvaluatedKey: $scope.lastEvaluatedKey 56 | }, function(error, data){ 57 | if (!error) { 58 | var photos = data.Items; 59 | $scope.lastEvaluatedKey = data.LastEvaluatedKey; 60 | $scope.$apply(function(){ 61 | $scope.photos = photos; 62 | }); 63 | } else { 64 | $log.error(error); 65 | } 66 | }); 67 | }; 68 | 69 | $scope.updatePhotos(); 70 | Blueimp.Gallery($('#links a'), $('#blueimp-gallery').data()); 71 | }); 72 | 73 | -------------------------------------------------------------------------------- /viewer/app/scripts/services/AWS.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* global AWS, DynamoDB */ 3 | /** 4 | * @ngdoc function 5 | * @name MSLImageExplorerApp.service:AWS 6 | * @description 7 | * # AWS 8 | * Service of the MSLImageExplorerApp. It initializes AWS SDK with providing credentials 9 | * either from Cognito Identity service or environment variables as specified in ENV service. 10 | * It initializes a DynamoDB client instance. The instance is exposed as AWS.dynamoDB. 11 | */ 12 | angular.module('MSLImageExplorerApp'). 13 | service('AWS', function ($log, ENV){ 14 | if (ENV.useCognitoIdentity) { // Uses Cognito Idenity to get AWS credentials 15 | // Needs to set us-east-1 as the default to get credentials by Cognito 16 | AWS.config.update({region: 'us-east-1'}); 17 | AWS.config.credentials = new AWS.CognitoIdentityCredentials({ 18 | AccountId: ENV.awsAccountId, 19 | IdentityPoolId: ENV.identityPoolId, 20 | RoleArn: ENV.unauthRoleArn 21 | }); 22 | 23 | AWS.config.credentials.getId(function(){ 24 | localStorage.setItem('userid', AWS.config.credentials.params.IdentityId); 25 | $log.info('AWS credentials initialized with Cognito Identity'); 26 | }); 27 | } else { // Uses environment variables to get AWS credentials. 28 | AWS.config.credentials = new AWS.Credentials({ 29 | accessKeyId: ENV.accessKeyId, 30 | secretAccessKey: ENV.secretAccessKey 31 | }); 32 | $log.info('AWS credentials initialized with environment variables'); 33 | $log.debug('AWS Accesss Key is ' + ENV.accessKeyId); 34 | localStorage.setItem('userid', ENV.userId); 35 | $log.debug('User ID is set to ' + localStorage.getItem('userid')); 36 | } 37 | 38 | /** 39 | * DynamoDB client object. 40 | */ 41 | AWS.dynamoDB = new DynamoDB( 42 | new AWS.DynamoDB({ 43 | region: ENV.dynamoDBRegion, 44 | endpoint: ENV.dynamoDBEndpoint 45 | })); 46 | 47 | return AWS; 48 | }); 49 | -------------------------------------------------------------------------------- /viewer/app/scripts/services/blueimp.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | global blueimp 4 | */ 5 | /** 6 | * @ngdoc function 7 | * @name MSLImageExplorerApp.service:Blueimp 8 | * @description 9 | * # Blueimp 10 | * Service of the MSLImageExplorerApp. Exposes blueimp Javascript object as a service. 11 | */ 12 | angular.module('MSLImageExplorerApp'). 13 | service('Blueimp', function (){ 14 | return blueimp; 15 | }); -------------------------------------------------------------------------------- /viewer/app/scripts/services/mars-photos.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /** 3 | * @ngdoc function 4 | * @name MSLImageExplorerApp.service:MarsPhotosDBAccess 5 | * @description 6 | * # MarsPhotosDBAccess 7 | * Service of the MSLImageExplorerApp. The service offers utility functions to 8 | * access DynamoDB tables. 9 | */ 10 | angular.module('MSLImageExplorerApp').service('MarsPhotosDBAccess', function ($log, ENV, AWS){ 11 | var MarsPhotosDBAccess = { 12 | /** 13 | * Queries photos table with the date index and give the result 14 | * to callback function. 15 | * @param {object} queryParams - The query parameters used in the query. The parameter 16 | * hashKey is required. The lastEvaluatedKey and rangeKey are optional. 17 | * @param {function} callback - The callback function used to return the result. 18 | */ 19 | queryWithDateIndex: function(queryParams, callback){ 20 | assertHashKey(queryParams); 21 | assertFunction(callback); 22 | 23 | var params = { 24 | TableName: ENV.photosTable, 25 | KeyConditions: [ 26 | AWS.dynamoDB.Condition('Mission+InstrumentID', 'EQ', queryParams.hashKey) 27 | ], 28 | IndexName: 'date-gsi', 29 | Limit: 5, 30 | ScanIndexForward: false 31 | }; 32 | if(hasLastEvaluatedKey(queryParams)) { 33 | params.ExclusiveStartKey = queryParams.lastEvaluatedKey; 34 | } 35 | if(hasRangeKey(queryParams)){ 36 | params.KeyConditions.push (AWS.dynamoDB.Condition('TimeStamp', 'LE', queryParams.rangeKey)); 37 | } 38 | logRequest('query', params); 39 | AWS.dynamoDB.query(params, callback); 40 | }, 41 | 42 | /** 43 | * Queries photos table with the vote index and give the result 44 | * to callback function. 45 | * @param {object} queryParams - The query parameters used in the query. The parameter 46 | * hashKey is required. The lastEvaluatedKey and rangeKey are optional. 47 | * @param {function} callback - The callback function used to return the result. 48 | */ 49 | queryWithVoteIndex: function(queryParams, callback) { 50 | assertHashKey(queryParams); 51 | assertFunction(callback); 52 | 53 | var params = { 54 | TableName: ENV.photosTable, 55 | KeyConditions: [ 56 | AWS.dynamoDB.Condition('Mission+InstrumentID', 'EQ', queryParams.hashKey) 57 | ], 58 | IndexName: 'vote-gsi', 59 | ScanIndexForward: false 60 | }; 61 | if(hasLastEvaluatedKey(queryParams)) { 62 | params.ExclusiveStartKey = queryParams.lastEvaluatedKey; 63 | } 64 | if(hasRangeKey(queryParams)){ 65 | params.KeyConditions.push (AWS.dynamoDB.Condition('votes', 'LE', queryParams.rangeKey)); 66 | } 67 | logRequest('query', params); 68 | AWS.dynamoDB.query(params, callback); 69 | }, 70 | 71 | /** 72 | * Queries user votes table and give the result to the callback function. 73 | * @param {object} queryParams - The query parameters used in the query. The parameter 74 | * hashKey is required. The lastEvaluatedKey is optional. 75 | * @param {function} callback - The callback function used to return the result. 76 | */ 77 | queryUserVotedPhotos: function(queryParams, callback) { 78 | assertHashKey(queryParams); 79 | assertFunction(callback); 80 | 81 | var params = { 82 | TableName: ENV.userVotesTable, 83 | KeyConditions: AWS.dynamoDB.Condition('userid', 'EQ', queryParams.hashKey), 84 | ScanIndexForward: false 85 | }; 86 | if(hasLastEvaluatedKey(queryParams)) { 87 | params.ExclusiveStartKey = queryParams.lastEvaluatedKey; 88 | } 89 | logRequest('query', params); 90 | AWS.dynamoDB.query(params, callback); 91 | }, 92 | 93 | /** 94 | * Votes on the specified photo and notes that in the votes table. 95 | * It uses conditional update 96 | * and succeeds only if the user votes on the photo for the first time. 97 | * @param {String} userid - The ID of the user who votes on the photo. 98 | * @param {Object} photo - The photo object to vote on. 99 | * @param {function} callback - The callback function used to return the result or error. 100 | */ 101 | voteOnPhoto: function(userid, photo, callback) { 102 | assertUserid(userid); 103 | assertPhoto(photo); 104 | assertFunction(callback); 105 | 106 | var item = {}; 107 | for (var key in photo){ 108 | // Skip thumbnail data and project other metadata 109 | if(key === 'data'){ 110 | continue; 111 | } 112 | item[key] = photo[key]; 113 | } 114 | item.userid = userid; 115 | var params = { 116 | TableName: ENV.userVotesTable, 117 | Item: item, 118 | Expected: AWS.dynamoDB.Condition('imageid', 'NULL') 119 | }; 120 | logRequest('putItem', params); 121 | AWS.dynamoDB.putItem(params, function(error) { 122 | if (!error) { 123 | $log.debug('Liked image successfully'); 124 | incrementVotesCount(photo.imageid, callback); 125 | } else { 126 | if(error.code === 'ConditionalCheckFailedException'){ 127 | callback('You have already voted on this image'); 128 | } else { 129 | $log.error(error); 130 | } 131 | } 132 | }); 133 | }, 134 | 135 | 136 | /** 137 | * List of instruments and its display names. 138 | */ 139 | instrumentList: { 140 | 'fcam': {id: 'fcam', name: 'Front Hazcam'}, 141 | 'ccam': {id: 'ccam', name: 'Chemcam RMI'}, 142 | 'mastcam_right': {id: 'mastcam_right', name: 'Right Mastcam'}, 143 | 'mastcam_left': {id: 'mastcam_left', name: 'Left Mastcam'}, 144 | 'mahli': {id: 'mahli', name: 'MAHLI'}, 145 | 'mardi': {id: 'mardi', name: 'MARDI'} 146 | }, 147 | 148 | /** 149 | * Default mission ID 150 | */ 151 | defaultMission: 'curiosity', 152 | 153 | /** 154 | * Default instrument ID 155 | */ 156 | defaultInstrument: 'fcam' 157 | }; 158 | 159 | /** 160 | * Increments the voting count in the photo table. It gets the updated 161 | * votes count in return. The function is meant to be used privately 162 | * in MarsPhoto service. 163 | * @param {String} imageid - The ID of the photo to increment vote count. 164 | * @param {function} callback - The callback function used to return the result or error. 165 | */ 166 | var incrementVotesCount = function(imageid, callback) { 167 | var params = { 168 | TableName: ENV.photosTable, 169 | Key: { imageid: imageid }, 170 | UpdateExpression: 'add votes :v', 171 | ExpressionAttributeValues: {':v': 1}, 172 | ReturnValues: 'UPDATED_NEW' 173 | }; 174 | 175 | logRequest('updateItem', params); 176 | AWS.dynamoDB.updateItem(params, callback); 177 | }; 178 | 179 | var assertHashKey = function(queryParams){ 180 | if(!queryParams || typeof queryParams.hashKey === 'undefined'){ 181 | throw 'A required parameter, hash key is missing.'; 182 | } 183 | }; 184 | 185 | var assertPhoto = function(photo) { 186 | if(! photo || typeof photo.imageid === 'undefined'){ 187 | throw 'Invalid object was given as a photo: ' + photo; 188 | } 189 | }; 190 | 191 | var assertUserid = function(userid) { 192 | if(typeof userid === 'undefined'){ 193 | throw 'User ID was invalid: ' + userid; 194 | } 195 | }; 196 | 197 | var assertFunction = function(callback){ 198 | if(typeof callback !== 'function'){ 199 | throw 'No valid callback function was given: ' + callback; 200 | } 201 | }; 202 | 203 | var hasLastEvaluatedKey = function(queryParams){ 204 | return queryParams && typeof queryParams.lastEvaluatedKey !== 'undefined'; 205 | }; 206 | 207 | var hasRangeKey = function(queryParams){ 208 | return queryParams && typeof queryParams.rangeKey !== 'undefined'; 209 | }; 210 | 211 | var logRequest = function(method, params){ 212 | $log.debug('Requesting DynamoDB to ' + method + ' with the following parameters'); 213 | $log.debug(params); 214 | }; 215 | 216 | return MarsPhotosDBAccess; 217 | }); 218 | -------------------------------------------------------------------------------- /viewer/app/styles/main.scss: -------------------------------------------------------------------------------- 1 | $icon-font-path: "../bower_components/bootstrap-sass-official/assets/fonts/bootstrap/"; 2 | // bower:scss 3 | @import "bootstrap-sass-official/assets/stylesheets/_bootstrap.scss"; 4 | // endbower 5 | 6 | .browsehappy { 7 | margin: 0.2em 0; 8 | background: #ccc; 9 | color: #000; 10 | padding: 0.2em 0; 11 | } 12 | 13 | /* Space out content a bit */ 14 | body { 15 | background: url('../images/mars.jpg') no-repeat center center fixed; 16 | -webkit-background-size: cover; 17 | -moz-background-size: cover; 18 | -o-background-size: cover; 19 | background-size: cover; 20 | color:#fff; 21 | background-color:#333; 22 | font-family: 'Open Sans',Arial,Helvetica,Sans-Serif; 23 | padding-top: 70px; 24 | } 25 | 26 | .clickable { 27 | cursor: pointer; 28 | } 29 | 30 | .in-parent { 31 | width: 100%; 32 | } 33 | 34 | /* Everything but the jumbotron gets side spacing for mobile first views */ 35 | .header, 36 | .marketing, 37 | .footer { 38 | padding-left: 15px; 39 | padding-right: 15px; 40 | } 41 | 42 | /* Custom page header */ 43 | .header { 44 | border-bottom: 1px solid #e5e5e5; 45 | 46 | /* Make the masthead heading the same height as the navigation */ 47 | h3 { 48 | margin-top: 0; 49 | margin-bottom: 0; 50 | line-height: 40px; 51 | padding-bottom: 19px; 52 | } 53 | } 54 | 55 | /* Custom page footer */ 56 | .footer { 57 | padding-top: 19px; 58 | color: #777; 59 | border-top: 1px solid #e5e5e5; 60 | } 61 | 62 | .container-narrow > hr { 63 | margin: 30px 0; 64 | } 65 | 66 | /* Main marketing message and sign up button */ 67 | .jumbotron { 68 | text-align: center; 69 | border-bottom: 1px solid #e5e5e5; 70 | background: url(../images/rover_jumbotron.jpg) no-repeat ; 71 | -webkit-background-size: cover; 72 | -moz-background-size: cover; 73 | -o-background-size: cover; 74 | background-size: cover; 75 | height: 480px; 76 | 77 | .btn { 78 | font-size: 21px; 79 | padding: 14px 24px; 80 | } 81 | } 82 | 83 | .title { 84 | text-align: center; 85 | color: grey; 86 | font-size: 32px; 87 | } 88 | 89 | .full-width { 90 | width: 100%; 91 | } 92 | 93 | /* Supporting marketing content */ 94 | .marketing { 95 | margin: 40px 0; 96 | 97 | p + h4 { 98 | margin-top: 28px; 99 | } 100 | } 101 | 102 | .fill-parent { 103 | width: 100%; 104 | } 105 | 106 | .rewind { 107 | position: fixed; 108 | top: 4em; 109 | left: 0px; 110 | text-decoration: none; 111 | color: #000000; 112 | background-color: rgba(235, 235, 235, 0.60); 113 | font-size: 12px; 114 | padding: 1em; 115 | transition: 0.5s linear all; 116 | } 117 | 118 | .fast-forward { 119 | position: fixed; 120 | bottom: 4em; 121 | left: 0px; 122 | text-decoration: none; 123 | color: #000000; 124 | background-color: rgba(235, 235, 235, 0.60); 125 | font-size: 12px; 126 | padding: 1em; 127 | transition: 0.5s linear all; 128 | } 129 | 130 | .copyright { 131 | position: fixed; 132 | bottom: 0px; 133 | right: 0px; 134 | text-decoration: none; 135 | color: rgba(235, 235, 235, 0.60); 136 | font-size: 12px; 137 | padding: 0em; 138 | } 139 | 140 | /* Responsive: Portrait tablets and up */ 141 | @media screen and (min-width: 768px) { 142 | .container { 143 | max-width: 730px; 144 | } 145 | 146 | /* Remove the padding we set earlier */ 147 | .header, 148 | .marketing, 149 | .footer { 150 | padding-left: 0; 151 | padding-right: 0; 152 | } 153 | /* Space out the masthead */ 154 | .header { 155 | margin-bottom: 30px; 156 | } 157 | /* Remove the bottom border on the jumbotron for visual effect */ 158 | .jumbotron { 159 | border-bottom: 0; 160 | } 161 | } 162 | 163 | .glyphicon-2x{ 164 | font-size: 40px; 165 | } 166 | .glyphicon-3x{ 167 | font-size: 60px; 168 | } 169 | 170 | .modal-dialog { 171 | color: black; 172 | } -------------------------------------------------------------------------------- /viewer/app/styles/timeline.scss: -------------------------------------------------------------------------------- 1 | .timeline { 2 | list-style: none; 3 | padding: 20px 0 20px; 4 | position: relative; 5 | } 6 | 7 | .timeline-item { 8 | opacity: 0; 9 | -webkit-transition: all 0.8s; 10 | -moz-transition: all 0.8s; 11 | transition: all 0.8s; 12 | } 13 | 14 | .timeline-item.active { 15 | opacity: 1.0; 16 | } 17 | 18 | .timeline:before { 19 | top: 0; 20 | bottom: 0; 21 | position: absolute; 22 | content: " "; 23 | width: 3px; 24 | background-color: #eeeeee; 25 | left: 50%; 26 | margin-left: -1.5px; 27 | } 28 | 29 | .timeline > li { 30 | margin-bottom: 20px; 31 | position: relative; 32 | } 33 | 34 | .timeline > li:before, 35 | .timeline > li:after { 36 | content: " "; 37 | display: table; 38 | } 39 | 40 | .timeline > li:after { 41 | clear: both; 42 | } 43 | 44 | .timeline > li:before, 45 | .timeline > li:after { 46 | content: " "; 47 | display: table; 48 | } 49 | 50 | .timeline > li:after { 51 | clear: both; 52 | } 53 | 54 | .timeline > li > .timeline-panel { 55 | width: 32%; 56 | margin-left: 15%; 57 | float: left; 58 | border: 1px solid #d4d4d4; 59 | border-radius: 2px; 60 | padding: 20px; 61 | background-color: #ffffff; 62 | color: black; 63 | position: relative; 64 | -webkit-box-shadow: 0 1px 6px rgba(255, 255, 255, 0.175); 65 | box-shadow: 0 1px 6px rgba(255, 255, 255, 0.175); 66 | } 67 | 68 | .timeline > li > .timeline-panel:before { 69 | position: absolute; 70 | top: 26px; 71 | right: -15px; 72 | display: inline-block; 73 | border-top: 15px solid transparent; 74 | border-left: 15px solid #ccc; 75 | border-right: 0 solid #ccc; 76 | border-bottom: 15px solid transparent; 77 | content: " "; 78 | } 79 | 80 | .timeline > li > .timeline-panel:after { 81 | position: absolute; 82 | top: 27px; 83 | right: -14px; 84 | display: inline-block; 85 | border-top: 14px solid transparent; 86 | border-left: 14px solid #fff; 87 | border-right: 0 solid #fff; 88 | border-bottom: 14px solid transparent; 89 | content: " "; 90 | } 91 | 92 | .timeline > li > .timeline-date { 93 | line-height: 50px; 94 | font-size: 1.4em; 95 | position: absolute; 96 | top: 16px; 97 | left: 50%; 98 | margin-left: 40px; 99 | } 100 | 101 | .timeline > li > .timeline-badge { 102 | color: #fff; 103 | width: 50px; 104 | height: 50px; 105 | line-height: 50px; 106 | font-size: 1.4em; 107 | text-align: center; 108 | position: absolute; 109 | top: 16px; 110 | left: 50%; 111 | margin-left: -25px; 112 | background-color: #999999; 113 | z-index: 100; 114 | border-top-right-radius: 50%; 115 | border-top-left-radius: 50%; 116 | border-bottom-right-radius: 50%; 117 | border-bottom-left-radius: 50%; 118 | } 119 | 120 | 121 | .timeline > li.timeline-inverted > .timeline-panel { 122 | float: right; 123 | margin-right: 15%; 124 | } 125 | 126 | .timeline > li.timeline-inverted > .timeline-panel:before { 127 | border-left-width: 0; 128 | border-right-width: 15px; 129 | left: -15px; 130 | right: auto; 131 | } 132 | 133 | .timeline > li.timeline-inverted > .timeline-panel:after { 134 | border-left-width: 0; 135 | border-right-width: 14px; 136 | left: -14px; 137 | right: auto; 138 | } 139 | 140 | .timeline > li.timeline-inverted > .timeline-date { 141 | line-height: 50px; 142 | font-size: 1.4em; 143 | position: absolute; 144 | text-align: right; 145 | top: 16px; 146 | left: 0%; 147 | margin-left: 0px; 148 | margin-right: 40px; 149 | right: 50%; 150 | } 151 | 152 | 153 | .timeline-badge.primary { 154 | background-color: #2e6da4 !important; 155 | } 156 | 157 | .timeline-badge.success { 158 | background-color: #3f903f !important; 159 | } 160 | 161 | .timeline-badge.warning { 162 | background-color: #f0ad4e !important; 163 | } 164 | 165 | .timeline-badge.danger { 166 | background-color: #d9534f !important; 167 | } 168 | 169 | .timeline-badge.info { 170 | background-color: #5bc0de !important; 171 | } 172 | 173 | .timeline-title { 174 | margin-top: 0; 175 | color: inherit; 176 | } 177 | 178 | .timeline-body > p, 179 | .timeline-body > ul { 180 | margin-bottom: 0; 181 | } 182 | 183 | .timeline-body > p + p { 184 | margin-top: 5px; 185 | } 186 | 187 | @media (max-width: 767px) { 188 | ul.timeline:before { 189 | left: 40px; 190 | } 191 | 192 | ul.timeline > li > .timeline-panel { 193 | width: calc(100% - 90px); 194 | width: -moz-calc(100% - 90px); 195 | width: -webkit-calc(100% - 90px); 196 | } 197 | 198 | ul.timeline > li > .timeline-badge { 199 | left: 15px; 200 | margin-left: 0; 201 | top: 16px; 202 | } 203 | 204 | ul.timeline > li > .timeline-panel { 205 | float: right; 206 | margin-left: 0%; 207 | } 208 | 209 | .timeline > li.timeline-inverted > .timeline-panel { 210 | float: right; 211 | margin-right: 0%; 212 | } 213 | 214 | 215 | ul.timeline > li > .timeline-panel:before { 216 | border-left-width: 0; 217 | border-right-width: 15px; 218 | left: -15px; 219 | right: auto; 220 | } 221 | 222 | ul.timeline > li > .timeline-panel:after { 223 | border-left-width: 0; 224 | border-right-width: 14px; 225 | left: -14px; 226 | right: auto; 227 | } 228 | } -------------------------------------------------------------------------------- /viewer/app/views/includes/head.jade: -------------------------------------------------------------------------------- 1 | title= title 2 | meta(charset='utf-8') 3 | // build:css(.) styles/vendor.css 4 | link(rel='stylesheet', href='bower_components/bootstrap/dist/css/bootstrap.min.css') 5 | link(rel='stylesheet', href='bower_components/blueimp-gallery/css/blueimp-gallery.css') 6 | link(rel='stylesheet', href='bower_components/blueimp-bootstrap-image-gallery/css/bootstrap-image-gallery.css') 7 | // bower:css 8 | link(rel='stylesheet', href='bower_components/font-awesome/css/font-awesome.min.css') 9 | // endbower 10 | // endbuild 11 | 12 | // build:css(.tmp) styles/main.css 13 | link(rel='stylesheet', href='styles/timeline.css') 14 | link(rel='stylesheet', href='styles/main.css') 15 | // endbuild -------------------------------------------------------------------------------- /viewer/app/views/layout.jade: -------------------------------------------------------------------------------- 1 | doctype html 2 | html(lang='en') 3 | head 4 | include includes/head 5 | 6 | body(ng-app="MSLImageExplorerApp") 7 | 8 | 9 | .container-fluid 10 | 11 | block content 12 | 13 | block footer 14 | 15 | // build:js(.) scripts/vendor.js 16 | // bower:js 17 | // endbower 18 | script(src='bower_components/blueimp-gallery/js/jquery.blueimp-gallery.min.js') 19 | script(src='bower_components/blueimp-bootstrap-image-gallery/js/bootstrap-image-gallery.js') 20 | script(src='bower_components/dynamodb-doc/dynamodb-doc.min.js') 21 | // endbuild 22 | 23 | script(src='scripts/app.js') 24 | script(src='scripts/config.js') 25 | script(src='scripts/controllers/timeline.js') 26 | script(src='scripts/controllers/top-voted.js') 27 | script(src='scripts/controllers/favorites.js') 28 | script(src='scripts/controllers/sidemenu.js') 29 | script(src='scripts/controllers/dialog.js') 30 | script(src='scripts/services/AWS.js') 31 | script(src='scripts/services/mars-photos.js') 32 | script(src='scripts/services/blueimp.js') 33 | 34 | block scripts 35 | 36 | -------------------------------------------------------------------------------- /viewer/app/views/partials/dialog.jade: -------------------------------------------------------------------------------- 1 | .modal-header 2 | h4 {{title}} 3 | .modal-body 4 | p {{message}} 5 | .modal-footer 6 | button.btn.btn-primary(ng-click="close()") OK 7 | -------------------------------------------------------------------------------- /viewer/app/views/partials/image-gallery.jade: -------------------------------------------------------------------------------- 1 | div(ng-controller="SideMenuCtrl" ng-init="initSidr()" 2 | ng-include="'views/partials/sidemenu.html'" ) 3 | 4 | 5 | .col-md-8.col-md-offset-2 6 | h1 {{title}} 7 | blockquote 8 | p {{description}} 9 | 10 | #links 11 | ul 12 | li(ng-repeat="photo in photos track by photo.imageid" style="list-style: none;") 13 | a(href="{{photo.url}}" title="{{photo.imageid}}" data-gallery) 14 | //img(style="width:500px;" ng-src="{{photo.url}}") 15 | img(style="width:500px;" ng-src="http://s3.amazonaws.com/aws-dynamodb-mars-json-demo/thumbnails/{{photo.imageid}}.jpg") 16 | h3.pull-right(ng-show="photo.votes > 0") {{photo.votes}} votes 17 | 18 | // The Bootstrap Image Gallery lightbox, should be a child element of the document body 19 | #blueimp-gallery.blueimp-gallery.blueimp-gallery-controls(data-use-bootstrap-modal="false") 20 | // The container for the modal slides 21 | .slides 22 | // Controls for the borderless lightbox 23 | h3.title 24 | a.prev 25 | a.next 26 | a.close 27 | a.play-pause 28 | ol.indicator 29 | -------------------------------------------------------------------------------- /viewer/app/views/partials/rover-detail.jade: -------------------------------------------------------------------------------- 1 | .modal-header 2 | h4 Instrument Selection 3 | .modal-body 4 | img.slide.in-parent(ng-src="/images/rover.jpg") 5 | .modal-footer 6 | -------------------------------------------------------------------------------- /viewer/app/views/partials/sidemenu.jade: -------------------------------------------------------------------------------- 1 | nav.navbar.navbar-inverse.navbar-fixed-top(role="navigation") 2 | .container-fluid 3 | .navbar-header 4 | a.navbar-brand(href="/#/") MSL Image Explorer 5 | ul.nav.navbar-nav.navbar-right 6 | li.active 7 | a#sidemenu.clickable(ng-click="openSidr()") Mission Control 8 | 9 | #sidr(ng-show="isSidrInitialized") 10 | ul 11 | li 12 | #close-sidemenu.clickable(ng-click="closeSidr()") 13 | span.pull-right × 14 | span Mission Control 15 | li 16 | a.sidemenu-link(href="/#/timeline/{{instrument}}/{{time}}") Timeline 17 | ul#datepicker(ng-show="showDatePicker") 18 | li 19 | span 20 | .input-append.date.datepicker(data-orientation="left" data-autoclose="true") 21 | input(type="text" data-orientation="left" data-autoclose="true") 22 | span.add-on 23 | i.icon-th 24 | 25 | li 26 | a.sidemenu-link(href="/#/topVoted/{{instrument}}") Top Voted 27 | li 28 | a.sidemenu-link(href="/#/favorites") My Favorites 29 | li 30 | span.clickable(ng-click="showRoverDetails()") Select Instrument 31 | ul 32 | li 33 | span(ng-hide="showInstrumentList") Current: {{instrumentList[instrument].name}} 34 | ul#instrumentList(ng-show="showInstrumentList") 35 | li(ng-repeat="instrument in instrumentList" ng-class="{active: isActive(instrument)}" 36 | ng-click="setInstrument(instrument.id, $event)") 37 | span.clickable {{instrument.name}} 38 | -------------------------------------------------------------------------------- /viewer/app/views/partials/timeline.jade: -------------------------------------------------------------------------------- 1 | div(ng-controller="SideMenuCtrl" ng-include="'views/partials/sidemenu.html'") 2 | 3 | ul.timeline 4 | li.timeline-item(ng-repeat="photo in photos" ng-class-even="'timeline-inverted'") 5 | .timeline-badge 6 | i.fa.fa-camera 7 | .timeline-date 8 | p {{photo.time.creation_time}} 9 | .timeline-panel 10 | .timeline-title 11 | h4 {{photo.imageid}} 12 | p {{photo.time.creation_time}} 13 | .timeline-body(id="{{photo.imageid}}") 14 | a.mars-photos(href="{{photo.url}}" title="{{photo.imageid}}" data-gallery) 15 | img.img-responsive.fill-parent(ng-src="{{photo.url}}") 16 | p.description 17 | | Image from the rover's {{instrumentList[photo.instrument].name}} during 18 | | mission {{photo.mission ? photo.mission : mission}}. It was taken at 19 | | {{photo.time.creation_time}} and received on Earth 20 | | {{photo.time.received_in}} later. 21 | br 22 | h4.pull-left # of votes: {{photo.votes ? photo.votes : 0}} 23 | a.clickable.pull-right(ng-click="vote(photo)") 24 | span.glyphicon.glyphicon-2x.glyphicon-thumbs-up 25 | 26 | // The Bootstrap Image Gallery lightbox, should be a child element of the document body 27 | #blueimp-gallery.blueimp-gallery.blueimp-gallery-controls(data-use-bootstrap-modal="false") 28 | // The container for the modal slides 29 | .slides 30 | // Controls for the borderless lightbox 31 | h3.title 32 | a.prev 33 | a.next 34 | a.close 35 | a.play-pause 36 | ol.indicator 37 | -------------------------------------------------------------------------------- /viewer/bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MSLImageExplorerApp", 3 | "version": "0.0.0", 4 | "dependencies": { 5 | "angular": "1.2.16", 6 | "json3": "~3.3.1", 7 | "es5-shim": "~3.1.0", 8 | "bootstrap-sass-official": "~3.2.0", 9 | "angular-resource": "1.2.16", 10 | "angular-sanitize": "1.2.16", 11 | "angular-touch": "1.2.16", 12 | "angular-route": "1.2.16", 13 | "font-awesome": "~4.1.0", 14 | "angular-bootstrap": "~0.11.0", 15 | "sidr": "~1.2.1", 16 | "bootstrap-datepicker": "~1.3.0", 17 | "blueimp-gallery": "~2.15.1", 18 | "blueimp-bootstrap-image-gallery": "~3.1.1", 19 | "ui.bootstrap": "~0.11.0", 20 | "moment": "~2.8.3", 21 | "aws-sdk": "~2.0.22", 22 | "dynamodb-doc": "git://github.com/awslabs/dynamodb-document-js-sdk.git#4ee5768d3ec3fc036122d999f9e6749bba923d8c" 23 | }, 24 | "devDependencies": { 25 | "angular-mocks": "1.2.16" 26 | }, 27 | "appPath": "app" 28 | } 29 | -------------------------------------------------------------------------------- /viewer/lib/mynconf.coffee: -------------------------------------------------------------------------------- 1 | nconf = require 'nconf' 2 | 3 | nconf.argv() 4 | .env() 5 | 6 | nconf.defaults 7 | DYNAMODB_ENDPOINT_DEV: 'http://localhost:9000/dynamodb/' 8 | DYNAMODB_REGION_DEV: 'us-east-1' 9 | USE_COGNITO_DEV: false 10 | DYNAMODB_ENDPOINT_TEST: 'http://localhost:8080/dynamodb/' 11 | DYNAMODB_REGION_TEST: 'us-east-1' 12 | USE_COGNITO_TEST: false 13 | DYNAMODB_ENDPOINT_PROD: 'http://dynamodb.us-east-1.amazonaws.com/' 14 | DYNAMODB_REGION_PROD: 'us-east-1' 15 | USE_COGNITO_PROD: true 16 | AWS_ACCOUNT_ID: 'DummyAWSAccountID' 17 | COGNITO_IDENTITY_POOL_ID: 'DummyCognitoIdenityPoolID' 18 | COGNITO_UNAUTH_ROLE_ARN: 'DummyCognitoUnauthRoleARN' 19 | TABLE_PHOTOS: 'marsDemoImages' 20 | TABLE_USER_VOTES: 'userVotes' 21 | TABLE_RESOURCES: 'marsDemoResources' 22 | READ_CAPACITY_PHOTOS: 1 23 | WRITE_CAPACITY_PHOTOS: 1 24 | 25 | module.exports = nconf 26 | -------------------------------------------------------------------------------- /viewer/lib/prepare_tables.coffee: -------------------------------------------------------------------------------- 1 | util = require('./util') 2 | nconf = require('./mynconf') 3 | 4 | tables = [ 5 | { 6 | TableName: nconf.get('TABLE_PHOTOS') 7 | AttributeDefinitions: [ 8 | { AttributeName: 'imageid', AttributeType: 'S' } 9 | { AttributeName: 'Mission+InstrumentID', AttributeType: 'S' } 10 | { AttributeName: 'TimeStamp', AttributeType: 'N' } 11 | { AttributeName: 'votes', AttributeType: 'N' } 12 | ] 13 | KeySchema: [ 14 | { AttributeName: 'imageid', KeyType: 'HASH' } 15 | ] 16 | GlobalSecondaryIndexes: [ 17 | { 18 | IndexName: 'date-gsi' 19 | KeySchema: [ 20 | {AttributeName: 'Mission+InstrumentID', KeyType: 'HASH' } 21 | {AttributeName: 'TimeStamp', KeyType: 'RANGE' } 22 | ] 23 | Projection: 24 | ProjectionType: 'INCLUDE' 25 | NonKeyAttributes: [ 'url', 'time', 'instrument', 'votes', 'mission' ] 26 | ProvisionedThroughput: 27 | ReadCapacityUnits: nconf.get('READ_CAPACITY_PHOTOS') 28 | WriteCapacityUnits: nconf.get('WRITE_CAPACITY_PHOTOS') 29 | } 30 | { 31 | IndexName: 'vote-gsi' 32 | KeySchema: [ 33 | {AttributeName: 'Mission+InstrumentID', KeyType: 'HASH' } 34 | {AttributeName: 'votes', KeyType: 'RANGE' } 35 | ] 36 | Projection: 37 | ProjectionType: 'INCLUDE' 38 | NonKeyAttributes: [ 'url', 'time', 'instrument', 'TimeStamp', 'mission' ] 39 | ProvisionedThroughput: 40 | ReadCapacityUnits: nconf.get('READ_CAPACITY_PHOTOS') 41 | WriteCapacityUnits: nconf.get('WRITE_CAPACITY_PHOTOS') 42 | } 43 | ] 44 | ProvisionedThroughput: 45 | ReadCapacityUnits: nconf.get('READ_CAPACITY_PHOTOS') 46 | WriteCapacityUnits: nconf.get('WRITE_CAPACITY_PHOTOS') 47 | } 48 | { 49 | TableName: nconf.get('TABLE_USER_VOTES') 50 | AttributeDefinitions: [ 51 | { AttributeName: 'userid', AttributeType: 'S' } 52 | { AttributeName: 'imageid', AttributeType: 'S' } 53 | ] 54 | KeySchema: [ 55 | { AttributeName: 'userid', KeyType: 'HASH' } 56 | { AttributeName: 'imageid', KeyType: 'RANGE' } 57 | ] 58 | ProvisionedThroughput: 59 | ReadCapacityUnits: nconf.get('READ_CAPACITY_PHOTOS') 60 | WriteCapacityUnits: nconf.get('WRITE_CAPACITY_PHOTOS') 61 | } 62 | { 63 | TableName: nconf.get('TABLE_RESOURCES') 64 | AttributeDefinitions: [ 65 | { AttributeName: 'resource', AttributeType: 'S' } 66 | ] 67 | KeySchema: [ 68 | { AttributeName: 'resource', KeyType: 'HASH' } 69 | ] 70 | ProvisionedThroughput: 71 | ReadCapacityUnits: nconf.get('READ_CAPACITY_PHOTOS') 72 | WriteCapacityUnits: nconf.get('WRITE_CAPACITY_PHOTOS') 73 | } 74 | ] 75 | 76 | for table in tables 77 | util.createTable(table, nconf.get('delete_table_if_exists') 78 | (error) -> 79 | console.error error 80 | (tableName) -> 81 | console.log "Table #{tableName} is ready" 82 | ) 83 | -------------------------------------------------------------------------------- /viewer/lib/random_votes.coffee: -------------------------------------------------------------------------------- 1 | util = require('./util') 2 | nconf = require('./mynconf') 3 | 4 | 5 | fetchPhotos = (lastEvaluatedKey, done) -> 6 | params = 7 | TableName: nconf.get('TABLE_PHOTOS') 8 | AttributesToGet: ['imageid'] 9 | 10 | if lastEvaluatedKey 11 | params.ExclusiveStartKey = lastEvaluatedKey 12 | 13 | util.dynamoDB.scan( 14 | params 15 | (error, data) -> 16 | unless error 17 | done(data) 18 | else 19 | console.error error 20 | ) 21 | 22 | voteRandom = (data) -> 23 | for photo in data.Items 24 | votes = Math.floor((Math.random() * 1000) + 1) 25 | console.log "Voting #{votes} on #{photo.imageid}" 26 | params = 27 | TableName: nconf.get('TABLE_PHOTOS') 28 | Key: 29 | imageid: photo.imageid 30 | AttributeUpdates: 31 | votes: 32 | Action: 'ADD' 33 | Value: votes 34 | 35 | util.dynamoDB.updateItem( 36 | params 37 | (error, data) -> 38 | console.error error if error 39 | ) 40 | if data.LastEvaluatedKey 41 | fetchPhotos(data.LastEvaluatedKey, voteRandom) 42 | 43 | 44 | 45 | 46 | 47 | fetchPhotos(null, voteRandom) 48 | -------------------------------------------------------------------------------- /viewer/lib/util.coffee: -------------------------------------------------------------------------------- 1 | AWS = require('aws-sdk') 2 | DOC = require('dynamodb-doc') 3 | nconf = require('./mynconf') 4 | 5 | AWS.config.credentials = new AWS.Credentials ( 6 | accessKeyId: nconf.get('AWS_ACCESS_KEY') 7 | secretAccessKey: nconf.get('AWS_SECRET_ACCESS_KEY') 8 | ) 9 | 10 | dynamoDB = new DOC.DynamoDB( 11 | new AWS.DynamoDB( 12 | endpoint: nconf.get('DYNAMODB_ENDPOINT') 13 | region: nconf.get('DYNAMODB_REGION') 14 | ) 15 | ) 16 | 17 | isReady = (tableName, callback) -> 18 | dynamoDB.describeTable({ TableName: tableName }, (error, data) -> 19 | unless error 20 | callback(data.Table.TableStatus == 'ACTIVE') 21 | else 22 | console.error(error.message) 23 | ) 24 | 25 | notExists = (tableName, callback) -> 26 | dynamoDB.describeTable({ TableName: tableName }, (error, data) -> 27 | if error and error.code == 'ResourceNotFoundException' 28 | callback(true) 29 | else 30 | callback(false) 31 | ) 32 | 33 | waitUntil = (args, cond, ready) -> 34 | repeat = -> waitUntil(args, cond, ready) 35 | cond(args, (conditionMet) -> 36 | if conditionMet 37 | console.log("Done") 38 | ready(args) 39 | else 40 | console.log("Waiting for operation to complete...") 41 | setTimeout(repeat, 200) 42 | ) 43 | 44 | createTable = (tableParams, deleteIfExists, err, done) -> 45 | if deleteIfExists 46 | deleteTableAndWaitUntilRemoved(tableParams.TableName, 47 | (error) -> 48 | console.error error 49 | () -> 50 | createTableAndWaitUntilReady(tableParams, err, done) 51 | ) 52 | else 53 | createTableAndWaitUntilReady(tableParams, err, done) 54 | 55 | createTableAndWaitUntilReady = (tableParams, err, done) -> 56 | console.log "creating table #{tableParams.TableName} on #{nconf.get('DYNAMODB_ENDPOINT')}" 57 | dynamoDB.createTable(tableParams, (error, data) -> 58 | unless error 59 | waitUntil(tableParams.TableName, isReady, done) 60 | else 61 | if error.code == 'ResourceInUseException' 62 | console.info "Table #{tableParams.TableName} already exists" 63 | waitUntil(tableParams.TableName, isReady, done) 64 | else 65 | console.error error.message 66 | err(error.message) if err 67 | ) 68 | 69 | deleteTableAndWaitUntilRemoved = (tableName, err, done) -> 70 | console.log("Deleting table #{tableName}") 71 | dynamoDB.deleteTable({TableName: tableName}, (error, data) -> 72 | unless error 73 | waitUntil(tableName, notExists, done) 74 | else 75 | if error.code == 'ResourceNotFoundException' 76 | waitUntil(tableName, notExists, done) 77 | else 78 | err(error) if err 79 | ) 80 | 81 | putItem = (tableName, item, err, done) -> 82 | params = 83 | TableName: tableName 84 | Item: item 85 | 86 | dynamoDB.putItem(params, (error, data) -> 87 | unless error 88 | done(item) if done 89 | else 90 | err(error) if err 91 | ) 92 | 93 | 94 | module.exports.createTable = createTable 95 | module.exports.putItem = putItem 96 | module.exports.dynamoDB = dynamoDB 97 | -------------------------------------------------------------------------------- /viewer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MSLImageExplorerApp", 3 | "version": "0.1.0", 4 | "dependencies": {}, 5 | "devDependencies": { 6 | "aws-sdk": "^2.0.22", 7 | "dynamodb-doc": "git://github.com/awslabs/dynamodb-document-js-sdk.git#4ee5768d3ec3fc036122d999f9e6749bba923d8c", 8 | "coffee-script": "^1.8.0", 9 | "grunt": "^0.4.1", 10 | "grunt-autoprefixer": "^0.7.2", 11 | "grunt-bg-shell": "^2.3.1", 12 | "grunt-concurrent": "^0.5.0", 13 | "grunt-connect-proxy": "^0.1.11", 14 | "grunt-contrib-clean": "^0.5.0", 15 | "grunt-contrib-compass": "^0.7.2", 16 | "grunt-contrib-concat": "^0.4.0", 17 | "grunt-contrib-connect": "^0.7.1", 18 | "grunt-contrib-copy": "^0.5.0", 19 | "grunt-contrib-cssmin": "^0.9.0", 20 | "grunt-contrib-htmlmin": "^0.3.0", 21 | "grunt-contrib-imagemin": "^0.7.0", 22 | "grunt-contrib-jade": "^0.12.0", 23 | "grunt-contrib-jshint": "^0.10.0", 24 | "grunt-contrib-uglify": "^0.4.0", 25 | "grunt-contrib-watch": "^0.6.1", 26 | "grunt-curl": "^2.0.2", 27 | "grunt-filerev": "^0.2.1", 28 | "grunt-google-cdn": "^0.4.0", 29 | "grunt-if-missing": "^1.0.0", 30 | "grunt-karma": "~0.8.3", 31 | "grunt-newer": "^0.7.0", 32 | "grunt-ng-annotate": "^0.4.0", 33 | "grunt-ng-constant": "^1.0.0", 34 | "grunt-svgmin": "^0.4.0", 35 | "grunt-tar.gz": "0.0.3", 36 | "grunt-usemin": "^2.1.1", 37 | "grunt-wiredep": "^1.7.0", 38 | "jshint-stylish": "^0.2.0", 39 | "karma": "~0.12.21", 40 | "karma-jasmine": "^0.2.0", 41 | "karma-ng-html2js-preprocessor": "^0.1.0", 42 | "karma-phantomjs-launcher": "~0.1.4", 43 | "load-grunt-tasks": "^0.4.0", 44 | "nconf": "^0.6.9", 45 | "time-grunt": "^0.3.1" 46 | }, 47 | "repository": { 48 | "type": "git", 49 | "url": "https://github.com/awslabs/aws-dynamodb-mars-json-demo" 50 | }, 51 | "engines": { 52 | "node": ">=0.10.0" 53 | }, 54 | "scripts": { 55 | "start": "grunt serve", 56 | "test": "grunt test" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /viewer/test/.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node": true, 3 | "browser": true, 4 | "esnext": true, 5 | "bitwise": true, 6 | "camelcase": true, 7 | "curly": true, 8 | "eqeqeq": true, 9 | "immed": true, 10 | "indent": 2, 11 | "latedef": true, 12 | "newcap": true, 13 | "noarg": true, 14 | "quotmark": "single", 15 | "regexp": true, 16 | "undef": true, 17 | "unused": true, 18 | "strict": true, 19 | "trailing": true, 20 | "smarttabs": true, 21 | "globals": { 22 | "after": false, 23 | "afterEach": false, 24 | "angular": false, 25 | "before": false, 26 | "beforeEach": false, 27 | "browser": false, 28 | "describe": false, 29 | "expect": false, 30 | "inject": false, 31 | "it": false, 32 | "jasmine": false, 33 | "spyOn": false 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /viewer/test/karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration 2 | // http://karma-runner.github.io/0.12/config/configuration-file.html 3 | // Generated on 2014-08-19 using 4 | // generator-karma 0.8.3 5 | 6 | module.exports = function(config) { 7 | 'use strict'; 8 | 9 | config.set({ 10 | // enable / disable watching file and executing tests whenever any file changes 11 | autoWatch: true, 12 | 13 | // base path, that will be used to resolve files and exclude 14 | basePath: '../', 15 | 16 | // testing framework to use (jasmine/mocha/qunit/...) 17 | frameworks: ['jasmine'], 18 | 19 | preprocessors: { 20 | '**/*.html': ['ng-html2js'] 21 | }, 22 | 23 | // list of files / patterns to load in the browser 24 | files: [ 25 | // bower:js 26 | 'bower_components/jquery/dist/jquery.js', 27 | 'bower_components/es5-shim/es5-shim.js', 28 | 'bower_components/angular/angular.js', 29 | 'bower_components/json3/lib/json3.js', 30 | 'bower_components/angular-resource/angular-resource.js', 31 | 'bower_components/angular-sanitize/angular-sanitize.js', 32 | 'bower_components/angular-touch/angular-touch.js', 33 | 'bower_components/angular-route/angular-route.js', 34 | 'bower_components/angular-bootstrap/ui-bootstrap-tpls.js', 35 | 'bower_components/sidr/jquery.sidr.min.js', 36 | 'bower_components/bootstrap/dist/js/bootstrap.js', 37 | 'bower_components/bootstrap-datepicker/js/bootstrap-datepicker.js', 38 | 'bower_components/blueimp-gallery/js/blueimp-helper.js', 39 | 'bower_components/blueimp-gallery/js/blueimp-gallery.js', 40 | 'bower_components/blueimp-gallery/js/blueimp-gallery-fullscreen.js', 41 | 'bower_components/blueimp-gallery/js/blueimp-gallery-indicator.js', 42 | 'bower_components/blueimp-gallery/js/blueimp-gallery-video.js', 43 | 'bower_components/blueimp-gallery/js/blueimp-gallery-vimeo.js', 44 | 'bower_components/blueimp-gallery/js/blueimp-gallery-youtube.js', 45 | 'bower_components/blueimp-bootstrap-image-gallery/js/bootstrap-image-gallery.js', 46 | 'bower_components/moment/moment.js', 47 | 'bower_components/aws-sdk/dist/aws-sdk.js', 48 | 'bower_components/angular-mocks/angular-mocks.js', 49 | // endbower 50 | 'app/scripts/**/*.js', 51 | '.tmp/scripts/**/*.js', 52 | 'test/spec/**/*.js', 53 | 'dist/**/*.html' 54 | ], 55 | 56 | // list of files / patterns to exclude 57 | exclude: [ 58 | 'node_modules/**/*.html', 59 | 'bower_components/**/*.html' 60 | ], 61 | 62 | // web server port 63 | port: 8080, 64 | 65 | // Start these browsers, currently available: 66 | // - Chrome 67 | // - ChromeCanary 68 | // - Firefox 69 | // - Opera 70 | // - Safari (only Mac) 71 | // - PhantomJS 72 | // - IE (only Windows) 73 | browsers: [ 74 | 'PhantomJS' 75 | ], 76 | 77 | // Which plugins to enable 78 | plugins: [ 79 | 'karma-phantomjs-launcher', 80 | 'karma-ng-html2js-preprocessor', 81 | 'karma-jasmine' 82 | ], 83 | 84 | // Continuous Integration mode 85 | // if true, it capture browsers, run tests and exit 86 | singleRun: false, 87 | 88 | colors: true, 89 | 90 | // level of logging 91 | // possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG 92 | logLevel: config.LOG_INFO, 93 | 94 | proxies: { 95 | '/dynamodb/': 'http://localhost:8000/' 96 | }, 97 | 98 | ngHtml2JsPreprocessor: { 99 | cacheIdFromPath: function(filepath) { 100 | filepath = filepath.replace(/^dist\//, ''); 101 | filepath = filepath.replace(/^app\//, ''); 102 | return filepath; 103 | } 104 | } 105 | }); 106 | }; 107 | -------------------------------------------------------------------------------- /viewer/test/spec/controllers/favorites.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Controller: FavoritesCtrl', function () { 4 | 5 | // load the controller's module 6 | beforeEach(module('MSLImageExplorerApp')); 7 | 8 | 9 | var FavoritesCtrl, scope; 10 | 11 | var data = { 12 | Items: [ 13 | {imageid: '123'}, 14 | {imageid: '456'} 15 | ] 16 | }; 17 | 18 | var marsPhotos = { 19 | queryUserVotedPhotos: function(queryParams, callback){ 20 | callback(null, data); 21 | }, 22 | getThumbnails: function(photos, callback) { 23 | callback(); 24 | } 25 | }; 26 | 27 | var Blueimp = { 28 | Gallery: function() {} 29 | }; 30 | 31 | // Initialize the controller and a mock scope 32 | beforeEach(inject(function ($controller, $rootScope, $log) { 33 | scope = $rootScope.$new(); 34 | FavoritesCtrl = $controller('FavoritesCtrl', { 35 | $scope: scope, 36 | $log: $log, 37 | MarsPhotosDBAccess: marsPhotos, 38 | Blueimp: Blueimp 39 | }); 40 | })); 41 | 42 | it('should attach a list of user voted photos to the scope', function () { 43 | expect(scope.photos.length).toBe(2); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /viewer/test/spec/controllers/sidemenu.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Controller: SideMenuCtrl', function () { 4 | 5 | // load the controller's module 6 | beforeEach(module('MSLImageExplorerApp')); 7 | 8 | var SideMenuCtrl, scope; 9 | 10 | var modal = { 11 | open: function(){ 12 | var modalInstance = { 13 | shown: true, 14 | dismiss: function(){ 15 | modalInstance.shown = false; 16 | } 17 | }; 18 | console.log(modalInstance); 19 | return modalInstance; 20 | } 21 | }; 22 | 23 | var marsPhotos = { 24 | instrumentList: { 25 | 'fcam': {id: 'fcam', name: 'Chemcam RMI'}, 26 | 'mastcam_right': {id: 'mastcam_right', name: 'Right Mastcam'}, 27 | 'mastcam_left': {id: 'mastcam_left', name: 'Left Mastcam'}, 28 | 'mahli': {id: 'mahli', name: 'MAHLI'} 29 | } 30 | }; 31 | 32 | var parentReloaded = false; 33 | 34 | // Initialize the controller and a mock scope 35 | beforeEach(function(){ 36 | inject(function ($controller, $rootScope, $log) { 37 | scope = $rootScope.$new(); 38 | scope.$parent.reload = function(){ 39 | parentReloaded = true; 40 | }; 41 | 42 | SideMenuCtrl = $controller('SideMenuCtrl', { 43 | $scope: scope, 44 | $log: $log, 45 | $modal: modal, 46 | MarsPhotosDBAccess: marsPhotos 47 | }); 48 | }); 49 | }); 50 | 51 | it('should initialize the sidemenu at the first time the user opens', function () { 52 | expect(scope.isSidrInitialized).toBe(false); 53 | scope.openSidr(); 54 | expect(scope.isSidrInitialized).toBe(true); 55 | }); 56 | 57 | it('should show rover details modal window when asked', function(){ 58 | scope.showRoverDetails(); 59 | expect(scope.modalInstance.shown).toBe(true); 60 | expect(scope.showInstrumentList).toBe(true); 61 | }); 62 | 63 | it('should set instrument to its parent view, dismiss modal window and reload its parent view', function(){ 64 | scope.showRoverDetails(); 65 | scope.setInstrument('fcam', {}); 66 | expect(scope.$parent.instrument).toEqual('fcam'); 67 | expect(scope.modalInstance.shown).toBe(false); 68 | expect(parentReloaded).toBe(true); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /viewer/test/spec/controllers/timeline.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Controller: TimelineCtrl', function () { 4 | 5 | // load the controller's module 6 | beforeEach(module('MSLImageExplorerApp')); 7 | 8 | var TimelineCtrl, scope, log, controller, timeout, location, controllerArgs, path; 9 | 10 | var data = { 11 | /*jshint camelcase: false */ 12 | Items: [ 13 | {imageid: '123', time: {creation_timestamp_utc: 0, received_timestamp_utc: 0}}, 14 | {imageid: '456', time: {creation_timestamp_utc: 0, received_timestamp_utc: 0}} 15 | ] 16 | }; 17 | 18 | var marsPhotos = { 19 | queryWithDateIndex: function(queryParams, callback){ 20 | setTimeout(function(){ 21 | data.LastEvaluatedKey = { 22 | 'Mission+InstrumentID': queryParams.hashKey 23 | }; 24 | callback(null, data); 25 | }, 5); 26 | }, 27 | voteOnPhoto: function(userid, photo, callback){ 28 | if(photo.imageid === 'already_voted'){ 29 | callback('You have already voted'); 30 | } else { 31 | callback(null, {Attributes: {votes: photo.votes + 1}}); 32 | } 33 | }, 34 | getThumbnails: function(photos, callback) { 35 | callback(); 36 | }, 37 | defaultInstrument: 'fcam', 38 | defaultMission: 'curiosity' 39 | }; 40 | 41 | beforeEach(function(){ 42 | inject(function ($controller, $rootScope, $log) { 43 | $log.debug = function(msg) { console.log(msg); }; 44 | $log.error = function(msg) { console.error(msg); }; 45 | scope = $rootScope.$new(); 46 | log = $log; 47 | location = { 48 | path: function(_path){ 49 | path = _path; 50 | } 51 | }; 52 | controller = $controller; 53 | timeout = function(callback) { 54 | callback(); 55 | }; 56 | 57 | controllerArgs = { 58 | $scope: scope, 59 | $log: log, 60 | $routeParams: {}, 61 | $timeout: timeout, 62 | $location: location, 63 | MarsPhotosDBAccess: marsPhotos 64 | }; 65 | TimelineCtrl = controller('TimelineCtrl', controllerArgs); 66 | }); 67 | }); 68 | 69 | it('should not be updating photos just after loading page', function(){ 70 | expect(scope.isUpdatingPhotos).toBe(false); 71 | expect(scope.photos.length).toBe(0); 72 | }); 73 | 74 | it('should show date picker on the mission control', function(){ 75 | expect(scope.showDatePicker).toBe(true); 76 | }); 77 | 78 | it('should attach a list of timeline photos to the scope once updatePhotos() is called', function(done){ 79 | scope.updatePhotos(); 80 | setTimeout(function(){ 81 | expect(scope.photos.length).toBe(2); 82 | done(); 83 | }, 10); 84 | }); 85 | 86 | 87 | it('should set isUpdatingPhotos flag while loading photos', function(done){ 88 | scope.updatePhotos(); 89 | expect(scope.isUpdatingPhotos).toBe(true); 90 | setTimeout(function(){ 91 | expect(scope.isUpdatingPhotos).toBe(false); 92 | done(); 93 | }, 10); 94 | }); 95 | 96 | it('should set instrument to default if it is not specified in routeParams', function (){ 97 | expect(scope.instrument).toBe(marsPhotos.defaultInstrument); 98 | expect(scope.missionInstrument).toBe(marsPhotos.defaultMission + '+' + marsPhotos.defaultInstrument); 99 | }); 100 | 101 | it('should set instrument to the one specified in routeParams', function (){ 102 | controllerArgs.$routeParams = {instrument: 'mastcam_right'}; 103 | TimelineCtrl = controller('TimelineCtrl', controllerArgs); 104 | expect(scope.instrument).toBe('mastcam_right'); 105 | expect(scope.missionInstrument).toBe(marsPhotos.defaultMission + '+mastcam_right'); 106 | }); 107 | 108 | it('should set time to if specified in routeParams', function (){ 109 | var time = new Date().getTime(); 110 | controllerArgs.$routeParams = {time: time}; 111 | TimelineCtrl = controller('TimelineCtrl', controllerArgs); 112 | expect(scope.time).toBe(time); 113 | }); 114 | 115 | it('should not set time if route param is not a valid integer', function (){ 116 | var time = 'not_integer'; 117 | controllerArgs.$routeParams = {time: time}; 118 | TimelineCtrl = controller('TimelineCtrl', controllerArgs); 119 | expect(scope.time).toBe(undefined); 120 | }); 121 | 122 | it('should set a function to reload', function(){ 123 | expect(typeof(scope.reload)).toBe('function'); 124 | }); 125 | 126 | it('should set path to /timeline/ + instrument ID when it is reloaded', function(){ 127 | scope.instrument = 'mastcam_left'; 128 | scope.reload(); 129 | expect(path).toBe('/timeline/mastcam_left'); 130 | }); 131 | 132 | it('should set path to /timeline/instrumentID/time if time is specified and reload is called', function(){ 133 | scope.instrument = 'mastcam_left'; 134 | var time = new Date().getTime(); 135 | scope.time = time; 136 | scope.reload(); 137 | expect(path).toBe('/timeline/mastcam_left/' + time); 138 | }); 139 | 140 | it('should update photos and set lastEvaluatedKey', function(done){ 141 | scope.missionInstrument = marsPhotos.defaultMission + '+mastcam_left'; 142 | scope.updatePhotos(); 143 | setTimeout(function(){ 144 | expect(scope.lastEvaluatedKey['Mission+InstrumentID']).toBe(marsPhotos.defaultMission + '+mastcam_left'); 145 | done(); 146 | }, 10); 147 | }); 148 | 149 | it('should increase # of votes for a photo which is not yet voted by the user', function(){ 150 | var photo = { 151 | imageid: 'not_voted_yet', 152 | votes: 1000 153 | }; 154 | scope.vote(photo); 155 | expect(photo.votes).toBe(1001); 156 | }); 157 | 158 | it('should not increase # of votes for a photo which is already voted by the user', function(){ 159 | var photo = { 160 | imageid: 'already_voted', 161 | votes: 1000 162 | }; 163 | scope.vote(photo); 164 | expect(photo.votes).toBe(1000); 165 | }); 166 | 167 | }); 168 | -------------------------------------------------------------------------------- /viewer/test/spec/controllers/top-voted.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('Controller: TopVotedCtrl', function () { 4 | 5 | // load the controller's module 6 | beforeEach(module('MSLImageExplorerApp')); 7 | 8 | var TopVotedCtrl, scope, log, controller, timeout, location, controllerArgs, path; 9 | 10 | var data = { 11 | Items: [ 12 | {imageid: '123'}, 13 | {imageid: '456'} 14 | ] 15 | }; 16 | 17 | var marsPhotos = { 18 | queryWithVoteIndex: function(queryParams, callback){ 19 | data.LastEvaluatedKey = { 20 | 'Mission+InstrumentID': queryParams.hashKey, 21 | 'votes': 100 22 | }; 23 | callback(null, data); 24 | }, 25 | getThumbnails: function(photos, callback){ 26 | callback(); 27 | }, 28 | defaultInstrument: 'fcam', 29 | defaultMission: 'curiosity' 30 | }; 31 | 32 | 33 | var Blueimp = { 34 | Gallery: function() {} 35 | }; 36 | 37 | beforeEach(function(){ 38 | inject(function ($controller, $rootScope, $log) { 39 | $log.debug = function(msg) { console.log(msg); }; 40 | scope = $rootScope.$new(); 41 | log = $log; 42 | location = { 43 | path: function(_path){ 44 | path = _path; 45 | } 46 | }; 47 | controller = $controller; 48 | timeout = function(callback) { 49 | callback(); 50 | }; 51 | 52 | controllerArgs = { 53 | $scope: scope, 54 | $log: log, 55 | $routeParams: {}, 56 | $timeout: timeout, 57 | $location: location, 58 | MarsPhotosDBAccess: marsPhotos, 59 | Blueimp: Blueimp 60 | }; 61 | TopVotedCtrl = controller('TopVotedCtrl', controllerArgs); 62 | }); 63 | }); 64 | 65 | it('should attach a list of user voted photos to the scope', function(){ 66 | expect(scope.photos.length).toBe(2); 67 | }); 68 | 69 | it('should set instrument to default if it is not specified in routeParams', function (){ 70 | expect(scope.instrument).toBe(marsPhotos.defaultInstrument); 71 | expect(scope.missionInstrument).toBe(marsPhotos.defaultMission + '+' + marsPhotos.defaultInstrument); 72 | }); 73 | 74 | it('should set instrument to the one specified in routeParams', function (){ 75 | controllerArgs.$routeParams = {instrument: 'mastcam_right'}; 76 | TopVotedCtrl = controller('TopVotedCtrl', controllerArgs); 77 | expect(scope.instrument).toBe('mastcam_right'); 78 | expect(scope.missionInstrument).toBe(marsPhotos.defaultMission + '+mastcam_right'); 79 | }); 80 | 81 | it('should set a function to reload', function(){ 82 | expect(typeof(scope.reload)).toBe('function'); 83 | }); 84 | 85 | it('should set path to /topVoted/ + instrument ID when it is reloaded', function(){ 86 | scope.instrument = 'mastcam_left'; 87 | scope.reload(); 88 | expect(path).toBe('/topVoted/mastcam_left'); 89 | }); 90 | 91 | it('should update photos and set lastEvaluatedKey', function(){ 92 | scope.missionInstrument = marsPhotos.defaultMission + '+mastcam_left'; 93 | scope.updatePhotos(); 94 | expect(scope.lastEvaluatedKey['Mission+InstrumentID']).toBe(marsPhotos.defaultMission + '+mastcam_left'); 95 | }); 96 | }); 97 | -------------------------------------------------------------------------------- /viewer/test/spec/views/image-gallery.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('View: ImageGallery', function() { 4 | beforeEach(module('MSLImageExplorerApp')); 5 | beforeEach(module('views/partials/image-gallery.html')); 6 | beforeEach(module('views/partials/sidemenu.html')); 7 | 8 | var FavoritesCtrl, scope, view; 9 | 10 | var data = { 11 | Items: [ 12 | {imageid: '123'}, 13 | {imageid: '456'} 14 | ] 15 | }; 16 | 17 | var marsPhotos = { 18 | queryUserVotedPhotos: function(queryParams, callback){ 19 | callback(null, data); 20 | }, 21 | getThumbnails: function(photos, callback) { 22 | callback(); 23 | } 24 | }; 25 | var Blueimp = { 26 | Gallery: function() {} 27 | }; 28 | 29 | // Initialize the controller and a mock scope 30 | beforeEach(inject(function ($templateCache, $compile, $controller, $rootScope, $log) { 31 | var html = $templateCache.get('views/partials/image-gallery.html'); 32 | scope = $rootScope.$new(); 33 | 34 | FavoritesCtrl = $controller('FavoritesCtrl', { 35 | $scope: scope, 36 | $log: $log, 37 | MarsPhotosDBAccess: marsPhotos, 38 | Blueimp: Blueimp 39 | }); 40 | view = $compile(angular.element(html))(scope); 41 | scope.$digest(); 42 | })); 43 | 44 | it('should show mars images the user liked', function() { 45 | expect(view.find('h1').text()).toEqual('Mars Images You Liked'); 46 | }); 47 | 48 | it('should show list of favorite images', function() { 49 | expect(view.find('#links img').length).toBe(2); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /viewer/test/spec/views/sidemenu.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('View: Sidemenu', function() { 4 | beforeEach(module('MSLImageExplorerApp')); 5 | beforeEach(module('views/partials/sidemenu.html')); 6 | 7 | var Ctrl, scope, view; 8 | 9 | // Initialize the controller and a mock scope 10 | beforeEach(inject(function ($templateCache, $compile, $controller, $rootScope, $modal, $log) { 11 | var html = $templateCache.get('views/partials/sidemenu.html'); 12 | scope = $rootScope.$new(); 13 | 14 | Ctrl = $controller('SideMenuCtrl', { 15 | $scope: scope, 16 | $log: $log, 17 | $modal: $modal 18 | }); 19 | view = $compile(angular.element(html))(scope); 20 | scope.$digest(); 21 | })); 22 | 23 | 24 | it('should have 3 side menu links', function(){ 25 | expect(view.find('.sidemenu-link').length).toBe(3); 26 | }); 27 | 28 | it('should have 4 instruments in the instrument list', function() { 29 | expect(view.find('#instrumentList li').length).toBe(4); 30 | }); 31 | 32 | }); 33 | 34 | 35 | -------------------------------------------------------------------------------- /viewer/test/spec/views/timeline.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | describe('View: Timeline', function() { 4 | beforeEach(module('MSLImageExplorerApp')); 5 | beforeEach(module('views/partials/timeline.html')); 6 | beforeEach(module('views/partials/sidemenu.html')); 7 | 8 | var Ctrl, scope, view; 9 | 10 | var data = { 11 | Items: [ 12 | /*jshint camelcase: false */ 13 | {imageid: '123', instrument: 'fcam', time: {creation_timestamp_utc: 0, received_timestamp_utc: 0}}, 14 | {imageid: '456', instrument: 'fcam', votes: 290, time: {creation_timestamp_utc: 0, received_timestamp_utc: 0}} 15 | ] 16 | }; 17 | 18 | var marsPhotos = { 19 | queryWithDateIndex: function(queryParams, callback){ 20 | callback(null, data); 21 | }, 22 | getThumbnails: function(photos, callback) { 23 | callback(); 24 | }, 25 | instrumentList: { 26 | fcam: {id: 'fcam', name: 'Chemcam RMI'} 27 | } 28 | }; 29 | 30 | // Initialize the controller and a mock scope 31 | beforeEach(inject(function ($templateCache, $compile, $controller, $rootScope, $modal, $log) { 32 | var html = $templateCache.get('views/partials/timeline.html'); 33 | scope = $rootScope.$new(); 34 | 35 | Ctrl = $controller('TimelineCtrl', { 36 | $scope: scope, 37 | $log: $log, 38 | MarsPhotosDBAccess: marsPhotos 39 | }); 40 | 41 | view = $compile(angular.element(html))(scope); 42 | scope.$digest(); 43 | scope.updatePhotos(); 44 | })); 45 | 46 | it('should show 2 images under timeline content div', function(){ 47 | expect(view.find('.timeline-item .img-responsive').length).toBe(2); 48 | }); 49 | 50 | it('should give imageid as the id for the corresponding panel div', function(){ 51 | expect(view.find('#123').length).toBe(1); 52 | expect(view.find('#456').length).toBe(1); 53 | }); 54 | 55 | it('should show # of votes on the panel if defined', function(){ 56 | expect(view.find('#456 h4').text()).toBe('# of votes: 290'); 57 | }); 58 | 59 | it('should show 0 if # of votes is not set in the photo object', function(){ 60 | expect(view.find('#123 h4').text()).toBe('# of votes: 0'); 61 | }); 62 | 63 | it('should contain instrument name in the photo description', function(){ 64 | expect(view.find('#123 .description').text()).toContain('Chemcam RMI'); 65 | }); 66 | 67 | }); 68 | 69 | 70 | --------------------------------------------------------------------------------