├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── _config.yml ├── api ├── Dockerfile ├── checkstyle-suppressions.xml ├── config │ ├── log4j.properties │ └── soundwaveapi.properties ├── pom.xml ├── scripts │ └── run_in_container.sh ├── soundwave-api-dev.yml ├── soundwave-api-prod.yml └── src │ ├── main │ ├── assembly │ │ └── assembly.xml │ ├── java │ │ └── com │ │ │ └── pinterest │ │ │ └── soundwave │ │ │ ├── ApiApplication.java │ │ │ ├── ApiConfiguration.java │ │ │ ├── annotations │ │ │ ├── IgnoreAdaptation.java │ │ │ ├── NestedField.java │ │ │ └── StringDate.java │ │ │ ├── api │ │ │ ├── EsAggregation.java │ │ │ ├── EsInstanceAdapter.java │ │ │ ├── EsQuery.java │ │ │ └── Fields.java │ │ │ ├── health │ │ │ ├── DBHealthChecker.java │ │ │ └── EsHealthChecker.java │ │ │ ├── resources │ │ │ ├── DailySnapshot.java │ │ │ ├── Health.java │ │ │ ├── Instance.java │ │ │ ├── InstanceCounter.java │ │ │ └── Query.java │ │ │ └── utils │ │ │ ├── ObjectAdapter.java │ │ │ └── Utils.java │ └── resources │ │ └── banner.txt │ └── test │ ├── java │ └── com │ │ └── pinterest │ │ └── soundwave │ │ └── resources │ │ └── DailySnapshotTest.java │ └── resources │ └── fixtures │ └── factsAndPackages.json ├── commons ├── checkstyle-suppressions.xml ├── config │ ├── log4j.properties │ └── soundwave.properties ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── assembly.xml │ └── java │ │ └── com │ │ └── pinterest │ │ ├── OperationStats.java │ │ ├── StatsUtil.java │ │ ├── aws │ │ └── AwsClientFactory.java │ │ ├── config │ │ └── Configuration.java │ │ ├── job │ │ ├── ExclusiveRecurringJobExecutor.java │ │ ├── ForeverRepeatedJobExecutor.java │ │ ├── JobConfig.java │ │ ├── JobExecutor.java │ │ ├── JobInfoStore.java │ │ ├── JobRunInfo.java │ │ ├── OwnershipDecider.java │ │ └── SqsTriggeredJobExecutor.java │ │ ├── soundwave │ │ ├── aws │ │ │ ├── AbstractEsInstanceFactory.java │ │ │ ├── AwsStatus.java │ │ │ ├── AwsUtilities.java │ │ │ ├── BasicEsInstanceFactory.java │ │ │ ├── CannotRetrieveFromEC2Exception.java │ │ │ ├── CloudInstanceStore.java │ │ │ ├── DailySnapshotStore.java │ │ │ ├── DailySnapshotStoreFactory.java │ │ │ ├── Ec2InstanceNotificationDetail.java │ │ │ ├── Ec2InstanceStore.java │ │ │ ├── Ec2InstanceUpdateHandler.java │ │ │ ├── Ec2NotificationHandler.java │ │ │ ├── MessageProcessingResult.java │ │ │ ├── MissingNameTagException.java │ │ │ ├── NotificationEvent.java │ │ │ └── SqsClient.java │ │ ├── bean │ │ │ ├── EsAwsStatus.java │ │ │ ├── EsDailySnapshotInstance.java │ │ │ ├── EsDocument.java │ │ │ ├── EsInstance.java │ │ │ ├── EsInstanceConfig.java │ │ │ ├── EsInstanceCountRecord.java │ │ │ ├── EsQueryResult.java │ │ │ ├── EsStatus.java │ │ │ ├── EsStoreMappingProperty.java │ │ │ ├── InvalidStateTransitionException.java │ │ │ ├── PinterestEsInstance.java │ │ │ └── State.java │ │ ├── elasticsearch │ │ │ ├── BulkRequestExecutor.java │ │ │ ├── CmdbInstanceStore.java │ │ │ ├── EsBulkResponseSummary.java │ │ │ ├── EsDailySnapshotStore.java │ │ │ ├── EsDailySnapshotStoreFactory.java │ │ │ ├── EsInstanceCounterStore.java │ │ │ ├── EsIterator.java │ │ │ ├── EsMapper.java │ │ │ ├── EsPropertyNamingStrategy.java │ │ │ ├── EsStore.java │ │ │ ├── InstanceCounterStore.java │ │ │ ├── PinterestCmdbInstanceStore.java │ │ │ └── ScrollableResponse.java │ │ ├── pinterest │ │ │ ├── EsFacts.java │ │ │ ├── EsFactsAndPackages.java │ │ │ ├── EsFactsWhitelist.java │ │ │ ├── EsInstanceStore.java │ │ │ ├── EsInstanceTags.java │ │ │ ├── EsMetaData.java │ │ │ ├── EsNameMetaData.java │ │ │ ├── EsPackages.java │ │ │ ├── EsServiceMapping.java │ │ │ ├── EsServiceMappingStore.java │ │ │ ├── HostImageInfoProvider.java │ │ │ ├── InstanceTagger.java │ │ │ ├── QuobleNameChecker.java │ │ │ └── ServiceMappingStore.java │ │ └── utils │ │ │ ├── JsonCompareUtil.java │ │ │ ├── StringArrayOrElementStringDeserializer.java │ │ │ ├── StringListOrElementDeserializer.java │ │ │ └── ThrowingFunction.java │ │ └── zookeeper │ │ ├── ZkClient.java │ │ ├── ZkJobInfoStore.java │ │ └── ZkScheduledJobOwnershipDecider.java │ └── test │ └── java │ └── com │ └── pinterest │ └── soundwave │ ├── bean │ ├── EsDailySnapshotInstanceTest.java │ └── EsInstanceTest.java │ ├── elasticsearch │ ├── DummyEsStore.java │ ├── ESServiceMappingStoreTest.java │ ├── EsDailySnapshotStoreTest.java │ ├── EsIteratorTest.java │ └── EsPropertyNamingStrategyTest.java │ ├── utils │ ├── InstanceTaggerTest.java │ └── JsonCompareUtilTest.java │ └── zookeeper │ └── ZkJobInfoStoreTest.java ├── docker-compose.yml ├── pinterest_v2_checks.xml ├── pom.xml ├── resources └── soundwavearch.jpg ├── soundwavedemo.png ├── terraform ├── .gitignore ├── soundwave_lambda.py ├── soundwave_lambda.py.zip └── soundwaveaws.tf ├── ui ├── Dockerfile ├── Readme.md ├── __init__.py ├── app.py ├── config.py ├── requirements.txt ├── static │ ├── css │ │ ├── bootstrap.min.css │ │ ├── bootstrap.min.css.map │ │ ├── buttons.bootstrap.min.css │ │ ├── dataTables.bootstrap.css │ │ ├── dataTables.bootstrap.min.css │ │ ├── font-awesome.css │ │ ├── font-awesome.min.css │ │ ├── jquery.dataTables.css │ │ ├── jsoneditor.css │ │ └── style.css │ ├── favicon.ico │ ├── fonts │ │ ├── FontAwesome.otf │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.svg │ │ ├── fontawesome-webfont.ttf │ │ ├── fontawesome-webfont.woff │ │ ├── fontawesome-webfont.woff2 │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ └── glyphicons-halflings-regular.woff2 │ ├── images │ │ ├── Sorting icons.psd │ │ ├── favicon.ico │ │ ├── sort_asc.png │ │ ├── sort_asc_disabled.png │ │ ├── sort_both.png │ │ ├── sort_desc.png │ │ └── sort_desc_disabled.png │ ├── js │ │ ├── bootstrap.min.js │ │ ├── buttons.bootstrap.min.js │ │ ├── buttons.flash.min.js │ │ ├── buttons.html5.min.js │ │ ├── clipboard.min.js │ │ ├── dataTables.bootstrap.min.js │ │ ├── dataTables.buttons.min.js │ │ ├── jquery-3.1.0.min.js │ │ ├── jquery.dataTables.min.js │ │ ├── jquery.jsoneditor.js │ │ └── renderjson.js │ └── swf │ │ └── flashExport.swf ├── templates │ ├── 404.html │ ├── 500.html │ ├── aggregations.html │ ├── index.html │ ├── instance.html │ ├── navbar.html │ ├── query.html │ └── reservations.html └── utils.py └── worker ├── Dockerfile ├── checkstyle-suppressions.xml ├── config ├── log4j.properties └── soundwaveworker.properties ├── pom.xml ├── scripts ├── index_settings.json ├── provision_index.sh └── run_in_container.sh └── src └── main ├── assembly └── assembly.xml └── java └── com └── pinterest ├── AwsServiceTagUpdater.java ├── BasicUploadTagsGenerator.java ├── DummyOwnershipDecider.java ├── DummyRecurringJob.java ├── JobManager.java ├── UploadTagsGenerator.java └── soundwave ├── WorkerMain.java └── job └── definitions ├── AwsInstanceStatusJob.java ├── DailyInstanceCountPerTypeJob.java ├── DailyRollupJob.java ├── ElasticSearchHealthCheckJob.java ├── HealthCheckJob.java └── ReconcileWithAwsJob.java /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | target/ 4 | *.pyc 5 | *.swp 6 | *~ 7 | .DS_Store 8 | .phutil_module_cache 9 | build.properties 10 | asterix/local 11 | obelix/server/local 12 | obelix/solr/overlays 13 | commons-thrift/gen-py/ 14 | driven-plugin*.jar 15 | *.hprof 16 | .venv/ 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk8 4 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /api/Dockerfile: -------------------------------------------------------------------------------- 1 | # Refer https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/ 2 | # for best practices maintaining this file 3 | 4 | # Build Java8 docker. Credit is to https://github.com/cogniteev/docker-oracle-java, 5 | # which is licensed under the MIT license 6 | # Pull base image. 7 | FROM ubuntu:14.04 8 | 9 | # Install Java. 10 | RUN \ 11 | apt-get update && \ 12 | apt-get install -y software-properties-common && \ 13 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 14 | add-apt-repository -y ppa:webupd8team/java && \ 15 | apt-get update && \ 16 | apt-get install -y oracle-java8-installer && \ 17 | rm -rf /var/lib/apt/lists/* && \ 18 | rm -rf /var/cache/oracle-jdk8-installer 19 | 20 | # Define commonly used JAVA_HOME variable 21 | ENV JAVA_HOME /usr/lib/jvm/java-8-oracle 22 | 23 | # Create log directory 24 | RUN mkdir -p /var/log/soundwave-worker/ 25 | 26 | # Create and set current directory 27 | WORKDIR /opt/soundwave-api 28 | 29 | # Add the build artifact under /opt, can be overridden by docker build 30 | ARG ARTIFACT_PATH=target/soundwave-api-0.1-SNAPSHOT-bin.tar.gz 31 | ADD $ARTIFACT_PATH /opt/soundwave-api/ 32 | 33 | # Default command to run service, do not override it in docker run unless have a good reason 34 | # Use "docker logs ID" to view stdout and stderr 35 | CMD ["scripts/run_in_container.sh"] -------------------------------------------------------------------------------- /api/checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /api/config/log4j.properties: -------------------------------------------------------------------------------- 1 | # log4j logging configuration. 2 | # root logger. 3 | log4j.rootLogger=INFO, CONSOLE, ROLLINGFILE 4 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 5 | log4j.appender.CONSOLE.Threshold=INFO 6 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] (%F:%L) %-5p %m%n 8 | # we will have to rely on external cron job to keep the logging space consumption under control. 9 | log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.ROLLINGFILE.Threshold=INFO 11 | log4j.appender.ROLLINGFILE.File=/var/log/soundwave-api/soundwave-api.log 12 | log4j.appender.ROLLINGFILE.append=true 13 | log4j.appender.ROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 14 | log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout 15 | log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d [%-6p] [%t] %c{1} - %m%n 16 | # access logger. 17 | log4j.logger.access=DEBUG, ACCESSROLLINGFILE 18 | log4j.additivity.access=false 19 | # we will have to rely on external cron job to keep the logging space consumption under control. 20 | log4j.appender.ACCESSROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 21 | log4j.appender.ACCESSROLLINGFILE.File=/var/log/soundwave-api/access.log 22 | log4j.appender.ACCESSROLLINGFILE.append=true 23 | log4j.appender.ACCESSROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 24 | log4j.appender.ACCESSROLLINGFILE.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.ACCESSROLLINGFILE.layout.ConversionPattern=%m%n 26 | # failed request logger. 27 | log4j.logger.failure=INFO, FAILUREROLLINGFILE 28 | log4j.additivity.failure=false 29 | # we will have to rely on external cron job to keep the logging space consumption under control. 30 | log4j.appender.FAILUREROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 31 | log4j.appender.FAILUREROLLINGFILE.File=/var/log/soundwave-api/failure.log 32 | log4j.appender.FAILUREROLLINGFILE.append=true 33 | log4j.appender.FAILUREROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 34 | log4j.appender.FAILUREROLLINGFILE.layout=org.apache.log4j.PatternLayout 35 | log4j.appender.FAILUREROLLINGFILE.layout.ConversionPattern=%m%n 36 | -------------------------------------------------------------------------------- /api/config/soundwaveapi.properties: -------------------------------------------------------------------------------- 1 | #SQS endpoint that AWS lambda pushes Ec2 notification events 2 | update_queue=https://sqs.us-east-1.amazonaws.com//soundwave-events 3 | 4 | #ElasticSearch 5 | es_cluster_lb=soundwave-store 6 | es_cluster_port=9300 7 | es_instance_index=soundwave_prod 8 | es_daily_snapshot_index=soundwave_ss 9 | es_cluster_name=elasticsearch 10 | 11 | -------------------------------------------------------------------------------- /api/scripts/run_in_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Start the process inside a docker container with java installed 3 | # Some settings can be overriden by environment variables as shown below 4 | 5 | ulimit -n 65536 6 | 7 | export SERVICENAME=soundwave-api 8 | export JAVA_MAIN=com.pinterest.soundwave.ApiApplication 9 | export CMD_LINE_ARG=server 10 | export APP_CONFIG_FILE=soundwave-api-prod.yml 11 | 12 | LOG4J_CONFIG_FILE=${LOG4J_CONFIG_FILE:=config/log4j.properties} 13 | CONFIG_FILE=${CONFIG_FILE:=config/soundwaveapi.properties} 14 | HEAP_SIZE=${HEAP_SIZE:=512m} 15 | NEW_SIZE=${NEW_SIZE:=256m} 16 | 17 | 18 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 19 | export PARENT_DIR="$(dirname $DIR)" 20 | 21 | LOG_DIR=/var/log/soundwave-worker 22 | CP=${PARENT_DIR}:${PARENT_DIR}/*:${PARENT_DIR}/lib/* 23 | 24 | exec java -server -Xmx${HEAP_SIZE} -Xms${HEAP_SIZE} -XX:NewSize=${NEW_SIZE} -XX:MaxNewSize=${NEW_SIZE} \ 25 | -verbosegc -Xloggc:${LOG_DIR}/gc.log \ 26 | -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 \ 27 | -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=100 -XX:GCLogFileSize=2M \ 28 | -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintClassHistogram \ 29 | -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParNewGC \ 30 | -XX:OnOutOfMemoryError="kill -9 %p" \ 31 | -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=60 -XX:+UseCMSInitiatingOccupancyOnly \ 32 | -XX:ErrorFile=${LOG_DIR}/jvm_error.log \ 33 | -cp ${CP} -Dlog4j.configuration=${LOG4J_CONFIG_FILE} -Dconfig.file=${CONFIG_FILE} \ 34 | -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false \ 35 | -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=10102 \ 36 | -Dfile.encoding=UTF-8 \ 37 | ${JAVA_MAIN} ${CMD_LINE_ARG} ${APP_CONFIG_FILE} 38 | -------------------------------------------------------------------------------- /api/soundwave-api-dev.yml: -------------------------------------------------------------------------------- 1 | server: 2 | applicationConnectors: 3 | - type: http 4 | port: 8080 5 | useForwardedHeaders: true 6 | adminConnectors: 7 | - type: http 8 | port: 8081 9 | 10 | requestLog: 11 | timeZone: UTC 12 | appenders: 13 | - type: file 14 | currentLogFilename: /Users/sjoshi/code/log-dev-api-soundwave/access.log 15 | threshold: ALL 16 | archive: true 17 | archivedLogFilenamePattern: /Users/sjoshi/code/log-dev-api-soundwave/access.%d.log.gz 18 | archivedFileCount: 10 19 | 20 | logging: 21 | 22 | # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. 23 | level: INFO 24 | appenders: 25 | - type: file 26 | currentLogFilename: /Users/sjoshi/code/log-dev-api-soundwave/soundwave-api-info.log 27 | threshold: ALL 28 | archive: true 29 | archivedLogFilenamePattern: /Users/sjoshi/code/log-dev-api-soundwave/soundwave-api-info.%d.log 30 | archivedFileCount: 10 31 | timeZone: UTC 32 | 33 | loggers: 34 | com.pinterest.soundwave.ApiApplication: DEBUG -------------------------------------------------------------------------------- /api/soundwave-api-prod.yml: -------------------------------------------------------------------------------- 1 | server: 2 | applicationConnectors: 3 | - type: http 4 | port: 80 5 | useForwardedHeaders: true 6 | 7 | adminConnectors: 8 | - type: http 9 | port: 8081 10 | 11 | requestLog: 12 | type: classic 13 | timeZone: UTC 14 | appenders: 15 | - type: file 16 | currentLogFilename: /var/log/soundwave-api/access.log 17 | threshold: ALL 18 | archive: true 19 | archivedLogFilenamePattern: /var/log/soundwave-api/access.%d.log.gz 20 | archivedFileCount: 10 21 | 22 | logging: 23 | # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. 24 | level: INFO 25 | appenders: 26 | - type: file 27 | currentLogFilename: /var/log/soundwave-api/soundwave-api-info.log 28 | threshold: ALL 29 | archive: true 30 | archivedLogFilenamePattern: /var/log/soundwave-api/soundwave-api-info.%d.log 31 | archivedFileCount: 10 32 | timeZone: UTC 33 | loggers: 34 | com.pinterest.soundwave.ApiApplication: DEBUG 35 | -------------------------------------------------------------------------------- /api/src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | bin 4 | 5 | 6 | tar.gz 7 | 8 | 9 | 10 | Dockerfile 11 | / 12 | 13 | 14 | soundwave-api-dev.yml 15 | / 16 | 17 | 18 | soundwave-api-prod.yml 19 | / 20 | 21 | 22 | 23 | 24 | scripts 25 | scripts 26 | 27 | *.sh 28 | 29 | 30 | 31 | teletraan 32 | teletraan 33 | 34 | 35 | 36 | 37 | config 38 | config 39 | 40 | *.properties 41 | *.xml 42 | 43 | 44 | 45 | target 46 | 47 | 48 | soundwave*.jar 49 | 50 | 51 | 52 | 53 | 54 | 55 | lib 56 | 57 | org.slf4j:slf4j-jdk14 58 | 59 | false 60 | runtime 61 | 62 | 63 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/ApiConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import io.dropwizard.Configuration; 21 | import io.federecio.dropwizard.swagger.SwaggerBundleConfiguration; 22 | 23 | public class ApiConfiguration extends Configuration { 24 | 25 | // TODO: implement service configuration 26 | @JsonProperty("swagger") 27 | public SwaggerBundleConfiguration swaggerBundleConfiguration; 28 | } 29 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/annotations/IgnoreAdaptation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.annotations; 18 | 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | 22 | /** 23 | * Use this annotation to ignore this field during the processing of object adapter 24 | */ 25 | @Retention(RetentionPolicy.RUNTIME) 26 | public @interface IgnoreAdaptation { 27 | 28 | } 29 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/annotations/NestedField.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.annotations; 18 | 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | 22 | @Retention(RetentionPolicy.RUNTIME) 23 | public @interface NestedField { 24 | String src(); 25 | } 26 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/annotations/StringDate.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.annotations; 18 | 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | 22 | @Retention(RetentionPolicy.RUNTIME) 23 | public @interface StringDate { 24 | } 25 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/api/EsAggregation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.api; 18 | 19 | 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | /** 23 | * Accepts a comma seperated list of strings 24 | */ 25 | public class EsAggregation { 26 | 27 | @JsonProperty("query") 28 | private String query; 29 | 30 | public String getQuery() { 31 | return query; 32 | } 33 | 34 | public void setQuery(String query) { 35 | this.query = query; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/api/EsQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.api; 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | @JsonIgnoreProperties(ignoreUnknown = true) 23 | public class EsQuery { 24 | 25 | @JsonProperty("query") 26 | private String queryString; 27 | 28 | @JsonProperty("fields") 29 | private String fields; 30 | 31 | public String getQueryString() { 32 | return queryString; 33 | } 34 | 35 | public void setQueryString(String queryString) { 36 | this.queryString = queryString; 37 | } 38 | 39 | public String getFields() { 40 | return fields; 41 | } 42 | 43 | public void setFields(String fields) { 44 | this.fields = fields; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/api/Fields.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.api; 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | @JsonIgnoreProperties(ignoreUnknown = true) 23 | public class Fields { 24 | 25 | @JsonProperty("fields") 26 | private String fields; 27 | 28 | public String getFields() { 29 | return fields; 30 | } 31 | 32 | public void setFields(String fields) { 33 | this.fields = fields; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/health/DBHealthChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.health; 18 | 19 | import com.codahale.metrics.health.HealthCheck; 20 | 21 | public interface DBHealthChecker { 22 | HealthCheck.Result check() throws Exception; 23 | } 24 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/health/EsHealthChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.health; 18 | 19 | import com.pinterest.soundwave.bean.EsStatus; 20 | import com.pinterest.soundwave.elasticsearch.CmdbInstanceStore; 21 | 22 | import com.codahale.metrics.health.HealthCheck; 23 | 24 | public class EsHealthChecker extends HealthCheck implements DBHealthChecker { 25 | 26 | private final CmdbInstanceStore cmdbInstanceStore; 27 | 28 | public EsHealthChecker(CmdbInstanceStore cmdbInstanceStore) { 29 | this.cmdbInstanceStore = cmdbInstanceStore; 30 | } 31 | 32 | @Override 33 | public Result check() throws Exception { 34 | 35 | String status = cmdbInstanceStore.checkStatus(); 36 | 37 | if (EsStatus.SUCCESS.isStatus(status)) { 38 | return Result.healthy(); 39 | 40 | } else if (EsStatus.ERROR.isStatus(status)) { 41 | return Result.unhealthy("Es Status is red"); 42 | 43 | } else if (EsStatus.TIMEOUT.isStatus(status)) { 44 | return Result.unhealthy("timeout"); 45 | 46 | } else { 47 | return Result.unhealthy("unknown issue"); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/resources/DailySnapshot.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.resources; 18 | 19 | 20 | import com.pinterest.soundwave.utils.Utils; 21 | import com.pinterest.OperationStats; 22 | import com.pinterest.soundwave.aws.DailySnapshotStore; 23 | import com.pinterest.soundwave.aws.DailySnapshotStoreFactory; 24 | import com.pinterest.soundwave.bean.EsDailySnapshotInstance; 25 | import com.pinterest.soundwave.elasticsearch.EsDailySnapshotStoreFactory; 26 | 27 | import org.apache.commons.collections.IteratorUtils; 28 | import org.joda.time.DateTime; 29 | import org.joda.time.format.DateTimeFormat; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | import java.util.HashMap; 34 | import java.util.Iterator; 35 | import java.util.List; 36 | import java.util.Map; 37 | import javax.validation.constraints.NotNull; 38 | import javax.ws.rs.Consumes; 39 | import javax.ws.rs.GET; 40 | import javax.ws.rs.Path; 41 | import javax.ws.rs.PathParam; 42 | import javax.ws.rs.Produces; 43 | import javax.ws.rs.core.MediaType; 44 | import javax.ws.rs.core.Response; 45 | 46 | @Path("/v2/") 47 | @Produces(MediaType.APPLICATION_JSON) 48 | @Consumes(MediaType.APPLICATION_JSON) 49 | public class DailySnapshot { 50 | 51 | private static final Logger logger = LoggerFactory.getLogger(DailySnapshot.class); 52 | private DailySnapshotStoreFactory factory = new EsDailySnapshotStoreFactory(); 53 | 54 | @GET 55 | @Path("/dailysnapshot/{day}") 56 | public Response getDailySnapshot(@PathParam("day") 57 | @NotNull String day) { 58 | 59 | OperationStats opStats = new OperationStats("cmdb_api", "get_dailysnapshot", new HashMap<>()); 60 | Map tags = new HashMap<>(); 61 | 62 | try { 63 | DateTime time = DateTime.parse(day, DateTimeFormat.forPattern("yyyy-MM-dd")); 64 | DailySnapshotStore dailySnapshot = factory.getDailyStore(time); 65 | 66 | Iterator 67 | iter = dailySnapshot.getSnapshotInstances(); 68 | List ret = IteratorUtils.toList(iter); 69 | 70 | logger.info("Success: getDailySnapshot - {}", day); 71 | return Response.status(Response.Status.OK) 72 | .type(MediaType.APPLICATION_JSON) 73 | .entity(ret) 74 | .build(); 75 | 76 | } catch (Exception e) { 77 | 78 | return Utils.responseException(e, logger, opStats, tags); 79 | } 80 | 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/resources/Health.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.resources; 18 | 19 | import com.pinterest.OperationStats; 20 | 21 | import javax.ws.rs.GET; 22 | import javax.ws.rs.Path; 23 | import javax.ws.rs.core.Response; 24 | 25 | 26 | @Path("/_/_/") 27 | public class Health { 28 | 29 | @GET 30 | public Response healthCheckGet() { 31 | 32 | OperationStats opStats = new OperationStats("cmdb_api", "elb_health_check"); 33 | opStats.succeed(); 34 | 35 | return Response.status(Response.Status.OK).build(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /api/src/main/java/com/pinterest/soundwave/utils/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.pinterest.soundwave.utils; 18 | 19 | 20 | import com.pinterest.OperationStats; 21 | 22 | import org.apache.commons.lang.exception.ExceptionUtils; 23 | import org.slf4j.Logger; 24 | 25 | import java.util.Map; 26 | import javax.ws.rs.core.MediaType; 27 | import javax.ws.rs.core.Response; 28 | 29 | public class Utils { 30 | 31 | public static Response responseException(Exception e, Logger logger, 32 | OperationStats opStats, Map tags) { 33 | 34 | logger.error(ExceptionUtils.getFullStackTrace(e)); 35 | logger.error(ExceptionUtils.getRootCauseMessage(e)); 36 | 37 | tags.put("status", String.valueOf(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode())); 38 | tags.put("message", e.getClass().getSimpleName()); 39 | 40 | opStats.failed(tags); 41 | 42 | return Response.status(Response.Status.INTERNAL_SERVER_ERROR) 43 | .type(MediaType.APPLICATION_JSON) 44 | .build(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /api/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | 3 | api 4 | 5 | ================================================================================ 6 | 7 | -------------------------------------------------------------------------------- /api/src/test/java/com/pinterest/soundwave/resources/DailySnapshotTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.resources; 17 | 18 | import org.junit.Assert; 19 | import org.junit.Ignore; 20 | import org.junit.Test; 21 | 22 | import javax.ws.rs.core.Response; 23 | 24 | public class DailySnapshotTest { 25 | 26 | @Test 27 | @Ignore 28 | public void getDailySnapshot() throws Exception { 29 | DailySnapshot snapshot = new DailySnapshot(); 30 | Response resp = snapshot.getDailySnapshot("2016-12-12"); 31 | Assert.assertNotNull(resp); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /api/src/test/resources/fixtures/factsAndPackages.json: -------------------------------------------------------------------------------- 1 | { 2 | "facts": { 3 | "test": "test-facts" 4 | }, 5 | "pkgs": { 6 | "test": "test-pkgs" 7 | }, 8 | "unknown": { 9 | "test": "unknown" 10 | }, 11 | "unknown2": "unknown" 12 | } -------------------------------------------------------------------------------- /commons/checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /commons/config/log4j.properties: -------------------------------------------------------------------------------- 1 | # log4j logging configuration. 2 | # root logger. 3 | log4j.rootLogger=INFO, CONSOLE, ROLLINGFILE 4 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 5 | log4j.appender.CONSOLE.Threshold=INFO 6 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] (%F:%L) %-5p %m%n 8 | # we will have to rely on external cron job to keep the logging space consumption under control. 9 | log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.ROLLINGFILE.Threshold=INFO 11 | log4j.appender.ROLLINGFILE.File=/var/log/soundwave/soundwaveowrker.log 12 | log4j.appender.ROLLINGFILE.append=true 13 | log4j.appender.ROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 14 | log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout 15 | log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d [%-6p] [%t] %c{1} - %m%n 16 | 17 | # access logger. 18 | log4j.logger.access=INFO, ACCESSROLLINGFILE 19 | log4j.additivity.access=false 20 | # we will have to rely on external cron job to keep the logging space consumption under control. 21 | log4j.appender.ACCESSROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 22 | log4j.appender.ACCESSROLLINGFILE.File=/var/log/soundwaveworker/access.log 23 | log4j.appender.ACCESSROLLINGFILE.append=true 24 | log4j.appender.ACCESSROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 25 | log4j.appender.ACCESSROLLINGFILE.layout=org.apache.log4j.PatternLayout 26 | log4j.appender.ACCESSROLLINGFILE.layout.ConversionPattern=%m%n 27 | 28 | # failed request logger. 29 | log4j.logger.failure=INFO, FAILUREROLLINGFILE 30 | log4j.additivity.failure=false 31 | # we will have to rely on external cron job to keep the logging space consumption under control. 32 | log4j.appender.FAILUREROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 33 | log4j.appender.FAILUREROLLINGFILE.File=/var/log/soundwaveworker/failure.log 34 | log4j.appender.FAILUREROLLINGFILE.append=true 35 | log4j.appender.FAILUREROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 36 | log4j.appender.FAILUREROLLINGFILE.layout=org.apache.log4j.PatternLayout 37 | 38 | #Sqs events logger 39 | log4j.logger.sqsEventLogger=INFO, SQSEVENTROLLINGFILE 40 | log4j.appender.SQSEVENTROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 41 | log4j.appender.SQSEVENTROLLINGFILE.File=/var/log/cmdb/sqsevents.log 42 | log4j.appender.SQSEVENTROLLINGFILE.append=true 43 | log4j.appender.SQSEVENTROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 44 | log4j.appender.SQSEVENTROLLINGFILE.layout=org.apache.log4j.PatternLayout 45 | log4j.appender.SQSEVENTROLLINGFILE.layout.ConversionPattern=%m%n 46 | 47 | #Message Processing API logger 48 | log4j.logger.messageProcessingLog=INFO, MESSAGEPROCESSINGFILE 49 | log4j.appender.MESSAGEPROCESSINGFILE=org.apache.log4j.DailyRollingFileAppender 50 | log4j.appender.MESSAGEPROCESSINGFILE.File=/var/log/soundwaveworker/messageprocessing.log 51 | log4j.appender.MESSAGEPROCESSINGFILE.append=true 52 | log4j.appender.MESSAGEPROCESSINGFILE.DatePattern='.'yyyy-MM-dd-HH 53 | log4j.appender.MESSAGEPROCESSINGFILE.layout=org.apache.log4j.PatternLayout 54 | log4j.appender.MESSAGEPROCESSINGFILE.layout.ConversionPattern=%m%n 55 | 56 | 57 | #Scheduled Job logger 58 | log4j.logger.scheduledJobLog=INFO, SCHEDULEDJOBFILE 59 | log4j.appender.SCHEDULEDJOBFILE=org.apache.log4j.DailyRollingFileAppender 60 | log4j.appender.SCHEDULEDJOBFILE.File=/var/log/soundwaveworker/scheduledjob.log 61 | log4j.appender.SCHEDULEDJOBFILE.append=true 62 | log4j.appender.SCHEDULEDJOBFILE.DatePattern='.'yyyy-MM-dd-HH 63 | log4j.appender.SCHEDULEDJOBFILE.layout=org.apache.log4j.PatternLayout 64 | log4j.appender.SCHEDULEDJOBFILE.layout.ConversionPattern=%m%n 65 | -------------------------------------------------------------------------------- /commons/config/soundwave.properties: -------------------------------------------------------------------------------- 1 | #Common Service settings 2 | thrift_server_port=8090 3 | ostrich_port=9999 4 | service_name=cmdb 5 | cluster_name=cmdb 6 | max_connection_idle_in_minute=6 7 | max_concurrent_requests=2000 8 | server_set_hostname_prefix=cmdb 9 | server_set_enabled=true 10 | enforce_serverset_hostname_prefix=true 11 | server_set_path=/discovery/soundwave/prod 12 | slow_request_threshold_millis=1000 13 | 14 | 15 | 16 | #SQS endpoint that AWS lambda pushes Ec2 notification events 17 | update_queue=aws.soundwave-events.queue 18 | 19 | #ElasticSearch 20 | es_cluster_lb=soundwave-store 21 | es_cluster_port=9300 22 | es_instance_index = soundwave_prod 23 | es_daily_snapshot_index=soundwave_ss 24 | es_cluster_name=elasticsearch 25 | 26 | #Zookeeper 27 | zk_connection_string=soundwave-zk:2181 28 | zk_path =/soundwave/prod 29 | 30 | #JobManager Settings 31 | num_subscriber=4 32 | aws_region=us-east-1 33 | aws_call_ratelimit = 2 34 | send_service_mapping_to_aws=false 35 | 36 | #Ec2 Handler setting 37 | instance_factory = com.pinterest.soundwave.aws.BasicEsInstanceFactory 38 | 39 | aws_tag_generator = com.pinterest.BasicUploadTagsGenerator 40 | 41 | #Using IAM role or not 42 | use_instance_profile = false 43 | -------------------------------------------------------------------------------- /commons/src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | bin 4 | 5 | 6 | tar.gz 7 | 8 | 9 | 10 | 11 | lib 12 | 13 | org.slf4j:slf4j-jdk14 14 | 15 | false 16 | runtime 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/OperationStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest; 17 | 18 | import com.twitter.ostrich.stats.Stats; 19 | import org.apache.commons.lang3.time.StopWatch; 20 | 21 | import java.util.Map; 22 | 23 | public final class OperationStats { 24 | 25 | private String methodName; 26 | private String statsName; 27 | private StopWatch watch; 28 | private Map tags; 29 | 30 | public OperationStats(String methodName, String statsName) { 31 | this(methodName, statsName, null); 32 | } 33 | 34 | public OperationStats(String methodName, String statsName, Map tags) { 35 | this.methodName = methodName; 36 | this.statsName = statsName; 37 | this.tags = tags; 38 | Stats.incr(StatsUtil.getStatsName(methodName, statsName, tags)); 39 | watch = new StopWatch(); 40 | watch.start(); 41 | } 42 | 43 | 44 | public void succeed() { 45 | this.watch.stop(); 46 | Stats 47 | .incr(StatsUtil 48 | .getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.SUCCESS, tags)); 49 | Stats.addMetric( 50 | StatsUtil.getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.TIME, tags), 51 | (int) watch.getTime()); 52 | } 53 | 54 | public void succeed(Map additionalTags) { 55 | 56 | this.watch.stop(); 57 | 58 | if (tags == null) { 59 | this.tags = additionalTags; 60 | } else { 61 | this.tags.putAll(additionalTags); 62 | } 63 | 64 | Stats 65 | .incr(StatsUtil 66 | .getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.SUCCESS, tags)); 67 | Stats.addMetric( 68 | StatsUtil.getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.TIME, tags), 69 | (int) watch.getTime()); 70 | } 71 | 72 | public void failed() { 73 | this.watch.stop(); 74 | Stats 75 | .incr(StatsUtil 76 | .getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.FAILURE, tags)); 77 | Stats.addMetric( 78 | StatsUtil.getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.TIME, tags), 79 | (int) watch.getTime()); 80 | } 81 | 82 | public void failed(Map additionalTags) { 83 | 84 | this.watch.stop(); 85 | 86 | if (tags == null) { 87 | this.tags = additionalTags; 88 | } else { 89 | this.tags.putAll(additionalTags); 90 | } 91 | 92 | Stats 93 | .incr(StatsUtil 94 | .getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.FAILURE, tags)); 95 | Stats.addMetric( 96 | StatsUtil.getStatsName(this.methodName, this.statsName, StatsUtil.StatsType.TIME, tags), 97 | (int) watch.getTime()); 98 | } 99 | 100 | public long getTime() { 101 | return this.watch.getTime(); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/aws/AwsClientFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.aws; 17 | 18 | import com.pinterest.config.Configuration; 19 | 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; 22 | import com.amazonaws.auth.InstanceProfileCredentialsProvider; 23 | import com.amazonaws.regions.Region; 24 | import com.amazonaws.services.ec2.AmazonEC2Client; 25 | import com.amazonaws.services.s3.AmazonS3Client; 26 | import com.amazonaws.services.sqs.AmazonSQSClient; 27 | import com.google.common.base.Preconditions; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | /** 32 | * A factory client to get various AWS service client 33 | */ 34 | public class AwsClientFactory { 35 | 36 | private static Logger logger = LoggerFactory.getLogger(AwsClientFactory.class); 37 | private static AWSCredentialsProvider credentialsProvider = null; 38 | 39 | private static synchronized AWSCredentialsProvider getCredentialProvider() { 40 | if (credentialsProvider == null) { 41 | boolean useInstanceProfile = Configuration.getProperties().getBoolean( 42 | "use_instance_profile", false); 43 | 44 | if (useInstanceProfile) { 45 | credentialsProvider = new InstanceProfileCredentialsProvider(); 46 | } else { 47 | //If not specifies use Ec2 instance profile, use default chain 48 | credentialsProvider = new DefaultAWSCredentialsProviderChain(); 49 | } 50 | } 51 | return credentialsProvider; 52 | } 53 | 54 | public static AmazonSQSClient createSQSClient() { 55 | return new AmazonSQSClient(getCredentialProvider()); 56 | } 57 | 58 | public static AmazonSQSClient createSQSClient(Region region) { 59 | Preconditions.checkNotNull(region); 60 | AmazonSQSClient client = new AmazonSQSClient(getCredentialProvider()); 61 | client.setRegion(region); 62 | return client; 63 | } 64 | 65 | public static AmazonEC2Client createEC2Client() { 66 | return new AmazonEC2Client(getCredentialProvider()); 67 | } 68 | 69 | public static AmazonEC2Client createEC2Client(Region region) { 70 | Preconditions.checkNotNull(region); 71 | AmazonEC2Client client = new AmazonEC2Client(getCredentialProvider()); 72 | client.setRegion(region); 73 | return client; 74 | } 75 | 76 | public static AmazonS3Client createS3Client() { 77 | return new AmazonS3Client(getCredentialProvider()); 78 | } 79 | 80 | public static AmazonS3Client createS3Client(Region region) { 81 | Preconditions.checkNotNull(region); 82 | AmazonS3Client s3Client = new AmazonS3Client(getCredentialProvider()); 83 | s3Client.setRegion(region); 84 | return s3Client; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/config/Configuration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.config; 17 | 18 | import org.apache.commons.configuration.PropertiesConfiguration; 19 | import org.apache.commons.lang.exception.ExceptionUtils; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | /** 24 | * Simple Wrapper on Configrations 25 | */ 26 | public final class Configuration { 27 | 28 | private static final Logger logger = LoggerFactory.getLogger(Configuration.class); 29 | private static final Configuration instance = new Configuration(); 30 | 31 | private PropertiesConfiguration properties; 32 | 33 | private Configuration() { 34 | 35 | try { 36 | String file = System.getProperty("config.file", "config/soundwave.properties"); 37 | logger.info("Configuration file is {}", file); 38 | properties = new PropertiesConfiguration(file); 39 | } catch (Exception ex) { 40 | logger.error("Fail to load configuration {}. Error is {}", 41 | properties == null ? "null" : properties.getFileName(), 42 | ExceptionUtils.getRootCauseMessage(ex)); 43 | } 44 | } 45 | 46 | public static PropertiesConfiguration getProperties() { 47 | return instance.properties; 48 | } 49 | 50 | public static void setProperties(PropertiesConfiguration configuration) { 51 | instance.properties = configuration; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/job/ForeverRepeatedJobExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.job; 17 | 18 | import com.pinterest.config.Configuration; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.util.concurrent.ExecutorService; 24 | import java.util.concurrent.Executors; 25 | import java.util.concurrent.ScheduledExecutorService; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | /** 29 | * A Job exector that runs function in a while loop 30 | */ 31 | public class ForeverRepeatedJobExecutor implements JobExecutor { 32 | 33 | private static final Logger logger = LoggerFactory.getLogger(ForeverRepeatedJobExecutor.class); 34 | private static final ScheduledExecutorService execService = Executors.newScheduledThreadPool( 35 | Configuration.getProperties().getInt("num_subscriber", 1)); 36 | private long interval; 37 | private Runnable runnable; 38 | 39 | public ForeverRepeatedJobExecutor(long interval, Runnable runnable) { 40 | this.setInterval(interval); 41 | this.setRunnable(runnable); 42 | } 43 | 44 | public long getInterval() { 45 | return interval; 46 | } 47 | 48 | protected void setInterval(long interval) { 49 | this.interval = interval; 50 | } 51 | 52 | public Runnable getRunnable() { 53 | return runnable; 54 | } 55 | 56 | protected void setRunnable(Runnable runnable) { 57 | this.runnable = runnable; 58 | } 59 | 60 | @Override 61 | public void execute() { 62 | ExecutorService executorService = Executors.newSingleThreadExecutor(); 63 | executorService.submit(() -> { 64 | execService.scheduleAtFixedRate(getRunnable(), 0, getInterval(), TimeUnit.MILLISECONDS); 65 | }); 66 | } 67 | 68 | protected void runOnce() { 69 | this.getRunnable().run(); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/job/JobConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.job; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | public final class JobConfig { 21 | 22 | @JsonProperty("isDisabled") 23 | private boolean isDisabled; 24 | 25 | 26 | public boolean isDisabled() { 27 | return isDisabled; 28 | } 29 | 30 | public void setDisabled(boolean disabled) { 31 | isDisabled = disabled; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/job/JobExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.job; 17 | 18 | /*** 19 | * Naive Job executor 20 | */ 21 | public interface JobExecutor { 22 | 23 | void execute(); 24 | } 25 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/job/JobInfoStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.job; 17 | 18 | /** 19 | * Abstract interfaces to retrieve the latest job run info. This is main used 20 | * to decide the right scheduled time 21 | */ 22 | public abstract class JobInfoStore { 23 | 24 | public abstract JobRunInfo getLatestRun(String jobType) throws Exception; 25 | 26 | public abstract void updateLatestRun(JobRunInfo info) throws Exception; 27 | 28 | public abstract boolean isJobDisabled(String jobType) throws Exception; 29 | } 30 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/job/JobRunInfo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.job; 17 | 18 | import java.util.Date; 19 | 20 | /** 21 | * On Job runinfo result 22 | */ 23 | public final class JobRunInfo { 24 | 25 | public String error; 26 | private Date startTime; 27 | private Date endTime; 28 | private boolean succeed; 29 | private int retryCount; 30 | private String jobName; 31 | private String node; 32 | 33 | public String getJobName() { 34 | return jobName; 35 | } 36 | 37 | public void setJobName(String jobName) { 38 | this.jobName = jobName; 39 | } 40 | 41 | public String getNode() { 42 | return node; 43 | } 44 | 45 | public void setNode(String node) { 46 | this.node = node; 47 | } 48 | 49 | public Date getStartTime() { 50 | return startTime; 51 | } 52 | 53 | public void setStartTime(Date startTime) { 54 | this.startTime = startTime; 55 | } 56 | 57 | public Date getEndTime() { 58 | return endTime; 59 | } 60 | 61 | public void setEndTime(Date endTime) { 62 | this.endTime = endTime; 63 | } 64 | 65 | public boolean isSucceed() { 66 | return succeed; 67 | } 68 | 69 | public void setSucceed(boolean succeed) { 70 | this.succeed = succeed; 71 | } 72 | 73 | public int getRetryCount() { 74 | return retryCount; 75 | } 76 | 77 | public void setRetryCount(int retryCount) { 78 | this.retryCount = retryCount; 79 | } 80 | 81 | public String getError() { 82 | return error; 83 | } 84 | 85 | public void setError(String error) { 86 | this.error = error; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/job/OwnershipDecider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.job; 17 | 18 | public interface OwnershipDecider { 19 | 20 | boolean isOwner(); 21 | 22 | String getNodeName(); 23 | } 24 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/AbstractEsInstanceFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.pinterest.soundwave.bean.EsInstance; 19 | 20 | import com.amazonaws.services.ec2.model.Instance; 21 | import com.fasterxml.jackson.databind.ObjectMapper; 22 | 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | public abstract class AbstractEsInstanceFactory { 27 | 28 | protected ObjectMapper mapper = new ObjectMapper(); 29 | 30 | public abstract EsInstance createFromEC2(Instance awsInstance) throws Exception; 31 | 32 | public void setCloudInstanceStore(CloudInstanceStore store) {} 33 | 34 | protected HashMap getAwsInstanceProperties(Instance awsInstance) throws Exception { 35 | HashMap map = mapper.readValue(mapper.writeValueAsString(awsInstance), HashMap.class); 36 | 37 | if (awsInstance.getMonitoring() != null && awsInstance.getMonitoring().getState() != null) { 38 | //Have to comply with the current AWS_V1 schema 39 | map.put("monitoring", awsInstance.getMonitoring().getState().toString()); 40 | } 41 | 42 | if (awsInstance.getPlacement() != null 43 | && awsInstance.getPlacement().getAvailabilityZone() != null) { 44 | //Be backward compatible for tools 45 | Map placement = (Map) map.get("placement"); 46 | if (placement != null) { 47 | placement.put("availability_zone", awsInstance.getPlacement().getAvailabilityZone()); 48 | } 49 | } 50 | return map; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/AwsStatus.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.amazonaws.services.ec2.model.InstanceStatus; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import java.util.List; 22 | 23 | public class AwsStatus { 24 | 25 | @JsonProperty("raw") 26 | private InstanceStatus raw; 27 | 28 | @JsonProperty("codes") 29 | private List codes; 30 | 31 | public InstanceStatus getRaw() { 32 | return raw; 33 | } 34 | 35 | public void setRaw(InstanceStatus raw) { 36 | this.raw = raw; 37 | } 38 | 39 | public List getCodes() { 40 | return codes; 41 | } 42 | 43 | public void setCodes(List codes) { 44 | this.codes = codes; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/AwsUtilities.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.amazonaws.services.ec2.model.Instance; 19 | import com.amazonaws.services.ec2.model.Tag; 20 | import org.apache.commons.lang.StringUtils; 21 | 22 | import java.util.List; 23 | 24 | public final class AwsUtilities { 25 | public static Tag getAwsTag(Instance awsInstance, String tagName) { 26 | List tags = awsInstance.getTags(); 27 | java.util.Optional tag = 28 | tags.stream().filter(t -> StringUtils.equals(t.getKey(), tagName)).findFirst(); 29 | return tag.isPresent() ? tag.get() : null; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/BasicEsInstanceFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.pinterest.soundwave.bean.EsInstance; 19 | 20 | import com.amazonaws.services.ec2.model.Instance; 21 | import com.google.common.base.Preconditions; 22 | import org.joda.time.DateTime; 23 | import org.joda.time.DateTimeZone; 24 | 25 | import java.util.Date; 26 | 27 | public final class BasicEsInstanceFactory extends AbstractEsInstanceFactory{ 28 | 29 | @Override 30 | public EsInstance createFromEC2(Instance awsInstance) throws Exception { 31 | Preconditions.checkNotNull(awsInstance); 32 | 33 | EsInstance esInstance = new EsInstance(); 34 | 35 | esInstance.setId(awsInstance.getInstanceId()); 36 | esInstance.setState(awsInstance.getState().getName()); 37 | esInstance.setLocation(awsInstance.getPlacement().getAvailabilityZone()); 38 | 39 | //Region=location-last char. This is what CMDBV1 and people on internet do. 40 | //There should be a better way. Right now, keep as what it is 41 | esInstance.setRegion( 42 | esInstance.getLocation().substring(0, esInstance.getLocation().length() - 1)); 43 | esInstance.setAwsLaunchTime(awsInstance.getLaunchTime()); 44 | esInstance.setSubnetId(awsInstance.getSubnetId()); 45 | esInstance.setVpcId(awsInstance.getVpcId()); 46 | 47 | 48 | //Convert AWS instance to a map of property bags and save it. 49 | esInstance.getCloud() 50 | .put("aws", getAwsInstanceProperties(awsInstance)); 51 | 52 | Date utcNow = DateTime.now(DateTimeZone.UTC).toDate(); 53 | esInstance.setCreatedTime(utcNow); 54 | esInstance.setUpdatedTime(utcNow); 55 | 56 | return esInstance; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/CannotRetrieveFromEC2Exception.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | public class CannotRetrieveFromEC2Exception extends Exception { 19 | 20 | public CannotRetrieveFromEC2Exception(String message) { 21 | super(message); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/CloudInstanceStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.amazonaws.regions.Region; 19 | import com.amazonaws.services.ec2.AmazonEC2Client; 20 | import com.amazonaws.services.ec2.model.AvailabilityZone; 21 | import com.amazonaws.services.ec2.model.Instance; 22 | import com.amazonaws.services.ec2.model.Tag; 23 | import com.amazonaws.services.ec2.model.InstanceAttribute; 24 | import com.amazonaws.services.ec2.model.ReservedInstances; 25 | 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | public interface CloudInstanceStore { 30 | 31 | Instance getInstance(String instanceId) throws Exception; 32 | 33 | InstanceAttribute getInstanceAttribute(String instanceId, String attributeName) throws Exception; 34 | 35 | List getInstances(Region region) throws Exception; 36 | 37 | Map getUserData(String instanceId) throws Exception; 38 | 39 | List getAvailabilityZones(Region region) throws Exception; 40 | 41 | List getInstancesForZone(AvailabilityZone zone, AmazonEC2Client client) 42 | throws Exception; 43 | 44 | Map> getInstancesGroupedByZone(Region region) throws Exception; 45 | 46 | Map> getInstancesMapForZone( 47 | AvailabilityZone zone, AmazonEC2Client client) throws Exception; 48 | 49 | Map> getReservedInstancesGroupedByZone(Region region) 50 | throws Exception; 51 | 52 | Map> getReservedInstancesForZone( 53 | AvailabilityZone zone, AmazonEC2Client client) throws Exception; 54 | 55 | void setTagsForInstances(List instanceIds, List tags) throws Exception; 56 | 57 | List getReservedInstances(Region region); 58 | } 59 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/DailySnapshotStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.pinterest.soundwave.bean.EsDailySnapshotInstance; 19 | 20 | import java.util.Iterator; 21 | import java.util.List; 22 | 23 | public interface DailySnapshotStore { 24 | 25 | EsDailySnapshotInstance getInstanceById(String instanceId) throws Exception; 26 | 27 | long updateOrInsert(EsDailySnapshotInstance instance) throws Exception; 28 | 29 | long update(EsDailySnapshotInstance instance) throws Exception; 30 | 31 | Iterator getSnapshotInstances() throws Exception; 32 | 33 | void bulkInsert(List instances) throws Exception; 34 | 35 | void bulkUpdate(List instances) throws Exception; 36 | } 37 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/DailySnapshotStoreFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import org.joda.time.DateTime; 19 | 20 | public interface DailySnapshotStoreFactory { 21 | 22 | DailySnapshotStore getDailyStore(DateTime time); 23 | } 24 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/Ec2InstanceNotificationDetail.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | /** 21 | * The detail of the instance notification 22 | */ 23 | public class Ec2InstanceNotificationDetail { 24 | 25 | @JsonProperty("state") 26 | private String state; 27 | 28 | @JsonProperty("instance-id") 29 | private String instanceId; 30 | 31 | 32 | public String getState() { 33 | return state; 34 | } 35 | 36 | public void setState(String state) { 37 | this.state = state; 38 | } 39 | 40 | public String getInstanceId() { 41 | return instanceId; 42 | } 43 | 44 | public void setInstanceId(String instanceId) { 45 | this.instanceId = instanceId; 46 | } 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/Ec2NotificationHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.pinterest.soundwave.bean.EsInstance; 19 | 20 | import java.util.function.BiConsumer; 21 | 22 | public interface Ec2NotificationHandler { 23 | 24 | MessageProcessingResult processEvent(NotificationEvent event) throws Exception; 25 | 26 | void registerOnCreationSuccessEventHandler( 27 | BiConsumer instance); 28 | } 29 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/MessageProcessingResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import org.joda.time.DateTime; 19 | import org.joda.time.DateTimeZone; 20 | 21 | import java.util.Date; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | 26 | public class MessageProcessingResult { 27 | 28 | public static final String EVENTCREATEDFIELD = "EventCreatedTime"; 29 | private String instanceId; 30 | private String messageType; 31 | private long duration; 32 | private boolean succeed; 33 | private MessageProcessingError error; 34 | private Date timestamp; 35 | private Map info; 36 | private long toSqsTime; 37 | 38 | public MessageProcessingResult(NotificationEvent event) { 39 | this.setInfo(new HashMap<>()); 40 | this.setMessageType(event.getDetail().getState()); 41 | this.setInstanceId(event.getDetail().getInstanceId()); 42 | this.putEventCreatedTime(event.getTime()); 43 | this.timestamp = DateTime.now(DateTimeZone.UTC).toDate(); 44 | this.error = MessageProcessingError.NOERROR; 45 | 46 | } 47 | 48 | public MessageProcessingError getError() { 49 | return error; 50 | } 51 | 52 | public void setError(MessageProcessingError error) { 53 | this.error = error; 54 | setSucceed(error == MessageProcessingError.NOERROR); 55 | } 56 | 57 | public String getInstanceId() { 58 | return instanceId; 59 | } 60 | 61 | public void setInstanceId(String instanceId) { 62 | this.instanceId = instanceId; 63 | } 64 | 65 | public String getMessageType() { 66 | return messageType; 67 | } 68 | 69 | public void setMessageType(String messageType) { 70 | this.messageType = messageType; 71 | } 72 | 73 | public Map getInfo() { 74 | return info; 75 | } 76 | 77 | public void setInfo(Map info) { 78 | this.info = info; 79 | } 80 | 81 | public boolean isSucceed() { 82 | return succeed; 83 | } 84 | 85 | public void setSucceed(boolean succeed) { 86 | this.succeed = succeed; 87 | } 88 | 89 | public long getDuration() { 90 | return duration; 91 | } 92 | 93 | public void setDuration(long duration) { 94 | this.duration = duration; 95 | } 96 | 97 | public Date getTimestamp() { 98 | return timestamp; 99 | } 100 | 101 | public void setTimestamp(Date timestamp) { 102 | this.timestamp = timestamp; 103 | } 104 | 105 | public void putEventCreatedTime(Date date) { 106 | this.getInfo().put(EVENTCREATEDFIELD, date); 107 | } 108 | 109 | public long getToSqsTime() { 110 | return toSqsTime; 111 | } 112 | 113 | public void setToSqsTime(long toSqsTime) { 114 | this.toSqsTime = toSqsTime; 115 | } 116 | 117 | public enum MessageProcessingError { 118 | NOERROR, 119 | NO_NAME_TAG, 120 | NOT_EXIST_IN_EC_2, 121 | NOT_EXIST_IN_ES, 122 | QUOBLE_NAME_NOT_AVAILABLE, 123 | STILL_PENDING, 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/MissingNameTagException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | /** 19 | * The exception indicates that there is no name tag on 20 | * the EC2 instance. 21 | */ 22 | public class MissingNameTagException extends Exception { 23 | 24 | public MissingNameTagException(String message) { 25 | super(message); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/NotificationEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.amazonaws.services.sqs.model.Message; 19 | import com.fasterxml.jackson.annotation.JsonIgnore; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | import java.util.Date; 23 | 24 | /** 25 | * The event message definition from the SQS notification 26 | */ 27 | public class NotificationEvent { 28 | 29 | @JsonProperty("account") 30 | private String account; 31 | 32 | @JsonProperty("region") 33 | private String region; 34 | 35 | @JsonProperty("detail") 36 | private Ec2InstanceNotificationDetail detail; 37 | 38 | @JsonProperty("detail-type") 39 | private String detailType; 40 | 41 | @JsonProperty("source") 42 | private String source; 43 | 44 | @JsonProperty("version") 45 | private long version; 46 | 47 | @JsonProperty("time") 48 | private Date time; 49 | 50 | @JsonProperty("id") 51 | private String id; 52 | 53 | @JsonProperty("resources") 54 | private String[] resources; 55 | 56 | @JsonIgnore 57 | private Message sourceMessage; 58 | 59 | private Date sqsSentTime; 60 | 61 | public String getDetailType() { 62 | return detailType; 63 | } 64 | 65 | public void setDetailType(String detailType) { 66 | this.detailType = detailType; 67 | } 68 | 69 | public String getSource() { 70 | return source; 71 | } 72 | 73 | public void setSource(String source) { 74 | this.source = source; 75 | } 76 | 77 | public long getVersion() { 78 | return version; 79 | } 80 | 81 | public void setVersion(long version) { 82 | this.version = version; 83 | } 84 | 85 | public Date getTime() { 86 | return time; 87 | } 88 | 89 | public void setTime(Date time) { 90 | this.time = time; 91 | } 92 | 93 | public String getId() { 94 | return id; 95 | } 96 | 97 | public void setId(String id) { 98 | this.id = id; 99 | } 100 | 101 | public String[] getResources() { 102 | return resources; 103 | } 104 | 105 | public void setResources(String[] resources) { 106 | this.resources = resources; 107 | } 108 | 109 | public Date getSqsSentTime() { 110 | return sqsSentTime; 111 | } 112 | 113 | public void setSqsSentTime(Date sqsSentTime) { 114 | this.sqsSentTime = sqsSentTime; 115 | } 116 | 117 | public Message getSourceMessage() { 118 | return sourceMessage; 119 | } 120 | 121 | public void setSourceMessage(Message sourceMessage) { 122 | this.sourceMessage = sourceMessage; 123 | } 124 | 125 | public String getAccount() { 126 | return account; 127 | } 128 | 129 | public void setAccount(String account) { 130 | this.account = account; 131 | } 132 | 133 | public String getRegion() { 134 | return region; 135 | } 136 | 137 | public void setRegion(String region) { 138 | this.region = region; 139 | } 140 | 141 | public Ec2InstanceNotificationDetail getDetail() { 142 | return detail; 143 | } 144 | 145 | public void setDetail(Ec2InstanceNotificationDetail detail) { 146 | this.detail = detail; 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/aws/SqsClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.aws; 17 | 18 | import com.amazonaws.services.sqs.AmazonSQSClient; 19 | import com.amazonaws.services.sqs.model.Message; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import com.fasterxml.jackson.databind.SerializationFeature; 22 | import com.google.common.base.Preconditions; 23 | import org.apache.commons.lang.exception.ExceptionUtils; 24 | import org.joda.time.DateTime; 25 | import org.joda.time.DateTimeZone; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.util.Map; 30 | 31 | 32 | public class SqsClient { 33 | 34 | private static final Logger logger = LoggerFactory.getLogger(SqsClient.class); 35 | private static final Logger sqsEventLogger = LoggerFactory.getLogger("sqsEventLogger"); 36 | private static final Logger 37 | messageProcessingLogger = 38 | LoggerFactory.getLogger("messageProcessingLog"); 39 | 40 | private AmazonSQSClient client; 41 | private Ec2NotificationHandler handler; 42 | private ObjectMapper objectMapper; 43 | 44 | 45 | public SqsClient(Ec2NotificationHandler handler) { 46 | Preconditions.checkNotNull(handler); 47 | objectMapper = 48 | new ObjectMapper().configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); 49 | this.handler = handler; 50 | } 51 | 52 | public boolean processMessage(Message msg) { 53 | try { 54 | MessageProcessingResult result = null; 55 | NotificationEvent event = 56 | objectMapper.readValue(msg.getBody(), NotificationEvent.class); 57 | 58 | logger.info("Receive event {} with state {} created at {}", 59 | event.getDetail().getInstanceId(), 60 | event.getDetail().getState(), event.getTime()); 61 | 62 | sqsEventLogger.info(objectMapper.writeValueAsString(event)); 63 | Map attributes = msg.getAttributes(); 64 | 65 | if (attributes.containsKey("SentTimestamp")) { 66 | DateTime sentTime = new DateTime(Long.parseLong(attributes.get("SentTimestamp")), 67 | DateTimeZone.UTC); 68 | event.setSqsSentTime(sentTime.toDate()); 69 | } 70 | 71 | if (handler != null) { 72 | result = handler.processEvent(event); 73 | if (result != null) { 74 | messageProcessingLogger.info(objectMapper.writeValueAsString(result)); 75 | return result.isSucceed(); 76 | } 77 | } 78 | } catch (Exception e) { 79 | logger.warn("Error Process message:{}", ExceptionUtils.getRootCauseMessage(e)); 80 | } 81 | return false; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/EsAwsStatus.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import com.pinterest.soundwave.aws.AwsStatus; 19 | 20 | import com.fasterxml.jackson.annotation.JsonIgnore; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | 23 | /** 24 | * Bean represents the AwsStatus of an instance in Es 25 | */ 26 | public class EsAwsStatus implements EsDocument { 27 | 28 | @JsonProperty("id") 29 | private String id; 30 | 31 | @JsonProperty("aws_status") 32 | private AwsStatus awsStatus; 33 | 34 | @Override 35 | public String getId() { 36 | return id; 37 | } 38 | 39 | @Override 40 | public void setId(String id) { 41 | this.id = id; 42 | } 43 | 44 | @JsonIgnore 45 | private long version; 46 | 47 | public long getVersion() { 48 | return version; 49 | } 50 | 51 | public void setVersion(long version) { 52 | this.version = version; 53 | } 54 | 55 | public AwsStatus getAwsStatus() { 56 | return awsStatus; 57 | } 58 | 59 | public void setAwsStatus(AwsStatus awsStatus) { 60 | this.awsStatus = awsStatus; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/EsDocument.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | public interface EsDocument { 19 | 20 | String getId(); 21 | 22 | void setId(String id); 23 | 24 | long getVersion(); 25 | 26 | void setVersion(long version); 27 | } 28 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/EsQueryResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import java.util.HashMap; 22 | 23 | 24 | public class EsQueryResult extends HashMap implements EsDocument { 25 | 26 | @JsonProperty("id") 27 | private String id; 28 | 29 | @JsonIgnore 30 | private long version; 31 | 32 | @Override 33 | public String getId() { 34 | return id; 35 | } 36 | 37 | @Override 38 | public void setId(String id) { 39 | this.id = id; 40 | } 41 | 42 | @Override 43 | public long getVersion() { 44 | return version; 45 | } 46 | 47 | @Override 48 | public void setVersion(long version) { 49 | this.version = version; 50 | } 51 | 52 | 53 | } 54 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/EsStatus.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import org.apache.commons.lang.StringUtils; 19 | 20 | public enum EsStatus { 21 | SUCCESS("success"), 22 | TIMEOUT("timeout"), 23 | ERROR("error"), 24 | UNKNOWN("unknown"); 25 | 26 | private final String text; 27 | 28 | EsStatus(final String text) { 29 | this.text = text; 30 | } 31 | 32 | public boolean isStatus(String val) { 33 | return StringUtils.equals(val, this.text); 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return text; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/EsStoreMappingProperty.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import java.lang.annotation.ElementType; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.RetentionPolicy; 21 | import java.lang.annotation.Target; 22 | 23 | @Retention(RetentionPolicy.RUNTIME) 24 | @Target(ElementType.FIELD) 25 | public @interface EsStoreMappingProperty { 26 | 27 | String value() default ""; 28 | 29 | Class store(); 30 | 31 | boolean ignore() default false; 32 | } 33 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/InvalidStateTransitionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | public class InvalidStateTransitionException extends Exception { 19 | 20 | public InvalidStateTransitionException(String message) { 21 | super(message); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/PinterestEsInstance.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import java.util.Map; 21 | 22 | public class PinterestEsInstance extends EsInstance { 23 | 24 | @JsonProperty("service_mapping") 25 | private String[] serviceMappings; 26 | 27 | @JsonProperty("svc_tag") 28 | private String[] serviceTag; 29 | 30 | @JsonProperty("sys_tag") 31 | private String[] sysTag; 32 | 33 | @JsonProperty("usage_tag") 34 | private String[] usageTag; 35 | 36 | @JsonProperty("nodepool") 37 | private String nodePool; 38 | 39 | @JsonProperty("deployment") 40 | private String deployment; 41 | 42 | 43 | @JsonProperty("facts") 44 | private Map facts; 45 | 46 | @JsonProperty("pkgs") 47 | private Map pkgs; 48 | 49 | @JsonProperty("config") 50 | private EsInstanceConfig config; 51 | 52 | 53 | @JsonProperty("defunct_count") 54 | private int defunctCount; 55 | 56 | @JsonProperty("pc") 57 | private int pc; 58 | 59 | public String getNodePool() { 60 | return nodePool; 61 | } 62 | 63 | public void setNodePool(String nodePool) { 64 | this.nodePool = nodePool; 65 | } 66 | 67 | public Map getFacts() { 68 | return facts; 69 | } 70 | 71 | public void setFacts(Map facts) { 72 | this.facts = facts; 73 | } 74 | 75 | public Map getPkgs() { 76 | return pkgs; 77 | } 78 | 79 | public void setPkgs(Map pkgs) { 80 | this.pkgs = pkgs; 81 | } 82 | 83 | 84 | public EsInstanceConfig getConfig() { 85 | return config; 86 | } 87 | 88 | public void setConfig(EsInstanceConfig config) { 89 | this.config = config; 90 | } 91 | 92 | 93 | public String[] getServiceMappings() { 94 | return serviceMappings; 95 | } 96 | 97 | public void setServiceMappings(String[] serviceMappings) { 98 | this.serviceMappings = serviceMappings; 99 | } 100 | 101 | public String[] getServiceTag() { 102 | return serviceTag; 103 | } 104 | 105 | public void setServiceTag(String[] serviceTag) { 106 | this.serviceTag = serviceTag; 107 | } 108 | 109 | public String[] getSysTag() { 110 | return sysTag; 111 | } 112 | 113 | public void setSysTag(String[] sysTag) { 114 | this.sysTag = sysTag; 115 | } 116 | 117 | public String[] getUsageTag() { 118 | return usageTag; 119 | } 120 | 121 | public void setUsageTag(String[] usageTag) { 122 | this.usageTag = usageTag; 123 | } 124 | 125 | public String getDeployment() { 126 | return deployment; 127 | } 128 | 129 | public void setDeployment(String deployment) { 130 | this.deployment = deployment; 131 | } 132 | 133 | public int getDefunctCount() { 134 | return defunctCount; 135 | } 136 | 137 | public void setDefunctCount(int defunctCount) { 138 | this.defunctCount = defunctCount; 139 | } 140 | 141 | public int getPc() { 142 | return pc; 143 | } 144 | 145 | public void setPc(int pc) { 146 | this.pc = pc; 147 | } 148 | 149 | } 150 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/bean/State.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import org.apache.commons.lang.StringUtils; 19 | 20 | public enum State { 21 | RUNNING("running"), 22 | TERMINATED("terminated"), 23 | SHUTTINGDOWN("shutting-down"), 24 | STOPPED("stopped"), 25 | PENDING("pending"), 26 | DEFUNCT("defunct"); 27 | 28 | private final String text; 29 | 30 | private State(final String text) { 31 | this.text = text; 32 | } 33 | 34 | public boolean isState(String val) { 35 | return StringUtils.equals(val, this.text); 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return text; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/BulkRequestExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | import org.elasticsearch.action.ActionRequest; 19 | import org.elasticsearch.action.bulk.BulkProcessor; 20 | import org.elasticsearch.action.bulk.BulkResponse; 21 | import org.elasticsearch.client.Client; 22 | import org.elasticsearch.common.unit.ByteSizeUnit; 23 | import org.elasticsearch.common.unit.ByteSizeValue; 24 | import org.elasticsearch.common.unit.TimeValue; 25 | 26 | import java.util.concurrent.TimeUnit; 27 | 28 | /** 29 | * A simple wrapper for ElasticsSearch Bulk request 30 | */ 31 | public class BulkRequestExecutor implements BulkProcessor.Listener { 32 | 33 | private BulkProcessor processor; 34 | private BulkResponse response; 35 | private Throwable throwable; 36 | 37 | public BulkRequestExecutor(Client client) { 38 | processor = BulkProcessor.builder(client, this).build(); 39 | } 40 | 41 | public BulkRequestExecutor(Client client, int bulkActions, long maxSizeMegaBytes, 42 | int concurrentRequest, 43 | TimeValue flushInterval) { 44 | processor = 45 | BulkProcessor.builder(client, this).setBulkActions(bulkActions) 46 | .setBulkSize(new ByteSizeValue(maxSizeMegaBytes, ByteSizeUnit.MB)) 47 | .setConcurrentRequests(concurrentRequest).setFlushInterval(flushInterval).build(); 48 | } 49 | 50 | @Override 51 | public void beforeBulk(long l, org.elasticsearch.action.bulk.BulkRequest bulkRequest) { 52 | this.response = null; 53 | this.throwable = null; 54 | } 55 | 56 | @Override 57 | public void afterBulk(long l, org.elasticsearch.action.bulk.BulkRequest bulkRequest, 58 | BulkResponse bulkResponse) { 59 | this.response = bulkResponse; 60 | } 61 | 62 | @Override 63 | public void afterBulk(long l, org.elasticsearch.action.bulk.BulkRequest bulkRequest, 64 | Throwable throwable) { 65 | this.throwable = throwable; 66 | } 67 | 68 | public BulkResponse execute() throws Exception { 69 | processor.awaitClose(300, TimeUnit.SECONDS); 70 | if (response == null && throwable != null) { 71 | throw new Exception(throwable); 72 | } 73 | 74 | return response; 75 | } 76 | 77 | public void add(ActionRequest request) { 78 | this.processor.add(request); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/CmdbInstanceStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | 19 | import com.pinterest.soundwave.bean.EsAwsStatus; 20 | import com.pinterest.soundwave.bean.EsDailySnapshotInstance; 21 | import com.pinterest.soundwave.bean.EsInstance; 22 | import com.pinterest.soundwave.bean.EsQueryResult; 23 | import com.pinterest.soundwave.pinterest.EsInstanceTags; 24 | import com.pinterest.soundwave.pinterest.EsMetaData; 25 | import com.pinterest.soundwave.pinterest.EsNameMetaData; 26 | 27 | import com.amazonaws.regions.Region; 28 | 29 | import java.util.Date; 30 | import java.util.HashMap; 31 | import java.util.Iterator; 32 | import java.util.List; 33 | import java.util.Map; 34 | 35 | public interface CmdbInstanceStore { 36 | 37 | E getInstanceById(String instanceId) throws Exception; 38 | 39 | long updateOrInsertInstance(EsInstance instance) throws Exception; 40 | 41 | long update(EsInstance instance) throws Exception; 42 | 43 | Iterator getRunningAndRecentTerminatedInstances(int days) 44 | throws Exception; 45 | 46 | Iterator getRunningInstances() throws Exception; 47 | 48 | Iterator getRunningInstances(Region region) throws Exception; 49 | 50 | Iterator getRunningInstances(Region region, String[] fields) throws Exception; 51 | 52 | Iterator getRecentlyTerminatedInstances(Region region, int days) throws Exception; 53 | 54 | Iterator getRunningAndTerminatedInstanceTags(int terminatedDays) throws Exception; 55 | 56 | Iterator getRunningAndTerminatedAwsStatus(Region region, int days) throws Exception; 57 | 58 | void bulkInsert(List instances) throws Exception; 59 | 60 | void bulkUpdate(List instances) throws Exception; 61 | 62 | void bulkUpdateInstanceTags(List instances) throws Exception; 63 | 64 | void bulkUpdateAwsStatus(List awsStatuses) throws Exception; 65 | 66 | EsAwsStatus getAwsStatus(String instanceId) throws Exception; 67 | 68 | String checkStatus() throws Exception; 69 | 70 | Iterator getInstanceCreatedBetween(Date start, Date end) throws Exception; 71 | 72 | Iterator getMetaData(String fieldName, String fieldValue) throws Exception; 73 | 74 | Iterator query(String query, String[] includeFields) throws Exception; 75 | 76 | Iterator getRunningInstancesWithFields(String[] includeFields) throws Exception; 77 | 78 | Iterator getMetaDataByName(String field, String name) throws Exception; 79 | 80 | Map getAggregations(List aggregationParams) throws Exception; 81 | 82 | Class getInstanceClass(); 83 | 84 | } 85 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/EsBulkResponseSummary.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | import org.apache.commons.lang.StringUtils; 19 | import org.elasticsearch.action.bulk.BulkItemResponse; 20 | import org.elasticsearch.action.bulk.BulkResponse; 21 | import org.elasticsearch.action.index.IndexResponse; 22 | import org.elasticsearch.action.update.UpdateResponse; 23 | 24 | import java.util.HashMap; 25 | import java.util.List; 26 | import java.util.Map; 27 | 28 | public class EsBulkResponseSummary { 29 | 30 | private Map succeeded = new HashMap<>(); 31 | private Map failed = new HashMap<>(); 32 | 33 | public EsBulkResponseSummary(List responses) { 34 | for (BulkResponse br : responses) { 35 | for (BulkItemResponse itemResponse : br.getItems()) { 36 | if (itemResponse.isFailed()) { 37 | failed.put(getItemId(itemResponse), itemResponse.getFailureMessage()); 38 | } else { 39 | succeeded.put(getItemId(itemResponse), itemResponse.getOpType()); 40 | } 41 | } 42 | } 43 | } 44 | 45 | public Map getSucceeded() { 46 | return succeeded; 47 | } 48 | 49 | public Map getFailed() { 50 | return failed; 51 | } 52 | 53 | private String getItemId(BulkItemResponse response) { 54 | String id = response.getId(); 55 | if (response.getResponse() instanceof UpdateResponse) { 56 | id = ((UpdateResponse) response.getResponse()).getId(); 57 | } else if (response.getResponse() instanceof IndexResponse) { 58 | id = ((IndexResponse) response.getResponse()).getId(); 59 | } 60 | return id; 61 | } 62 | 63 | public String getFailedMessage(int max) { 64 | int count = 0; 65 | StringBuilder stringBuilder = new StringBuilder(); 66 | for (Map.Entry entry : failed.entrySet()) { 67 | if (StringUtils.indexOf(entry.getValue(), "VersionConflictEngineException") < 0) { 68 | stringBuilder.append(String.format("id:%s error:%s", entry.getKey(), entry.getValue())); 69 | count++; 70 | if (count >= max) { 71 | break; 72 | } 73 | } 74 | } 75 | return stringBuilder.toString(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/EsDailySnapshotStoreFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | import com.pinterest.soundwave.aws.DailySnapshotStore; 19 | import com.pinterest.soundwave.aws.DailySnapshotStoreFactory; 20 | 21 | import org.joda.time.DateTime; 22 | 23 | import java.util.concurrent.ConcurrentHashMap; 24 | 25 | public class EsDailySnapshotStoreFactory implements DailySnapshotStoreFactory { 26 | 27 | private static ConcurrentHashMap 28 | s_Cache = 29 | new ConcurrentHashMap<>(); 30 | 31 | @Override 32 | public DailySnapshotStore getDailyStore(DateTime day) { 33 | String key = day.toString("yyyy-MM-dd"); 34 | EsDailySnapshotStore ret = s_Cache.get(key); 35 | if (ret == null) { 36 | ret = new EsDailySnapshotStore(day); 37 | s_Cache.put(key, ret); 38 | } 39 | return ret; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/EsIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | import com.pinterest.soundwave.utils.ThrowingFunction; 19 | 20 | import com.google.common.base.Preconditions; 21 | import org.apache.commons.lang.NotImplementedException; 22 | 23 | import java.util.Iterator; 24 | import java.util.List; 25 | import java.util.NoSuchElementException; 26 | 27 | public class EsIterator implements Iterator { 28 | 29 | private ScrollableResponse> response; 30 | private int currentListCursor; 31 | private ThrowingFunction>, ScrollableResponse>> 32 | retrieveNextFunction; 33 | 34 | 35 | public EsIterator(ScrollableResponse> response, 36 | ThrowingFunction>, ScrollableResponse>> 37 | retrieveCallback) { 38 | Preconditions.checkNotNull(response); 39 | 40 | this.retrieveNextFunction = retrieveCallback; 41 | this.response = response; 42 | this.currentListCursor = 0; 43 | } 44 | 45 | public ScrollableResponse> getResponse() { 46 | return response; 47 | } 48 | 49 | public void setResponse(ScrollableResponse> response) { 50 | this.response = response; 51 | } 52 | 53 | public boolean hasNext() { 54 | boolean ret; 55 | if (response.isScrollToEnd()) { 56 | ret = currentListCursor < response.getValue().size(); 57 | } else { 58 | if (currentListCursor >= response.getValue().size()) { 59 | try { 60 | response = this.retrieveNextFunction.apply(response); 61 | currentListCursor = 0; 62 | ret = currentListCursor < response.getValue().size(); 63 | } catch (Exception e) { 64 | throw new RuntimeException(e); 65 | } 66 | } else { 67 | ret = true; 68 | } 69 | } 70 | return ret; 71 | } 72 | 73 | public T next() { 74 | if (hasNext()) { 75 | return response.getValue().get(currentListCursor++); 76 | } else { 77 | throw new NoSuchElementException(); 78 | } 79 | } 80 | 81 | public void remove() { 82 | throw new NotImplementedException("remove is not a supported action"); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/InstanceCounterStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | 19 | import com.pinterest.soundwave.bean.EsInstanceCountRecord; 20 | 21 | import java.util.Date; 22 | import java.util.Iterator; 23 | import java.util.List; 24 | 25 | public interface InstanceCounterStore { 26 | void bulkInsert(List records) throws Exception; 27 | 28 | Iterator getCountRecordsByDate(Date date) throws Exception; 29 | } 30 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/PinterestCmdbInstanceStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | import com.pinterest.soundwave.pinterest.EsFacts; 19 | import com.pinterest.soundwave.pinterest.EsFactsAndPackages; 20 | import com.pinterest.soundwave.pinterest.EsInstanceTags; 21 | import com.pinterest.soundwave.pinterest.EsPackages; 22 | 23 | import java.util.Iterator; 24 | import java.util.List; 25 | 26 | public interface PinterestCmdbInstanceStore { 27 | 28 | EsFactsAndPackages getFactsAndPackagesById(String instanceId) throws Exception; 29 | 30 | long updateOrInsertFactsAndPackages(EsFactsAndPackages factsAndPackages) throws Exception; 31 | 32 | EsFacts getFactsById(String instanceId) throws Exception; 33 | 34 | long updateFacts(EsFacts facts) throws Exception; 35 | 36 | EsPackages getPackagesById(String instanceId) throws Exception; 37 | 38 | long updatePackages(EsPackages pkgs) throws Exception; 39 | 40 | Iterator getMissingServiceMappingInstances() throws Exception; 41 | 42 | List getDeployments() throws Exception; 43 | 44 | List getNodePools() throws Exception; 45 | 46 | } 47 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/elasticsearch/ScrollableResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | public class ScrollableResponse { 19 | 20 | private T value; 21 | private String continousToken; 22 | private boolean scrollToEnd; 23 | 24 | 25 | public ScrollableResponse() { 26 | 27 | } 28 | 29 | public T getValue() { 30 | return value; 31 | } 32 | 33 | public void setValue(T value) { 34 | this.value = value; 35 | } 36 | 37 | public String getContinousToken() { 38 | return continousToken; 39 | } 40 | 41 | public void setContinousToken(String continousToken) { 42 | this.continousToken = continousToken; 43 | } 44 | 45 | 46 | public boolean isScrollToEnd() { 47 | return scrollToEnd; 48 | } 49 | 50 | public void setScrollToEnd(boolean scrollToEnd) { 51 | this.scrollToEnd = scrollToEnd; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/EsFacts.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnore; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | public class EsFacts { 26 | 27 | @JsonProperty("id") 28 | private String id; 29 | 30 | @JsonProperty("facts") 31 | private Map facts = new HashMap<>(); 32 | 33 | @JsonIgnore 34 | private long version; 35 | 36 | public long getVersion() { 37 | return version; 38 | } 39 | 40 | public void setVersion(long version) { 41 | this.version = version; 42 | } 43 | 44 | public String getId() { 45 | return id; 46 | } 47 | 48 | public void setId(String id) { 49 | this.id = id; 50 | } 51 | 52 | public Map getFacts() { 53 | return facts; 54 | } 55 | 56 | public void setFacts(Map facts) { 57 | this.facts = facts; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/EsFactsAndPackages.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | import com.fasterxml.jackson.annotation.JsonProperty; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | public class EsFactsAndPackages { 24 | 25 | @JsonProperty("id") 26 | private String id; 27 | 28 | @JsonProperty("facts") 29 | private Map facts = new HashMap<>(); 30 | 31 | @JsonProperty("pkgs") 32 | private Map pkgs = new HashMap<>(); 33 | 34 | public String getId() { 35 | return id; 36 | } 37 | 38 | public void setId(String id) { 39 | this.id = id; 40 | } 41 | 42 | public Map getFacts() { 43 | return facts; 44 | } 45 | 46 | public void setFacts(Map facts) { 47 | this.facts = facts; 48 | } 49 | 50 | public Map getPkgs() { 51 | return pkgs; 52 | } 53 | 54 | public void setPkgs(Map pkgs) { 55 | this.pkgs = pkgs; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/EsInstanceTags.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | import com.pinterest.soundwave.bean.EsDocument; 19 | import com.pinterest.soundwave.bean.EsInstanceConfig; 20 | 21 | import com.fasterxml.jackson.annotation.JsonIgnore; 22 | import com.fasterxml.jackson.annotation.JsonProperty; 23 | 24 | import java.util.Date; 25 | 26 | /** 27 | * A bean class represent a instance and its tags 28 | */ 29 | public class EsInstanceTags implements EsDocument { 30 | 31 | @JsonProperty("service_mapping") 32 | private String[] serviceMappings; 33 | 34 | @JsonProperty("svc_tag") 35 | private String[] serviceTag; 36 | 37 | @JsonProperty("sys_tag") 38 | private String[] sysTag; 39 | 40 | @JsonProperty("usage_tag") 41 | private String[] usageTag; 42 | 43 | @JsonProperty("id") 44 | private String id; 45 | 46 | @JsonProperty("config") 47 | private EsInstanceConfig config; 48 | 49 | @JsonProperty("updated_time") 50 | private Date updatedTime; 51 | 52 | @JsonProperty("created_time") 53 | private Date createdTime; 54 | 55 | @JsonIgnore 56 | private long version; 57 | 58 | public long getVersion() { 59 | return version; 60 | } 61 | 62 | public void setVersion(long version) { 63 | this.version = version; 64 | } 65 | 66 | public EsInstanceConfig getConfig() { 67 | return config; 68 | } 69 | 70 | public void setConfig(EsInstanceConfig config) { 71 | this.config = config; 72 | } 73 | 74 | public Date getUpdatedTime() { 75 | return updatedTime; 76 | } 77 | 78 | public void setUpdatedTime(Date updatedTime) { 79 | this.updatedTime = updatedTime; 80 | } 81 | 82 | public String[] getServiceMappings() { 83 | return serviceMappings; 84 | } 85 | 86 | public void setServiceMappings(String[] serviceMappings) { 87 | this.serviceMappings = serviceMappings; 88 | } 89 | 90 | public String[] getServiceTag() { 91 | return serviceTag; 92 | } 93 | 94 | public void setServiceTag(String[] serviceTag) { 95 | this.serviceTag = serviceTag; 96 | } 97 | 98 | public String[] getSysTag() { 99 | return sysTag; 100 | } 101 | 102 | public void setSysTag(String[] sysTag) { 103 | this.sysTag = sysTag; 104 | } 105 | 106 | public String[] getUsageTag() { 107 | return usageTag; 108 | } 109 | 110 | public void setUsageTag(String[] usageTag) { 111 | this.usageTag = usageTag; 112 | } 113 | 114 | 115 | public Date getCreatedTime() { 116 | return createdTime; 117 | } 118 | 119 | public void setCreatedTime(Date createdTime) { 120 | this.createdTime = createdTime; 121 | } 122 | 123 | 124 | @Override 125 | public String getId() { 126 | return id; 127 | } 128 | 129 | @Override 130 | public void setId(String id) { 131 | this.id = id; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/EsNameMetaData.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | 19 | import com.pinterest.soundwave.bean.EsInstanceConfig; 20 | import com.pinterest.soundwave.bean.EsDocument; 21 | 22 | import com.fasterxml.jackson.annotation.JsonIgnore; 23 | import com.fasterxml.jackson.annotation.JsonProperty; 24 | 25 | import java.util.List; 26 | 27 | public class EsNameMetaData implements EsDocument { 28 | 29 | @JsonProperty("nodepool") 30 | private String nodePool; 31 | 32 | @JsonProperty("id") 33 | private String id; 34 | 35 | @JsonProperty("security_groups") 36 | private List securityGroups; 37 | 38 | @JsonProperty("location") 39 | private String location; 40 | 41 | @JsonProperty("region") 42 | private String region; 43 | 44 | @JsonProperty("deployment") 45 | private String deployment; 46 | 47 | @JsonProperty("config") 48 | private EsInstanceConfig config; 49 | 50 | @JsonIgnore 51 | private long version; 52 | 53 | public String getNodePool() { 54 | return nodePool; 55 | } 56 | 57 | public void setNodePool(String nodePool) { 58 | this.nodePool = nodePool; 59 | } 60 | 61 | @Override 62 | public String getId() { 63 | return id; 64 | } 65 | 66 | @Override 67 | public void setId(String id) { 68 | this.id = id; 69 | } 70 | 71 | public List getSecurityGroups() { 72 | return securityGroups; 73 | } 74 | 75 | public void setSecurityGroups(List securityGroups) { 76 | this.securityGroups = securityGroups; 77 | } 78 | 79 | public String getLocation() { 80 | return location; 81 | } 82 | 83 | public void setLocation(String location) { 84 | this.location = location; 85 | } 86 | 87 | public String getRegion() { 88 | return region; 89 | } 90 | 91 | public void setRegion(String region) { 92 | this.region = region; 93 | } 94 | 95 | public String getDeployment() { 96 | return deployment; 97 | } 98 | 99 | public void setDeployment(String deployment) { 100 | this.deployment = deployment; 101 | } 102 | 103 | public EsInstanceConfig getConfig() { 104 | return config; 105 | } 106 | 107 | public void setConfig(EsInstanceConfig config) { 108 | this.config = config; 109 | } 110 | 111 | @Override 112 | public long getVersion() { 113 | return version; 114 | } 115 | 116 | @Override 117 | public void setVersion(long version) { 118 | this.version = version; 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/EsPackages.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | 24 | public class EsPackages { 25 | 26 | 27 | @JsonProperty("id") 28 | private String id; 29 | 30 | @JsonProperty("pkgs") 31 | private Map pkgs = new HashMap<>(); 32 | 33 | @JsonIgnore 34 | private long version; 35 | 36 | public long getVersion() { 37 | return version; 38 | } 39 | 40 | public void setVersion(long version) { 41 | this.version = version; 42 | } 43 | 44 | public String getId() { 45 | return id; 46 | } 47 | 48 | public void setId(String id) { 49 | this.id = id; 50 | } 51 | 52 | public Map getPkgs() { 53 | return pkgs; 54 | } 55 | 56 | public void setPkgs(Map pkgs) { 57 | this.pkgs = pkgs; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/EsServiceMapping.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import org.apache.commons.lang.StringUtils; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | import java.util.regex.Pattern; 28 | 29 | /** 30 | * The class for the customize tagging 31 | */ 32 | @JsonIgnoreProperties(ignoreUnknown = true) 33 | public class EsServiceMapping { 34 | 35 | private static final Logger logger = LoggerFactory.getLogger(EsServiceMapping.class); 36 | 37 | @JsonIgnore 38 | List patterns = new ArrayList<>(); 39 | 40 | @JsonProperty("name") 41 | private String name; 42 | 43 | @JsonProperty("svc_tag") 44 | private String serviceTag; 45 | 46 | @JsonProperty("sys_tag") 47 | private String sysTag; 48 | 49 | @JsonProperty("usage_tag") 50 | private String usageTag; 51 | 52 | @JsonProperty("value") 53 | private String value; 54 | 55 | public String getName() { 56 | return name; 57 | } 58 | 59 | public void setName(String name) { 60 | this.name = name; 61 | } 62 | 63 | public String getServiceTag() { 64 | return serviceTag; 65 | } 66 | 67 | public void setServiceTag(String serviceTag) { 68 | this.serviceTag = serviceTag; 69 | } 70 | 71 | public String getSysTag() { 72 | return sysTag; 73 | } 74 | 75 | public void setSysTag(String sysTag) { 76 | this.sysTag = sysTag; 77 | } 78 | 79 | public String getUsageTag() { 80 | return usageTag; 81 | } 82 | 83 | public void setUsageTag(String usageTag) { 84 | this.usageTag = usageTag; 85 | } 86 | 87 | public String getValue() { 88 | return value; 89 | } 90 | 91 | public void setValue(String value) { 92 | this.value = value; 93 | } 94 | 95 | public void buildMatchPatterns() { 96 | if (StringUtils.isNotEmpty(this.getValue())) { 97 | patterns.clear(); 98 | String[] expressions = this.getValue().split(","); 99 | for (String expression : expressions) { 100 | if (StringUtils.isNotEmpty(expression)) { 101 | String formalizedPattern = StringUtils.strip(StringUtils.strip(expression, " "), "/"); 102 | patterns.add(Pattern.compile(formalizedPattern)); 103 | } 104 | } 105 | } 106 | } 107 | 108 | public boolean matches(String input) { 109 | boolean ret = false; 110 | for (Pattern pattern : this.patterns) { 111 | if (pattern.matcher(input).matches()) { 112 | logger.debug("Match {} for input {}", pattern, input); 113 | ret = true; 114 | break; 115 | } 116 | } 117 | return ret; 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/HostImageInfoProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | public interface HostImageInfoProvider { 19 | 20 | String getApplicationName(String hostImageId); 21 | } 22 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/QuobleNameChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | import com.pinterest.soundwave.aws.MessageProcessingResult; 19 | import com.pinterest.soundwave.bean.EsInstance; 20 | import com.pinterest.soundwave.bean.PinterestEsInstance; 21 | 22 | public final class QuobleNameChecker { 23 | 24 | public static void checkQuobleName(EsInstance instance, MessageProcessingResult result) { 25 | if (instance instanceof PinterestEsInstance) { 26 | PinterestEsInstance esInstance = (PinterestEsInstance) instance; 27 | if ("node".equals(esInstance.getDeployment()) 28 | && esInstance.getTags() != null 29 | && esInstance.getTags().containsKey("Qubole")) { 30 | //True Qubole name tag is available later 31 | result.setError(MessageProcessingResult.MessageProcessingError.QUOBLE_NAME_NOT_AVAILABLE); 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/pinterest/ServiceMappingStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.pinterest; 17 | 18 | import java.util.List; 19 | 20 | public interface ServiceMappingStore { 21 | 22 | List getServiceMappings() throws Exception; 23 | 24 | EsServiceMapping getServiceMappingByName(String name) throws Exception; 25 | 26 | long updateOrInsertServiceMapping(EsServiceMapping serviceMapping) throws Exception; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/utils/StringArrayOrElementStringDeserializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.utils; 17 | 18 | import com.fasterxml.jackson.core.JsonParser; 19 | import com.fasterxml.jackson.core.JsonProcessingException; 20 | import com.fasterxml.jackson.core.ObjectCodec; 21 | import com.fasterxml.jackson.databind.DeserializationContext; 22 | import com.fasterxml.jackson.databind.JsonDeserializer; 23 | import com.fasterxml.jackson.databind.JsonNode; 24 | import com.fasterxml.jackson.databind.node.ArrayNode; 25 | 26 | import java.io.IOException; 27 | import java.util.ArrayList; 28 | 29 | public class StringArrayOrElementStringDeserializer extends JsonDeserializer { 30 | 31 | 32 | public StringArrayOrElementStringDeserializer() { 33 | 34 | } 35 | 36 | @Override 37 | public Object deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) 38 | throws IOException, JsonProcessingException { 39 | ObjectCodec oc = jsonParser.getCodec(); 40 | JsonNode node = oc.readTree(jsonParser); 41 | if (node instanceof ArrayNode) { 42 | ArrayNode arrayNode = (ArrayNode) node; 43 | ArrayList ret = new ArrayList<>(); 44 | for (int i = 0; i < arrayNode.size(); i++) { 45 | ret.add(arrayNode.get(i).textValue()); 46 | } 47 | return ret.toArray(new String[0]); 48 | } else { 49 | return new String[]{node.textValue()}; 50 | } 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/utils/StringListOrElementDeserializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.utils; 17 | 18 | import com.fasterxml.jackson.core.JsonParser; 19 | import com.fasterxml.jackson.core.JsonProcessingException; 20 | import com.fasterxml.jackson.core.ObjectCodec; 21 | import com.fasterxml.jackson.databind.DeserializationContext; 22 | import com.fasterxml.jackson.databind.JsonDeserializer; 23 | import com.fasterxml.jackson.databind.JsonNode; 24 | import com.fasterxml.jackson.databind.node.ArrayNode; 25 | 26 | import java.io.IOException; 27 | import java.util.ArrayList; 28 | import java.util.Arrays; 29 | 30 | public class StringListOrElementDeserializer extends JsonDeserializer { 31 | 32 | 33 | public StringListOrElementDeserializer() { 34 | super(); 35 | } 36 | 37 | 38 | @Override 39 | public Object deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) 40 | throws IOException, JsonProcessingException { 41 | ObjectCodec oc = jsonParser.getCodec(); 42 | JsonNode node = oc.readTree(jsonParser); 43 | if (node instanceof ArrayNode) { 44 | ArrayNode arrayNode = (ArrayNode) node; 45 | ArrayList ret = new ArrayList<>(); 46 | for (int i = 0; i < arrayNode.size(); i++) { 47 | ret.add(arrayNode.get(i).textValue()); 48 | } 49 | return ret; 50 | } else { 51 | return Arrays.asList(node.textValue()); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/soundwave/utils/ThrowingFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.utils; 17 | 18 | @FunctionalInterface 19 | public interface ThrowingFunction { 20 | 21 | R apply(T t) throws Exception; 22 | } 23 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/zookeeper/ZkClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.zookeeper; 17 | 18 | import com.pinterest.config.Configuration; 19 | 20 | import org.apache.curator.framework.CuratorFramework; 21 | import org.apache.curator.framework.CuratorFrameworkFactory; 22 | import org.apache.curator.retry.ExponentialBackoffRetry; 23 | 24 | /** 25 | * A singleton instance for accessing zk 26 | */ 27 | public class ZkClient { 28 | 29 | private static CuratorFramework client = null; 30 | 31 | private ZkClient() {} 32 | 33 | public static CuratorFramework getClient() { 34 | if (client == null) { 35 | synchronized (ZkClient.class) { 36 | if (client == null) { 37 | client = CuratorFrameworkFactory 38 | .newClient(Configuration.getProperties().getString("zk_connection_string"), 39 | new ExponentialBackoffRetry(1000, 10000000)); //TODO. Make it read from config 40 | client.start(); 41 | } 42 | } 43 | } 44 | return client; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /commons/src/main/java/com/pinterest/zookeeper/ZkScheduledJobOwnershipDecider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.zookeeper; 17 | 18 | import com.pinterest.config.Configuration; 19 | import com.pinterest.job.OwnershipDecider; 20 | 21 | import com.google.common.base.Preconditions; 22 | import org.apache.commons.lang.StringUtils; 23 | import org.apache.curator.framework.recipes.leader.LeaderLatch; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import java.net.InetAddress; 28 | import java.util.UUID; 29 | 30 | /** 31 | * An ownership decider based on Zookeeper leader election. 32 | */ 33 | public class ZkScheduledJobOwnershipDecider implements OwnershipDecider { 34 | 35 | private static final Logger 36 | logger = 37 | LoggerFactory.getLogger(ZkScheduledJobOwnershipDecider.class); 38 | private static String nodeIdentifier; 39 | 40 | static { 41 | try { 42 | nodeIdentifier = 43 | InetAddress.getLocalHost().getHostName() + "_" + UUID.randomUUID().toString(); 44 | } catch (Exception e) { 45 | logger.warn("Cannot get the node name"); 46 | nodeIdentifier = "unknown" + UUID.randomUUID().toString(); 47 | } 48 | } 49 | 50 | private String zkPath; 51 | private LeaderLatch latch; 52 | 53 | 54 | private ZkScheduledJobOwnershipDecider(String zkPath) { 55 | Preconditions.checkArgument(StringUtils.isNotEmpty(zkPath)); 56 | this.zkPath = zkPath; 57 | } 58 | 59 | public static ZkScheduledJobOwnershipDecider buildDecider(String identifier) throws Exception { 60 | ZkScheduledJobOwnershipDecider decider = new ZkScheduledJobOwnershipDecider( 61 | String.format("%s/job/%s/owner", 62 | StringUtils.stripEnd(Configuration.getProperties().getString("zk_path"), "/"), 63 | identifier)); 64 | decider.latch = 65 | new LeaderLatch(ZkClient.getClient(), decider.zkPath, nodeIdentifier, 66 | LeaderLatch.CloseMode.NOTIFY_LEADER); 67 | decider.latch.start(); 68 | return decider; 69 | } 70 | 71 | @Override 72 | public boolean isOwner() { 73 | return latch.hasLeadership(); 74 | } 75 | 76 | @Override 77 | public String getNodeName() { 78 | return nodeIdentifier; 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /commons/src/test/java/com/pinterest/soundwave/bean/EsDailySnapshotInstanceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import org.joda.time.DateTime; 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.Date; 23 | 24 | public class EsDailySnapshotInstanceTest { 25 | 26 | @Test 27 | public void roundToSeconds() throws Exception { 28 | DateTime d = new DateTime(2016, 7, 7, 10, 11, 23, 123); 29 | Date d2 = EsDailySnapshotInstance.roundToSeconds(d.toDate()); 30 | Assert.assertEquals(d2.getTime(), d.getMillis() - 123); 31 | } 32 | 33 | @Test 34 | public void getStringOfPeriod() throws Exception { 35 | DateTime d = new DateTime(2016, 7, 7, 10, 11, 23, 123); 36 | Assert.assertEquals("15s", 37 | EsDailySnapshotInstance.getStringOfPeriod(d.minusSeconds(15).toDate(), d.toDate())); 38 | Assert.assertEquals("1m 15s", EsDailySnapshotInstance 39 | .getStringOfPeriod(d.minusMinutes(1).minusSeconds(15).toDate(), d.toDate())); 40 | Assert.assertEquals("2h 1m 15s", EsDailySnapshotInstance 41 | .getStringOfPeriod(d.minusHours(2).minusMinutes(1).minusSeconds(15).toDate(), d.toDate())); 42 | Assert.assertEquals("13D 2h 1m 15s", EsDailySnapshotInstance 43 | .getStringOfPeriod(d.minusDays(13).minusHours(2).minusMinutes(1).minusSeconds(15).toDate(), 44 | d.toDate())); 45 | Assert.assertEquals("11M 3D 2h 1m 15s", EsDailySnapshotInstance 46 | .getStringOfPeriod( 47 | d.minusMonths(11).minusDays(3).minusHours(2).minusMinutes(1).minusSeconds(15).toDate(), 48 | d.toDate())); 49 | Assert.assertEquals("1Y 2M 13D 2h 1m 15s", EsDailySnapshotInstance 50 | .getStringOfPeriod( 51 | d.minusYears(1).minusMonths(2).minusDays(13).minusHours(2).minusMinutes(1) 52 | .minusSeconds(15).toDate(), d.toDate())); 53 | 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /commons/src/test/java/com/pinterest/soundwave/bean/EsInstanceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.bean; 17 | 18 | import org.junit.Assert; 19 | import org.junit.Rule; 20 | import org.junit.Test; 21 | import org.junit.rules.ExpectedException; 22 | 23 | public class EsInstanceTest { 24 | 25 | @Rule 26 | public ExpectedException thrown = ExpectedException.none(); 27 | 28 | @Test 29 | public void setState() throws Exception { 30 | EsInstance inst = new EsInstance(); 31 | inst.setState(State.RUNNING.toString()); 32 | Assert.assertEquals(inst.getState(), State.RUNNING.toString()); 33 | inst.setState(State.RUNNING.toString()); 34 | Assert.assertEquals(inst.getState(), State.RUNNING.toString()); 35 | 36 | inst.setState(State.STOPPED.toString()); 37 | Assert.assertEquals(inst.getState(), State.STOPPED.toString()); 38 | 39 | inst.setState(State.RUNNING.toString()); 40 | Assert.assertEquals(inst.getState(), State.RUNNING.toString()); 41 | 42 | inst.setState(State.TERMINATED.toString()); 43 | Assert.assertEquals(inst.getState(), State.TERMINATED.toString()); 44 | 45 | thrown.expect(InvalidStateTransitionException.class); 46 | inst.setState(State.RUNNING.toString()); 47 | 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /commons/src/test/java/com/pinterest/soundwave/elasticsearch/DummyEsStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | public class DummyEsStore extends EsStore { 19 | 20 | @Override 21 | public String getDocTypeName() { 22 | return "testDocType"; 23 | } 24 | 25 | @Override 26 | public String getIndexName() { 27 | return "lida_aws_test"; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /commons/src/test/java/com/pinterest/soundwave/elasticsearch/ESServiceMappingStoreTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | import com.pinterest.soundwave.pinterest.EsServiceMapping; 19 | import com.pinterest.soundwave.pinterest.EsServiceMappingStore; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Ignore; 23 | import org.junit.Test; 24 | 25 | import java.util.List; 26 | 27 | public class ESServiceMappingStoreTest { 28 | 29 | @Test 30 | @Ignore 31 | public void getServiceMappings() throws Exception { 32 | EsServiceMappingStore store = new EsServiceMappingStore(); 33 | List mappings = store.getServiceMappings(); 34 | Assert.assertTrue(mappings.size() > 0); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /commons/src/test/java/com/pinterest/soundwave/elasticsearch/EsIteratorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.elasticsearch; 17 | 18 | import org.apache.commons.lang.ArrayUtils; 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import java.util.ArrayList; 23 | import java.util.Arrays; 24 | import java.util.List; 25 | 26 | public class EsIteratorTest { 27 | 28 | @Test 29 | public void noRetrieval() throws Exception { 30 | ScrollableResponse> scrollableResponse = new ScrollableResponse<>(); 31 | scrollableResponse.setValue(Arrays.asList(1, 2, 3, 4, 5)); 32 | scrollableResponse.setScrollToEnd(true); 33 | 34 | EsIterator l = new EsIterator<>(scrollableResponse, s -> scrollableResponse); 35 | int count = 0; 36 | ArrayList result = new ArrayList<>(); 37 | while (l.hasNext()) { 38 | result.add(l.next()); 39 | count++; 40 | } 41 | Assert.assertArrayEquals(new int[]{1, 2, 3, 4, 5}, 42 | ArrayUtils.toPrimitive(result.toArray(new Integer[0]))); 43 | } 44 | 45 | @Test 46 | public void onRetrieval() throws Exception { 47 | ScrollableResponse> scrollableResponse1 = new ScrollableResponse<>(); 48 | scrollableResponse1.setValue(Arrays.asList(1, 2, 3, 4, 5)); 49 | scrollableResponse1.setScrollToEnd(false); 50 | 51 | ScrollableResponse> scrollableResponse2 = new ScrollableResponse<>(); 52 | scrollableResponse2.setValue(Arrays.asList(1, 2, 3)); 53 | scrollableResponse2.setScrollToEnd(true); 54 | 55 | EsIterator l = new EsIterator<>(scrollableResponse1, s -> scrollableResponse2); 56 | int count = 0; 57 | ArrayList result = new ArrayList<>(); 58 | while (l.hasNext()) { 59 | result.add(l.next()); 60 | count++; 61 | } 62 | 63 | Assert.assertArrayEquals(new int[]{1, 2, 3, 4, 5, 1, 2, 3}, 64 | ArrayUtils.toPrimitive(result.toArray(new Integer[0]))); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /commons/src/test/java/com/pinterest/soundwave/utils/InstanceTaggerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.utils; 17 | 18 | import com.pinterest.soundwave.pinterest.EsServiceMapping; 19 | import com.pinterest.soundwave.pinterest.InstanceTagger; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Ignore; 23 | import org.junit.Test; 24 | 25 | import java.util.List; 26 | 27 | ; 28 | 29 | public class InstanceTaggerTest { 30 | 31 | @Test 32 | @Ignore 33 | public void findMapping() throws Exception { 34 | InstanceTagger tagger = InstanceTagger.getInstance(); 35 | List mapping = tagger.findMapping("pinalyticsv2-a01-regionserver-00039ea0"); 36 | Assert.assertNotNull(mapping); 37 | 38 | InstanceTagger.InstanceTags tags = tagger.getTags("cmp-pinshot-newkernel-0a0111e9"); 39 | Assert.assertEquals("pinshot", tags.getServiceMappings()[0]); 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /commons/src/test/java/com/pinterest/soundwave/zookeeper/ZkJobInfoStoreTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.zookeeper; 17 | 18 | import com.pinterest.job.JobRunInfo; 19 | import com.pinterest.zookeeper.ZkJobInfoStore; 20 | 21 | import org.joda.time.DateTime; 22 | import org.junit.Assert; 23 | import org.junit.Ignore; 24 | import org.junit.Test; 25 | 26 | public class ZkJobInfoStoreTest { 27 | 28 | @Test 29 | @Ignore 30 | public void jobInfoStoreTests() throws Exception { 31 | ZkJobInfoStore store = new ZkJobInfoStore(); 32 | JobRunInfo run = new JobRunInfo(); 33 | run.setSucceed(true); 34 | run.setJobName("TestJob"); 35 | run.setStartTime(DateTime.now().minusHours(1).toDate()); 36 | store.updateLatestRun(run); 37 | JobRunInfo saved = store.getLatestRun(run.getJobName()); 38 | Assert.assertTrue(run.isSucceed()); 39 | Assert.assertEquals(saved.getStartTime(), run.getStartTime()); 40 | 41 | } 42 | 43 | @Test 44 | @Ignore 45 | public void jobConfigTests() throws Exception { 46 | ZkJobInfoStore store = new ZkJobInfoStore(); 47 | boolean b = store.isJobDisabled("UpdateCloudHealth"); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Docker-Compose for Soundwave 2 | version: '2' 3 | services: 4 | soundwave-worker: 5 | build: ./worker 6 | environment: 7 | - CONFIG_FILE=config/soundwaveworker.properties 8 | - TEST_CONFIG=1 9 | - AWS_ACCESS_KEY_ID=<> 10 | - AWS_SECRET_ACCESS_KEY=<> 11 | 12 | soundwave-api: 13 | build: ./api 14 | environment: 15 | - CONFIG_FILE=config/soundwaveapi.properties 16 | ports: 17 | - "8080:80" 18 | 19 | soundwave-store: 20 | image: elasticsearch:1.7.6 21 | ports: 22 | - "9200:9200" 23 | - "9300:9300" 24 | 25 | soundwave-zk: 26 | image: zookeeper:3.3.6 27 | ports: 28 | - "2181:2181" 29 | 30 | soundwave-ui: 31 | build: ./ui 32 | ports: 33 | - "80:80" 34 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.pinterest.soundwave 5 | soundwave 6 | pom 7 | 0.1-SNAPSHOT 8 | soundwave modules 9 | 2017 10 | 11 | commons 12 | worker 13 | api 14 | 15 | 16 | 1.8 17 | 1.8 18 | 19 | 20 | -------------------------------------------------------------------------------- /resources/soundwavearch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/resources/soundwavearch.jpg -------------------------------------------------------------------------------- /soundwavedemo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/soundwavedemo.png -------------------------------------------------------------------------------- /terraform/.gitignore: -------------------------------------------------------------------------------- 1 | *.tfstate 2 | *.backup 3 | -------------------------------------------------------------------------------- /terraform/soundwave_lambda.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | 4 | def lambda_handler(event, context): 5 | # Get the service resource 6 | sqs = boto3.resource('sqs') 7 | # Get the queue. This returns an SQS.Queue instance 8 | queue = sqs.get_queue_by_name(QueueName='soundwave-events') 9 | # Write to queue 10 | response = queue.send_message(MessageBody=json.dumps(event)) 11 | -------------------------------------------------------------------------------- /terraform/soundwave_lambda.py.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/terraform/soundwave_lambda.py.zip -------------------------------------------------------------------------------- /ui/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2-onbuild 2 | 3 | # Create log directory 4 | RUN mkdir -p /var/log/soundwave_ui 5 | 6 | # expose the port 7 | EXPOSE 8080 8 | 9 | # launch the app 10 | CMD [ "python", "./app.py" ] 11 | -------------------------------------------------------------------------------- /ui/Readme.md: -------------------------------------------------------------------------------- 1 | **Soundwave UI** 2 | 3 | A flask application that provides a user interface for querying Soundwave backend (API / Elastic search store) 4 | 5 | -------------------------------------------------------------------------------- /ui/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/__init__.py -------------------------------------------------------------------------------- /ui/config.py: -------------------------------------------------------------------------------- 1 | """Config data for cmdb UI.""" 2 | SOUNDWAVE_API = "http://soundwave-api/v2/" 3 | SOUNDWAVE_HOST = "0.0.0.0" 4 | SOUNDWAVE_PORT = 80 5 | SOUNDWAVE_LOG_PATH = "/var/log/soundwave_ui/" 6 | SOUNDWAVE_ACCESS_LOG = "access.log" 7 | SOUNDWAVE_APP_LOG = "info.log" 8 | -------------------------------------------------------------------------------- /ui/requirements.txt: -------------------------------------------------------------------------------- 1 | CherryPy==8.1.2 2 | Flask==1.1.1 3 | requests==2.20.0 4 | -------------------------------------------------------------------------------- /ui/static/css/buttons.bootstrap.min.css: -------------------------------------------------------------------------------- 1 | div.dt-button-info{position:fixed;top:50%;left:50%;width:400px;margin-top:-100px;margin-left:-200px;background-color:white;border:2px solid #111;box-shadow:3px 3px 8px rgba(0,0,0,0.3);border-radius:3px;text-align:center;z-index:21}div.dt-button-info h2{padding:0.5em;margin:0;font-weight:normal;border-bottom:1px solid #ddd;background-color:#f3f3f3}div.dt-button-info>div{padding:1em}ul.dt-button-collection.dropdown-menu{display:block;z-index:2002;-webkit-column-gap:8px;-moz-column-gap:8px;-ms-column-gap:8px;-o-column-gap:8px;column-gap:8px}ul.dt-button-collection.dropdown-menu.fixed{position:fixed;top:50%;left:50%;margin-left:-75px;border-radius:0}ul.dt-button-collection.dropdown-menu.fixed.two-column{margin-left:-150px}ul.dt-button-collection.dropdown-menu.fixed.three-column{margin-left:-225px}ul.dt-button-collection.dropdown-menu.fixed.four-column{margin-left:-300px}ul.dt-button-collection.dropdown-menu>*{-webkit-column-break-inside:avoid;break-inside:avoid}ul.dt-button-collection.dropdown-menu.two-column{width:300px;padding-bottom:1px;-webkit-column-count:2;-moz-column-count:2;-ms-column-count:2;-o-column-count:2;column-count:2}ul.dt-button-collection.dropdown-menu.three-column{width:450px;padding-bottom:1px;-webkit-column-count:3;-moz-column-count:3;-ms-column-count:3;-o-column-count:3;column-count:3}ul.dt-button-collection.dropdown-menu.four-column{width:600px;padding-bottom:1px;-webkit-column-count:4;-moz-column-count:4;-ms-column-count:4;-o-column-count:4;column-count:4}div.dt-button-background{position:fixed;top:0;left:0;width:100%;height:100%;z-index:2001}@media screen and (max-width: 767px){div.dt-buttons{float:none;width:100%;text-align:center;margin-bottom:0.5em}div.dt-buttons a.btn{float:none}} 2 | -------------------------------------------------------------------------------- /ui/static/css/jsoneditor.css: -------------------------------------------------------------------------------- 1 | .json-editor { 2 | margin-left: 10px; 3 | padding: 20px; 4 | } 5 | 6 | .json-editor.expanded .item { 7 | display: block !important; 8 | } 9 | 10 | .json-editor .property, 11 | .json-editor .value { 12 | border: none; 13 | -webkit-border-radius: 5px; 14 | -moz-border-radius: 5px; 15 | border-radius: 5px; 16 | padding: 3px; 17 | margin-right: 2px; 18 | } 19 | 20 | .json-editor .value { 21 | background-color: #F2F2F2; 22 | width: 550px; 23 | } 24 | 25 | .json-editor .property { 26 | width: 154px; 27 | background-color: #D6D6D6; 28 | color: black; 29 | font-weight: bold; 30 | /*text-shadow: 1px 1px 1px white;*/ 31 | } 32 | 33 | .json-editor button.property { 34 | width: 160px; 35 | } 36 | 37 | .json-editor .editing { 38 | border: 1px solid gray; 39 | outline: none; 40 | } 41 | 42 | .json-editor .item { 43 | line-height: 20px; 44 | margin-top: 2px; 45 | } 46 | 47 | .json-editor .item:last-child { 48 | border-bottom: none; 49 | } 50 | 51 | .json-editor .item > .item { 52 | margin-left: 30px; 53 | display: none; 54 | } 55 | 56 | .json-editor .item.expanded > .item { 57 | display: block; 58 | } 59 | 60 | .json-editor .expander { 61 | background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9sJCxMmIIjsHfkAAAHTSURBVDjLzZPLaxNRFMZ/ZzKTNs10xHUyYFsfsZC2IOii2ftHCKKx/0G1brQlFhShCzfuXXUjuHErKGiVpr6qYK0bGyMoyCRVq5Mwkzku8ja6c+GFA5f7ne87zwv/zXn2bgcAVRVVNf5i0usLIO3Lq/cfze2trdfra48yHbCFqnYDnZjNbafcdPbtxpMgn893BXYqPxZO5o5fNwwDAUSkL0NVRYEoarB65+6tY5mJswBGN5okfP9nhyxA2nVJp9N9b/Vana/VaidLszeKIIgIhghhGDI/fx7H2ceZ06ewrDhRt2YdEBBpg0rKdRm1bZxRm+TIMNnsFN/39iiXyyDd3vSd8q6/dGB8XI8cndR7Dx5qpKqNqGmqqsUXm3o4k9GxiYN6f/3p44EMFAUFyzRZLiyxMjzEpcuL2LbNxYUL+LU6cStOEAQ9BfQKNPkoUPE8wjDEr9UxrSFKpRKWFW/ytH+s5sCoVImAWCzG8pVC08myiFpYk6vyhwz0m+04aKPRchE8z/ttD5SRZJJEIlEZ2MSV26sc2p/afL5RnOrZjQ653afp6Zmg+ulDbu7cXLFPYO3lG2ZnJrlWWByLxUyr09jWfrRnvfvlc/nqjZv+P/uEvwD/VNrBXhv7BAAAAABJRU5ErkJggg==") no-repeat center center; 62 | width: 16px; 63 | height: 11px; 64 | display: inline-block; 65 | margin-left: -15px; 66 | cursor: pointer; 67 | } 68 | 69 | .json-editor .item.expanded > .expander { 70 | background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9sKBBAdNcQGwvUAAAGjSURBVDjLzZNNS1tREIafOfdGjcaA65hFKQUpWIWCLgwI3fQnFMGN0Z/QiqsWKVgKLrpx78pNoZv+BUU0WNRC/dgoTaFCMWnUNoZ7b6aLm5ycS+quix44cJiPd953Zg78N2fv9BwAVRVVNXdccWMBpP04PPvmnxwdfd7Z2hyxzpZXtVNocqpwkssPjx6XtoNisdgBOK/8WnxamHhrjEEAEUkwVFUUaDYjNj58XH88cn8OwHSqSbpe/22T3XTX1rhtUKtWLUtDIlAQEYwIURRZexgGGBEX2IryXb3S8uXyeQYzGYc/XN/cUC6XQTq9SQIQI4RhxPMXSzyZLtjmGYHS/iGzM89iQX8DUBQUUr7P6+VXrPb1MpjNAnBVq1G/bdCT6iEIAkeACxDno0Dl8pIwDDGeZwONSJynybH6XaNSpQl4nmcLiUCz5YttKl1TUNWrTDabAEmwU0VR+gcGSKfTla5NXH2/wYOh3MGn0u4jZzcss3afxsbGg+r3r4WF+YXdBMDW/hemxh/yZvnlPc/zU7ax7Qm1tPz8cVFeebdW/2ef8A+6rb2Fr7Cr4wAAAABJRU5ErkJggg==") no-repeat center center; 71 | } 72 | 73 | .json-editor .item.appender > .property { 74 | background-color: #999999; 75 | cursor: pointer; 76 | width: 100px; 77 | } 78 | 79 | div.appender { 80 | display: none; 81 | } 82 | 83 | .json-editor .item.string > .property { 84 | background-color: #D6D6D6; 85 | } 86 | 87 | .json-editor .item.array > .property { 88 | background-color: #D6D6D6; 89 | } 90 | 91 | .json-editor .item.object > .property { 92 | background-color: #D6D6D6; 93 | } 94 | 95 | .json-editor .item.number > .property { 96 | background-color: #D6D6D6; 97 | } 98 | 99 | /*.json-editor .item.boolean { background-color: }*/ 100 | /*.json-editor .item.null { background-color: }*/ -------------------------------------------------------------------------------- /ui/static/css/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: 'Open Sans', sans-serif; 3 | } 4 | 5 | div.navbar { 6 | border-radius: 0px; 7 | } 8 | 9 | div.results-table { 10 | margin: 10px 40px; 11 | } 12 | 13 | div.query-box { 14 | margin: 10px 40px; 15 | } 16 | 17 | input.query-input { 18 | 19 | float: left; 20 | } 21 | 22 | i.fa-database { 23 | color: #5f9ea0; 24 | } 25 | 26 | button.query-btn { 27 | margin-top: 15px; 28 | margin-left: 10px; 29 | } 30 | 31 | label.query-checkbox { 32 | margin-top: 20px; 33 | } 34 | 35 | input:-webkit-autofill, 36 | input:-webkit-autofill:hover, 37 | input:-webkit-autofill:focus, 38 | input:-webkit-autofill:active { 39 | transition: background-color 5000s ease-in-out 0s; 40 | } 41 | 42 | button.copy-btn { 43 | margin-left: 40px; 44 | } 45 | 46 | .error-message { 47 | text-align: center; 48 | } 49 | -------------------------------------------------------------------------------- /ui/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/favicon.ico -------------------------------------------------------------------------------- /ui/static/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /ui/static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /ui/static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /ui/static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /ui/static/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /ui/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /ui/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /ui/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /ui/static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /ui/static/images/Sorting icons.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/images/Sorting icons.psd -------------------------------------------------------------------------------- /ui/static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/images/favicon.ico -------------------------------------------------------------------------------- /ui/static/images/sort_asc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/images/sort_asc.png -------------------------------------------------------------------------------- /ui/static/images/sort_asc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/images/sort_asc_disabled.png -------------------------------------------------------------------------------- /ui/static/images/sort_both.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/images/sort_both.png -------------------------------------------------------------------------------- /ui/static/images/sort_desc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/images/sort_desc.png -------------------------------------------------------------------------------- /ui/static/images/sort_desc_disabled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/images/sort_desc_disabled.png -------------------------------------------------------------------------------- /ui/static/js/buttons.bootstrap.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | Bootstrap integration for DataTables' Buttons 3 | ©2016 SpryMedia Ltd - datatables.net/license 4 | */ 5 | (function(c){"function"===typeof define&&define.amd?define(["jquery","datatables.net-bs","datatables.net-buttons"],function(a){return c(a,window,document)}):"object"===typeof exports?module.exports=function(a,b){a||(a=window);if(!b||!b.fn.dataTable)b=require("datatables.net-bs")(a,b).$;b.fn.dataTable.Buttons||require("datatables.net-buttons")(a,b);return c(b,a,a.document)}:c(jQuery,window,document)})(function(c){var a=c.fn.dataTable;c.extend(!0,a.Buttons.defaults,{dom:{container:{className:"dt-buttons btn-group"}, 6 | button:{className:"btn btn-default"},collection:{tag:"ul",className:"dt-button-collection dropdown-menu",button:{tag:"li",className:"dt-button"},buttonLiner:{tag:"a",className:""}}}});a.ext.buttons.collection.text=function(a){return a.i18n("buttons.collection",'Collection ')};return a.Buttons}); 7 | -------------------------------------------------------------------------------- /ui/static/js/dataTables.bootstrap.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | DataTables Bootstrap 3 integration 3 | ©2011-2015 SpryMedia Ltd - datatables.net/license 4 | */ 5 | (function(b){"function"===typeof define&&define.amd?define(["jquery","datatables.net"],function(a){return b(a,window,document)}):"object"===typeof exports?module.exports=function(a,d){a||(a=window);if(!d||!d.fn.dataTable)d=require("datatables.net")(a,d).$;return b(d,a,a.document)}:b(jQuery,window,document)})(function(b,a,d){var f=b.fn.dataTable;b.extend(!0,f.defaults,{dom:"<'row'<'col-sm-6'l><'col-sm-6'f>><'row'<'col-sm-12'tr>><'row'<'col-sm-5'i><'col-sm-7'p>>",renderer:"bootstrap"});b.extend(f.ext.classes, 6 | {sWrapper:"dataTables_wrapper form-inline dt-bootstrap",sFilterInput:"form-control input-sm",sLengthSelect:"form-control input-sm",sProcessing:"dataTables_processing panel panel-default"});f.ext.renderer.pageButton.bootstrap=function(a,h,r,m,j,n){var o=new f.Api(a),s=a.oClasses,k=a.oLanguage.oPaginate,t=a.oLanguage.oAria.paginate||{},e,g,p=0,q=function(d,f){var l,h,i,c,m=function(a){a.preventDefault();!b(a.currentTarget).hasClass("disabled")&&o.page()!=a.data.action&&o.page(a.data.action).draw("page")}; 7 | l=0;for(h=f.length;l",{"class":s.sPageButton+" "+g,id:0===r&&"string"===typeof c?a.sTableId+"_"+c:null}).append(b("",{href:"#", 8 | "aria-controls":a.sTableId,"aria-label":t[c],"data-dt-idx":p,tabindex:a.iTabIndex}).html(e)).appendTo(d),a.oApi._fnBindAction(i,{action:c},m),p++)}},i;try{i=b(h).find(d.activeElement).data("dt-idx")}catch(u){}q(b(h).empty().html('
    ').children("ul"),m);i&&b(h).find("[data-dt-idx="+i+"]").focus()};return f}); 9 | -------------------------------------------------------------------------------- /ui/static/swf/flashExport.swf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pinterest/soundwave/144c6640775fb3d94ad583c54de35f305737135a/ui/static/swf/flashExport.swf -------------------------------------------------------------------------------- /ui/templates/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Soundwave 404 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | s 14 | 15 | 16 | 17 |
    18 | {% include 'navbar.html' %} 19 |
    20 |

    oops ! I dont know this route

    21 |
    Please try the query page
    22 | 23 | -------------------------------------------------------------------------------- /ui/templates/500.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Soundwave 500 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
    18 | {% include 'navbar.html' %} 19 |
    20 |

    oops ! something is broken

    21 |
    Please try the query page
    22 | 23 | -------------------------------------------------------------------------------- /ui/templates/aggregations.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | soundwave aggregations 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
    21 | {% include 'navbar.html' %} 22 |
    23 |
    24 |
    25 |
    26 |
    27 | 28 | 29 |
    30 |
    31 | 32 |
    33 |
    34 |
    35 |
    36 |
    37 |
    38 |
    39 |
    40 |
    41 | 42 |
    43 | 44 |
    45 |
    46 |
    47 | 58 | 59 | -------------------------------------------------------------------------------- /ui/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Soundwave Home 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 |
    26 | {% include 'navbar.html' %} 27 |
    28 |
    29 | {% include 'query.html' %} 30 |
    31 |
    32 |
    33 |
    34 |
    35 |
    36 | 39 | 40 | {% for table_header in table_headers %} 41 | {% endfor %} 42 | 43 |
    {{ table_header }}
    44 |
    45 | 60 |
    61 | 62 | -------------------------------------------------------------------------------- /ui/templates/instance.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Soundwave Instance 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
    21 | {% include 'navbar.html' %} 22 |
    23 |
    24 |
    25 |
    26 |

    Instance Properties {{ id }}

    27 |
    28 |
    29 |
    30 |
    31 | 32 |
    33 | 34 |
    35 | 36 |
    37 | 38 | 43 |
    44 |
    45 | 56 | 57 | -------------------------------------------------------------------------------- /ui/templates/navbar.html: -------------------------------------------------------------------------------- 1 | 2 |
    3 |
    4 | Soundwave 5 | 10 |
    11 |
    -------------------------------------------------------------------------------- /ui/templates/query.html: -------------------------------------------------------------------------------- 1 | 2 |
    3 |
    4 |
    5 |
    6 |   7 | 8 |
    9 | 10 |
    11 | 12 | 13 |
    14 |
    15 | 16 |
    17 | 18 |
    19 | 23 |
    24 |
    25 |
    26 |
    27 |
    28 |
    29 | 30 |
    31 |
    32 | 37 | -------------------------------------------------------------------------------- /ui/templates/reservations.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Soundwave reservations 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 |
    26 | {% include 'navbar.html' %} 27 |
    28 |
    29 |
    30 |
    31 |
    32 | 33 | 34 |
    35 |
    36 | 37 |
    38 |
    39 |
    40 |
    41 |
    42 |
    43 |
    44 |
    45 |
    46 | 47 | 48 | {% for table_header in table_headers %} 49 | {% endfor %} 50 | 51 |
    {{ table_header }}
    52 |
    53 | 67 |
    68 | 69 | -------------------------------------------------------------------------------- /worker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Refer https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/ 2 | # for best practices maintaining this file 3 | 4 | # Build Java8 docker. Credit is to https://github.com/cogniteev/docker-oracle-java, 5 | # which is licensed under the MIT license 6 | # Pull base image. 7 | FROM ubuntu:14.04 8 | 9 | # Install Java. 10 | RUN \ 11 | apt-get update && \ 12 | apt-get install -y software-properties-common && \ 13 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 14 | add-apt-repository -y ppa:webupd8team/java && \ 15 | apt-get update && \ 16 | apt-get install -y oracle-java8-installer && \ 17 | rm -rf /var/lib/apt/lists/* && \ 18 | rm -rf /var/cache/oracle-jdk8-installer 19 | 20 | 21 | # Define working directory. 22 | WORKDIR /data 23 | 24 | # Define commonly used JAVA_HOME variable 25 | ENV JAVA_HOME /usr/lib/jvm/java-8-oracle 26 | 27 | 28 | # Create and set current directory 29 | WORKDIR /opt/soundwave-worker 30 | 31 | # Add the build artifact under /opt, can be overridden by docker build 32 | ARG ARTIFACT_PATH=target/soundwave-worker-0.1-SNAPSHOT-bin.tar.gz 33 | ADD $ARTIFACT_PATH /opt/soundwave-worker/ 34 | 35 | # Default command to run service, do not override it in docker run unless have a good reason 36 | # Use "docker logs ID" to view stdout and stderr 37 | CMD ["scripts/run_in_container.sh"] 38 | -------------------------------------------------------------------------------- /worker/checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /worker/config/log4j.properties: -------------------------------------------------------------------------------- 1 | # log4j logging configuration. 2 | # root logger. 3 | log4j.rootLogger=INFO, CONSOLE, ROLLINGFILE 4 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 5 | log4j.appender.CONSOLE.Threshold=INFO 6 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] (%F:%L) %-5p %m%n 8 | # we will have to rely on external cron job to keep the logging space consumption under control. 9 | log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.ROLLINGFILE.Threshold=INFO 11 | log4j.appender.ROLLINGFILE.File=/var/log/soundwave/soundwaveowrker.log 12 | log4j.appender.ROLLINGFILE.append=true 13 | log4j.appender.ROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 14 | log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout 15 | log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d [%-6p] [%t] %c{1} - %m%n 16 | 17 | # access logger. 18 | log4j.logger.access=INFO, ACCESSROLLINGFILE 19 | log4j.additivity.access=false 20 | # we will have to rely on external cron job to keep the logging space consumption under control. 21 | log4j.appender.ACCESSROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 22 | log4j.appender.ACCESSROLLINGFILE.File=/var/log/soundwaveworker/access.log 23 | log4j.appender.ACCESSROLLINGFILE.append=true 24 | log4j.appender.ACCESSROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 25 | log4j.appender.ACCESSROLLINGFILE.layout=org.apache.log4j.PatternLayout 26 | log4j.appender.ACCESSROLLINGFILE.layout.ConversionPattern=%m%n 27 | 28 | # failed request logger. 29 | log4j.logger.failure=INFO, FAILUREROLLINGFILE 30 | log4j.additivity.failure=false 31 | # we will have to rely on external cron job to keep the logging space consumption under control. 32 | log4j.appender.FAILUREROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 33 | log4j.appender.FAILUREROLLINGFILE.File=/var/log/soundwaveworker/failure.log 34 | log4j.appender.FAILUREROLLINGFILE.append=true 35 | log4j.appender.FAILUREROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 36 | log4j.appender.FAILUREROLLINGFILE.layout=org.apache.log4j.PatternLayout 37 | 38 | #Sqs events logger 39 | log4j.logger.sqsEventLogger=INFO, SQSEVENTROLLINGFILE 40 | log4j.appender.SQSEVENTROLLINGFILE=org.apache.log4j.DailyRollingFileAppender 41 | log4j.appender.SQSEVENTROLLINGFILE.File=/var/log/cmdb/sqsevents.log 42 | log4j.appender.SQSEVENTROLLINGFILE.append=true 43 | log4j.appender.SQSEVENTROLLINGFILE.DatePattern='.'yyyy-MM-dd-HH 44 | log4j.appender.SQSEVENTROLLINGFILE.layout=org.apache.log4j.PatternLayout 45 | log4j.appender.SQSEVENTROLLINGFILE.layout.ConversionPattern=%m%n 46 | 47 | #Message Processing API logger 48 | log4j.logger.messageProcessingLog=INFO, MESSAGEPROCESSINGFILE 49 | log4j.appender.MESSAGEPROCESSINGFILE=org.apache.log4j.DailyRollingFileAppender 50 | log4j.appender.MESSAGEPROCESSINGFILE.File=/var/log/soundwaveworker/messageprocessing.log 51 | log4j.appender.MESSAGEPROCESSINGFILE.append=true 52 | log4j.appender.MESSAGEPROCESSINGFILE.DatePattern='.'yyyy-MM-dd-HH 53 | log4j.appender.MESSAGEPROCESSINGFILE.layout=org.apache.log4j.PatternLayout 54 | log4j.appender.MESSAGEPROCESSINGFILE.layout.ConversionPattern=%m%n 55 | 56 | 57 | #Scheduled Job logger 58 | log4j.logger.scheduledJobLog=INFO, SCHEDULEDJOBFILE 59 | log4j.appender.SCHEDULEDJOBFILE=org.apache.log4j.DailyRollingFileAppender 60 | log4j.appender.SCHEDULEDJOBFILE.File=/var/log/soundwaveworker/scheduledjob.log 61 | log4j.appender.SCHEDULEDJOBFILE.append=true 62 | log4j.appender.SCHEDULEDJOBFILE.DatePattern='.'yyyy-MM-dd-HH 63 | log4j.appender.SCHEDULEDJOBFILE.layout=org.apache.log4j.PatternLayout 64 | log4j.appender.SCHEDULEDJOBFILE.layout.ConversionPattern=%m%n 65 | -------------------------------------------------------------------------------- /worker/config/soundwaveworker.properties: -------------------------------------------------------------------------------- 1 | #Common Service settings 2 | thrift_server_port=8090 3 | ostrich_port=9999 4 | service_name=cmdb 5 | cluster_name=cmdb 6 | max_connection_idle_in_minute=6 7 | max_concurrent_requests=2000 8 | server_set_hostname_prefix=cmdb 9 | server_set_enabled=true 10 | enforce_serverset_hostname_prefix=true 11 | server_set_path=/discovery/soundwave/prod 12 | slow_request_threshold_millis=1000 13 | 14 | #SQS endpoint that AWS lambda pushes Ec2 notification events 15 | update_queue= 16 | 17 | #ElasticSearch 18 | es_cluster_lb=soundwave-store 19 | es_cluster_port=9300 20 | es_instance_index = soundwave_prod 21 | es_daily_snapshot_index=soundwave_ss 22 | es_cluster_name=elasticsearch 23 | 24 | #Zookeeper 25 | zk_connection_string=soundwave-zk:2181 26 | zk_path =/soundwave/prod 27 | 28 | #JobManager Settings 29 | num_subscriber=4 30 | aws_region=us-east-1 31 | aws_call_ratelimit = 2 32 | send_service_mapping_to_aws=false 33 | 34 | #Ec2 Handler setting 35 | instance_factory = com.pinterest.soundwave.aws.BasicEsInstanceFactory 36 | 37 | aws_tag_generator = com.pinterest.BasicUploadTagsGenerator 38 | 39 | #Using IAM role or not 40 | use_instance_profile = false 41 | -------------------------------------------------------------------------------- /worker/scripts/index_settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "number_of_shards": 1, 4 | "number_of_replicas": 1, 5 | "analyzer": { 6 | "default": { 7 | "type": "standard" 8 | } 9 | }, 10 | "tokenizer": { 11 | "default": { 12 | "type": "standard" 13 | } 14 | } 15 | }, 16 | "mappings": { 17 | "_default_": { 18 | "dynamic_templates": [ 19 | { 20 | "template_catch": { 21 | "match": "*", 22 | "mapping": { 23 | "index": "not_analyzed" 24 | } 25 | } 26 | } 27 | ] 28 | }, 29 | "instance": { 30 | "dynamic_templates": [ 31 | { 32 | "template_time": { 33 | "match": "*_time", 34 | "mapping": { 35 | "type": "date", 36 | "format": "dateOptionalTime" 37 | } 38 | } 39 | }, 40 | { 41 | "template_address": { 42 | "match": "*_address", 43 | "mapping": { 44 | "type": "ip" 45 | } 46 | } 47 | } 48 | ] 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /worker/scripts/provision_index.sh: -------------------------------------------------------------------------------- 1 | curl -X PUT -d @index_settings.json $1 -------------------------------------------------------------------------------- /worker/scripts/run_in_container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Start the process inside a docker container with java installed 3 | # Some settings can be overriden by environment variables as shown below 4 | 5 | ulimit -n 65536 6 | 7 | 8 | export SERVICENAME=soundwave-worker 9 | export JAVA_MAIN=com.pinterest.soundwave.WorkerMain 10 | LOG4J_CONFIG_FILE=${LOG4J_CONFIG_FILE:=config/log4j.properties} 11 | CONFIG_FILE=${CONFIG_FILE:=config/soundwaveworker.properties} 12 | HEAP_SIZE=${HEAP_SIZE:=512m} 13 | NEW_SIZE=${NEW_SIZE:=256m} 14 | 15 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 16 | export PARENT_DIR="$(dirname $DIR)" 17 | 18 | LOG_DIR=/var/log/soundwave-worker 19 | CP=${PARENT_DIR}:${PARENT_DIR}/*:${PARENT_DIR}/lib/* 20 | 21 | java -server -Xmx${HEAP_SIZE} -Xms${HEAP_SIZE} -XX:NewSize=${NEW_SIZE} -XX:MaxNewSize=${NEW_SIZE} \ 22 | -verbosegc -Xloggc:${LOG_DIR}/gc.log \ 23 | -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 \ 24 | -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=100 -XX:GCLogFileSize=2M \ 25 | -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintClassHistogram \ 26 | -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParNewGC \ 27 | -XX:OnOutOfMemoryError="kill -9 %p" \ 28 | -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=60 -XX:+UseCMSInitiatingOccupancyOnly \ 29 | -XX:ErrorFile=${LOG_DIR}/jvm_error.log \ 30 | -cp ${CP} -Dlog4j.configuration=${LOG4J_CONFIG_FILE} -Dconfig.file=${CONFIG_FILE} \ 31 | -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false \ 32 | -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=10102 \ 33 | -Dfile.encoding=UTF-8 \ 34 | ${JAVA_MAIN} 35 | -------------------------------------------------------------------------------- /worker/src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | bin 4 | 5 | 6 | tar.gz 7 | 8 | 9 | 10 | scripts 11 | scripts 12 | 13 | *.sh 14 | 15 | 16 | 17 | config 18 | config 19 | 20 | *.properties 21 | *.xml 22 | 23 | 24 | 25 | target 26 | 27 | 28 | *.jar 29 | 30 | 31 | 32 | 33 | 34 | 35 | lib 36 | 37 | org.slf4j:slf4j-jdk14 38 | 39 | false 40 | runtime 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/AwsServiceTagUpdater.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest; 17 | 18 | import com.pinterest.soundwave.aws.Ec2InstanceStore; 19 | import com.pinterest.soundwave.bean.EsInstance; 20 | import com.pinterest.soundwave.aws.CloudInstanceStore; 21 | import com.pinterest.config.Configuration; 22 | 23 | import com.amazonaws.services.ec2.model.Tag; 24 | import com.google.common.base.Preconditions; 25 | import org.apache.commons.lang3.reflect.ConstructorUtils; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.util.Arrays; 30 | import java.util.List; 31 | 32 | public final class AwsServiceTagUpdater { 33 | 34 | private static final Logger logger = LoggerFactory.getLogger(AwsServiceTagUpdater.class); 35 | private static AwsServiceTagUpdater s_Instance = new AwsServiceTagUpdater(new Ec2InstanceStore()); 36 | 37 | private CloudInstanceStore cloudStore; 38 | private UploadTagsGenerator tagsGenerator; 39 | 40 | private AwsServiceTagUpdater(CloudInstanceStore store) { 41 | Preconditions.checkNotNull(store); 42 | this.cloudStore = store; 43 | String 44 | tagsGeneratorClass = 45 | Configuration.getProperties() 46 | .getString("aws_tag_generator", "BasicUploadTagsGenerator"); 47 | try { 48 | this.tagsGenerator = 49 | (UploadTagsGenerator) ConstructorUtils 50 | .invokeConstructor(Class.forName(tagsGeneratorClass), null); 51 | } catch (Exception ex) { 52 | 53 | } 54 | } 55 | 56 | public static AwsServiceTagUpdater getInstance() { 57 | return s_Instance; 58 | } 59 | 60 | public UploadTagsGenerator getTagsGenerator() { 61 | return tagsGenerator; 62 | } 63 | 64 | public void updateTags(EsInstance inst) { 65 | List updateTags = this.getTagsGenerator().getUpdateTags(inst); 66 | try { 67 | this.cloudStore.setTagsForInstances(Arrays.asList(inst.getId()), updateTags); 68 | } catch (Exception ex) { 69 | logger.warn("Faile to update tags for instance {}", inst.getId()); 70 | } 71 | } 72 | 73 | 74 | } 75 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/BasicUploadTagsGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest; 17 | 18 | import com.pinterest.soundwave.bean.EsInstance; 19 | 20 | import com.amazonaws.services.ec2.model.Tag; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | public class BasicUploadTagsGenerator extends UploadTagsGenerator { 26 | 27 | @Override 28 | public List getUpdateTags(EsInstance esInstance) { 29 | return new ArrayList<>(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/DummyOwnershipDecider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest; 17 | 18 | import com.pinterest.job.OwnershipDecider; 19 | 20 | /** 21 | * A dummy ownership decider that always return true 22 | */ 23 | public class DummyOwnershipDecider implements OwnershipDecider { 24 | 25 | @Override 26 | public boolean isOwner() { 27 | return true; 28 | } 29 | 30 | @Override 31 | public String getNodeName() { 32 | return "dummy"; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/DummyRecurringJob.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest; 17 | 18 | import java.util.Random; 19 | import java.util.concurrent.Callable; 20 | 21 | public class DummyRecurringJob implements Callable { 22 | 23 | private static Random random = new Random(System.currentTimeMillis()); 24 | private int failRate; 25 | private int executionSeconds; 26 | 27 | public DummyRecurringJob(int failRate, int executionSeconds) { 28 | this.failRate = failRate; 29 | this.executionSeconds = executionSeconds; 30 | } 31 | 32 | @Override 33 | public Boolean call() throws Exception { 34 | Thread.sleep(executionSeconds * 1000); 35 | if (failRate >= 100 - random.nextInt(100)) { 36 | //Either throw or return false 37 | if (random.nextBoolean()) { 38 | throw new Exception("Got an exception"); 39 | } else { 40 | return false; 41 | } 42 | } 43 | return true; 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/UploadTagsGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest; 17 | 18 | 19 | import com.pinterest.soundwave.bean.EsInstance; 20 | 21 | import com.amazonaws.services.ec2.model.Instance; 22 | import com.amazonaws.services.ec2.model.Tag; 23 | import com.google.common.base.Preconditions; 24 | 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | public abstract class UploadTagsGenerator { 29 | 30 | /** 31 | * Return a list of tags that need to be updated to the Ec2Instance. 32 | * The tags are either not in Ec2Instance tags or having different 33 | * values 34 | * @param ec2Instance 35 | * @param esInstance 36 | * @return A list of tags 37 | */ 38 | public List getUpdateTags(Instance ec2Instance, EsInstance esInstance) { 39 | Preconditions.checkNotNull(ec2Instance); 40 | Preconditions.checkNotNull(esInstance); 41 | List updateTags = new ArrayList<>(); 42 | 43 | List currentEc2Tag = ec2Instance.getTags(); 44 | List esUploadTags = getUpdateTags(esInstance); 45 | 46 | for (Tag tag : esUploadTags) { 47 | boolean shouldUpdate = true; 48 | for (Tag ec2Tag : currentEc2Tag) { 49 | if (ec2Tag.getKey().equals(tag.getKey()) && ec2Tag.getValue().equals(tag.getValue())) { 50 | shouldUpdate = false; 51 | break; 52 | } 53 | } 54 | 55 | if (shouldUpdate) { 56 | updateTags.add(tag); 57 | } 58 | } 59 | 60 | return updateTags; 61 | 62 | } 63 | 64 | /** 65 | * Get all tags needed to be added to Ec2Instance 66 | * @param esInstance 67 | * @return A list of tags 68 | */ 69 | public abstract List getUpdateTags(EsInstance esInstance); 70 | } 71 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/soundwave/WorkerMain.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave; 17 | 18 | 19 | import com.pinterest.JobManager; 20 | import com.pinterest.soundwave.aws.Ec2InstanceStore; 21 | import com.pinterest.soundwave.pinterest.EsInstanceStore; 22 | import com.pinterest.zookeeper.ZkJobInfoStore; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * Main method that initializes logger, service framework proxy, finagle server and ostrich 28 | */ 29 | public class WorkerMain { 30 | 31 | private static final Logger LOG = 32 | LoggerFactory.getLogger(WorkerMain.class); 33 | private static final Logger FAILURE_LOG = LoggerFactory.getLogger("failure"); 34 | private static final Logger SLOW_LOG = LoggerFactory.getLogger("slow"); 35 | 36 | private static final String MAX_CONNECTION_IDLE_CONFIG_KEY = "max_connection_idle_in_minute"; 37 | private static final String MAX_CONCURRENT_REQUESTS_KEY = "max_concurrent_requests"; 38 | private static final String OSTRICH_PORT_KEY = "ostrich_port"; 39 | private static final String SERVICE_NAME_KEY = "service_name"; 40 | private static final String CLUSTER_NAME_KEY = "cluster_name"; 41 | private static final String THRIFT_PORT_KEY = "thrift_server_port"; 42 | private static final String SERVER_SET_PATH_KEY = "server_set_path"; 43 | private static final String SLOW_REQUEST_THRESHOLD_MILLIS = "slow_request_threshold_millis"; 44 | 45 | public static void main(String[] args) { 46 | try { 47 | if (System.getProperty("config.file") == null) { 48 | System.setProperty("config.file", "soundwave.opensource.properties"); 49 | } 50 | //Start the Job Manager 51 | JobManager 52 | manager = 53 | new JobManager(new ZkJobInfoStore(), new Ec2InstanceStore(), new EsInstanceStore()); 54 | manager.start(); 55 | 56 | while (true) { 57 | Thread.sleep(1000); 58 | } 59 | 60 | //CHECKSTYLE_OFF: IllegalCatch 61 | } catch (Exception e) { 62 | LOG.error("Cannot start the service properly", e); 63 | System.exit(1); 64 | } 65 | //CHECKSTYLE_ON: IllegalCatch 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/soundwave/job/definitions/ElasticSearchHealthCheckJob.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.job.definitions; 17 | 18 | import com.pinterest.StatsUtil; 19 | import com.pinterest.soundwave.pinterest.EsInstanceStore; 20 | 21 | import com.twitter.ostrich.stats.Stats; 22 | import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; 23 | import org.elasticsearch.action.admin.indices.stats.IndexStats; 24 | 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | import java.util.concurrent.Callable; 28 | 29 | public class ElasticSearchHealthCheckJob implements Callable { 30 | 31 | private EsInstanceStore esInstanceStore = new EsInstanceStore(); 32 | 33 | @Override 34 | public Boolean call() throws Exception { 35 | logNodeStats(esInstanceStore.getNodesStats()); 36 | logIndexStats(esInstanceStore.getIndexStats()); 37 | return true; 38 | } 39 | 40 | private void logNodeStats(Map statsMap) { 41 | Map tags = new HashMap<>(); 42 | for (NodeStats stat : statsMap.values()) { 43 | tags.put("esnode", stat.getHostname()); 44 | Stats.setGauge(StatsUtil.getStatsName("eshealth", "heapUsedPercent", tags), 45 | stat.getJvm().getMem().getHeapUsedPrecent()); 46 | Stats.setGauge(StatsUtil.getStatsName("eshealth", "heapMaxMB", tags), 47 | stat.getJvm().getMem().getHeapMax().getMbFrac()); 48 | Stats.setGauge(StatsUtil.getStatsName("eshealth", "heapUsedMB", tags), 49 | stat.getJvm().getMem().getHeapUsed().getMbFrac()); 50 | Stats.setGauge(StatsUtil.getStatsName("eshealth", "upMinutes", tags), 51 | stat.getJvm().getUptime().getMinutesFrac()); 52 | Stats.setGauge(StatsUtil.getStatsName("eshealth", "docCount", tags), 53 | stat.getIndices().getDocs().getCount()); 54 | } 55 | } 56 | 57 | private void logIndexStats(Map indexStatsMap) { 58 | for (IndexStats stat : indexStatsMap.values()) { 59 | String indexKey = "esindex_" + stat.getIndex(); 60 | Stats.setGauge(StatsUtil.getStatsName(indexKey, "primaryDocCount"), 61 | stat.getPrimaries().getDocs().getCount()); 62 | Stats.setGauge(StatsUtil.getStatsName(indexKey, "totalDocCount"), 63 | stat.getTotal().getDocs().getCount()); 64 | Stats.setGauge(StatsUtil.getStatsName(indexKey, "shardsCount"), 65 | stat.getShards().length); 66 | Stats.setGauge(StatsUtil.getStatsName(indexKey, "primaryStoreSizeMB"), 67 | stat.getPrimaries().getStore().getSize().getMbFrac()); 68 | Stats.setGauge(StatsUtil.getStatsName(indexKey, "totalStoreSizeMB"), 69 | stat.getTotal().getStore().getSize().getMbFrac()); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /worker/src/main/java/com/pinterest/soundwave/job/definitions/HealthCheckJob.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Pinterest, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.pinterest.soundwave.job.definitions; 17 | 18 | import com.pinterest.StatsUtil; 19 | import com.pinterest.aws.AwsClientFactory; 20 | import com.pinterest.config.Configuration; 21 | 22 | import com.amazonaws.regions.Region; 23 | import com.amazonaws.regions.Regions; 24 | import com.amazonaws.services.sqs.AmazonSQSClient; 25 | import com.amazonaws.services.sqs.model.GetQueueAttributesResult; 26 | import com.twitter.ostrich.stats.Stats; 27 | import org.apache.commons.lang.exception.ExceptionUtils; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.util.Arrays; 32 | import java.util.Map; 33 | import java.util.concurrent.Callable; 34 | 35 | public class HealthCheckJob implements Callable { 36 | 37 | private static final Logger logger = LoggerFactory.getLogger(HealthCheckJob.class); 38 | private static final String QUEUELENGTHATTR = "ApproximateNumberOfMessages"; 39 | private static final String QUEUEINVISIBLEATTR = "ApproximateNumberOfMessagesNotVisible"; 40 | private final AmazonSQSClient sqsClient; 41 | private final String queueUrl; 42 | 43 | public HealthCheckJob() { 44 | sqsClient = AwsClientFactory.createSQSClient(Region.getRegion(Regions.US_EAST_1)); 45 | queueUrl = 46 | Configuration.getProperties().getString("update_queue"); 47 | } 48 | 49 | @Override 50 | public Boolean call() throws Exception { 51 | checkQueueLength(); 52 | return true; 53 | } 54 | 55 | private void checkQueueLength() { 56 | try { 57 | GetQueueAttributesResult 58 | result = 59 | sqsClient.getQueueAttributes(queueUrl, Arrays.asList(QUEUELENGTHATTR, 60 | QUEUEINVISIBLEATTR)); 61 | Map attrs = result.getAttributes(); 62 | 63 | if (attrs.containsKey(QUEUELENGTHATTR)) { 64 | Stats.addMetric(StatsUtil.getStatsName("healthcheck", "ec2queue_length"), 65 | Integer.parseInt(attrs.get(QUEUELENGTHATTR))); 66 | logger.info("Ec2 queue length is {}", attrs.get(QUEUELENGTHATTR)); 67 | } 68 | 69 | if (attrs.containsKey(QUEUEINVISIBLEATTR)) { 70 | Stats.addMetric(StatsUtil.getStatsName("healthcheck", "ec2queue_in_processing"), 71 | Integer.parseInt(attrs.get("ApproximateNumberOfMessagesNotVisible"))); 72 | logger.info("Ec2 queue in processing length is {}", attrs.get(QUEUEINVISIBLEATTR)); 73 | } 74 | 75 | } catch (Exception ex) { 76 | logger.warn(ExceptionUtils.getRootCauseMessage(ex)); 77 | logger.warn(ExceptionUtils.getFullStackTrace(ex)); 78 | } 79 | 80 | } 81 | } 82 | --------------------------------------------------------------------------------