├── .github └── workflows │ ├── nebula-ci.yml │ ├── nebula-publish.yml │ └── nebula-snapshot.yml ├── .gitignore ├── CONTRIBUTING.md ├── CONTRIBUTORS.md ├── LICENSE ├── OSSMETADATA ├── README.md ├── build.gradle ├── dependencies.lock ├── dynomitemanager-common ├── .gitignore ├── build.gradle ├── dependencies.lock └── src │ └── main │ └── java │ └── com │ └── netflix │ └── nfsidecar │ ├── aws │ ├── AWSMembership.java │ ├── AwsInstanceEnvIdentity.java │ ├── ClearCredential.java │ ├── IAMCredential.java │ ├── ICredential.java │ └── UpdateSecuritySettings.java │ ├── backup │ ├── Backup.java │ └── Restore.java │ ├── config │ ├── AWSCommonConfig.java │ ├── CassCommonConfig.java │ └── CommonConfig.java │ ├── configSource │ ├── AbstractConfigSource.java │ ├── CompositeConfigSource.java │ ├── DefaultConfigSource.java │ ├── IConfigSource.java │ ├── MemoryConfigSource.java │ ├── PropertiesConfigSource.java │ └── SystemPropertiesConfigSource.java │ ├── guice │ └── GuiceContext.java │ ├── identity │ ├── AppsInstance.java │ ├── IInstanceState.java │ ├── IMembership.java │ ├── InstanceEnvIdentity.java │ └── InstanceIdentity.java │ ├── instance │ ├── AwsInstanceDataRetriever.java │ ├── InstanceDataRetriever.java │ ├── LocalInstanceDataRetriever.java │ └── VpcInstanceDataRetriever.java │ ├── resources │ ├── SecurityGroupAdmin.java │ └── env │ │ ├── IEnvVariables.java │ │ └── InstanceEnvVariables.java │ ├── scheduler │ ├── BlockingSubmitThreadPoolExecutor.java │ ├── CronTimer.java │ ├── ExecutionException.java │ ├── GuiceJobFactory.java │ ├── NamedThreadPoolExecutor.java │ ├── SimpleTimer.java │ ├── Task.java │ ├── TaskMBean.java │ ├── TaskScheduler.java │ └── TaskTimer.java │ ├── supplier │ ├── EurekaHostSupplier.java │ ├── HostSupplier.java │ └── LocalHostSupplier.java │ ├── tokensdb │ ├── CassandraInstanceFactory.java │ ├── IAppsInstanceFactory.java │ └── InstanceDataDAOCassandra.java │ └── utils │ ├── BoundedExponentialRetryCallable.java │ ├── ExponentialRetryCallable.java │ ├── FifoQueue.java │ ├── ITokenManager.java │ ├── NFException.java │ ├── ProcessTuner.java │ ├── RetryableCallable.java │ ├── Sleeper.java │ ├── SystemUtils.java │ ├── ThreadSleeper.java │ └── TokenManager.java ├── dynomitemanager-core ├── .gitignore ├── build.gradle ├── dependencies.lock ├── java │ └── com │ │ └── netflix │ │ └── florida │ │ ├── defaultimpl │ │ └── test │ │ │ ├── BlankConfiguration.java │ │ │ ├── FakeEnvVariables.java │ │ │ ├── FakeInstanceDataRetriever.java │ │ │ ├── FakeInstanceIdentity.java │ │ │ ├── FakeInstanceState.java │ │ │ ├── FakeStorageProxy.java │ │ │ └── FloridaStandardTunerTest.java │ │ ├── sidecore │ │ └── monitoring │ │ │ └── test │ │ │ └── RedisInfoMetricsTaskTest.java │ │ └── utils │ │ └── test │ │ ├── ArdbConfParserTest.java │ │ ├── FakeSleeper.java │ │ ├── FloridaHealthCheckHandlerTest.java │ │ ├── RedisInfoParserTest.java │ │ ├── ServoMetricsTastTest.java │ │ └── TokenManagerTest.java └── src │ └── main │ └── java │ └── com │ └── netflix │ └── dynomitemanager │ ├── FloridaServer.java │ ├── aws │ ├── S3Backup.java │ └── S3Restore.java │ ├── backup │ ├── RestoreTask.java │ └── SnapshotTask.java │ ├── config │ ├── FloridaConfig.java │ └── InstanceState.java │ ├── dualAccount │ └── AwsRoleAssumptionCredential.java │ ├── dynomite │ ├── DynomiteProcessManager.java │ ├── DynomiteRest.java │ ├── DynomiteStandardTuner.java │ ├── DynomiteYamlTask.java │ ├── IDynomiteProcess.java │ └── ProxyAndStorageResetTask.java │ ├── monitoring │ ├── JedisFactory.java │ ├── ProcessMonitorTask.java │ ├── RedisInfoMetricsTask.java │ ├── ServoMetricsTask.java │ └── SimpleJedisFactory.java │ ├── resources │ └── DynomiteAdmin.java │ └── storage │ ├── ArdbRocksDbRedisCompatible.java │ ├── Bootstrap.java │ ├── JedisUtils.java │ ├── MemcachedStorageProxy.java │ ├── RedisInfoParser.java │ ├── RedisStorageProxy.java │ ├── StorageProcessManager.java │ ├── StorageProxy.java │ └── WarmBootstrapTask.java ├── dynomitemanager-web ├── .gitignore ├── build.gradle ├── dependencies.lock └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── netflix │ │ │ └── florida │ │ │ └── startup │ │ │ ├── Florida.java │ │ │ ├── FloridaModule.java │ │ │ └── JerseyModule.java │ ├── resources │ │ ├── application.properties │ │ └── laptop.properties │ └── webapp │ │ ├── favicon.ico │ │ └── index.html │ ├── smokeTest │ └── java │ │ └── com │ │ └── netflix │ │ └── florida │ │ └── SmokeTest.java │ └── test │ ├── TESTME.md │ └── testing-guidelines-diagram.png ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle /.github/workflows/nebula-ci.yml: -------------------------------------------------------------------------------- 1 | name: "CI" 2 | on: 3 | push: 4 | branches: 5 | - '*' 6 | tags-ignore: 7 | - '*' 8 | pull_request: 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | # test against JDK 8 16 | java: [ 8 ] 17 | name: CI with Java ${{ matrix.java }} 18 | steps: 19 | - uses: actions/checkout@v1 20 | - name: Setup git user 21 | run: | 22 | git config --global user.name "NetflixOss Maintainers" 23 | git config --global user.email "netflix-oss@netflix.com" 24 | - name: Setup jdk 25 | uses: actions/setup-java@v1 26 | with: 27 | java-version: ${{ matrix.java }} 28 | - uses: actions/cache@v1 29 | id: gradle-cache 30 | with: 31 | path: ~/.gradle/caches 32 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} 33 | restore-keys: | 34 | - ${{ runner.os }}-gradle- 35 | - uses: actions/cache@v1 36 | id: gradle-wrapper-cache 37 | with: 38 | path: ~/.gradle/wrapper 39 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 40 | restore-keys: | 41 | - ${{ runner.os }}-gradlewrapper- 42 | - name: Build with Gradle 43 | run: ./gradlew --info --stacktrace build 44 | env: 45 | CI_NAME: github_actions 46 | CI_BUILD_NUMBER: ${{ github.sha }} 47 | CI_BUILD_URL: 'https://github.com/${{ github.repository }}' 48 | CI_BRANCH: ${{ github.ref }} 49 | COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} 50 | -------------------------------------------------------------------------------- /.github/workflows/nebula-publish.yml: -------------------------------------------------------------------------------- 1 | name: "Publish candidate/release to NetflixOSS and Maven Central" 2 | on: 3 | push: 4 | tags: 5 | - v*.*.* 6 | - v*.*.*-rc.* 7 | release: 8 | types: 9 | - published 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v1 16 | - name: Setup git user 17 | run: | 18 | git config --global user.name "NetflixOss Maintainers" 19 | git config --global user.email "netflix-oss@netflix.com" 20 | - name: Setup jdk 8 21 | uses: actions/setup-java@v1 22 | with: 23 | java-version: 1.8 24 | - uses: actions/cache@v1 25 | id: gradle-cache 26 | with: 27 | path: ~/.gradle/caches 28 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} 29 | restore-keys: | 30 | - ${{ runner.os }}-gradle- 31 | - uses: actions/cache@v1 32 | id: gradle-wrapper-cache 33 | with: 34 | path: ~/.gradle/wrapper 35 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 36 | restore-keys: | 37 | - ${{ runner.os }}-gradlewrapper- 38 | - name: Publish candidate 39 | if: contains(github.ref, '-rc.') 40 | run: ./gradlew --info --stacktrace -Prelease.useLastTag=true candidate 41 | env: 42 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 43 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 44 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 45 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 46 | - name: Publish release 47 | if: (!contains(github.ref, '-rc.')) 48 | run: ./gradlew --info -Prelease.useLastTag=true final 49 | env: 50 | NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} 51 | NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} 52 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 53 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 54 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 55 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 56 | -------------------------------------------------------------------------------- /.github/workflows/nebula-snapshot.yml: -------------------------------------------------------------------------------- 1 | name: "Publish snapshot to NetflixOSS and Maven Central" 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | - name: Setup git user 16 | run: | 17 | git config --global user.name "NetflixOss Maintainers" 18 | git config --global user.email "netflix-oss@netflix.com" 19 | - name: Set up JDK 20 | uses: actions/setup-java@v1 21 | with: 22 | java-version: 8 23 | - uses: actions/cache@v2 24 | id: gradle-cache 25 | with: 26 | path: | 27 | ~/.gradle/caches 28 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} 29 | - uses: actions/cache@v2 30 | id: gradle-wrapper-cache 31 | with: 32 | path: | 33 | ~/.gradle/wrapper 34 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 35 | - name: Build 36 | run: ./gradlew build snapshot 37 | env: 38 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 39 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 40 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 41 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Gradle 2 | .gradle 3 | !gradle/wrapper/gradle-wrapper.jar 4 | .gradle/ 5 | 6 | # Builds 7 | build 8 | cleanup.sh 9 | 10 | # IDE 11 | .idea 12 | *.iws 13 | *.iml 14 | *.ipr 15 | 16 | # Mac files 17 | .DS_Store 18 | lib-cov 19 | 20 | # Packages 21 | *.jar 22 | *.war 23 | *.ear 24 | 25 | # Temp files / working directories 26 | *.swp 27 | *.*~ 28 | *.bkp 29 | 30 | # Compressed files 31 | *.bz 32 | *.gz 33 | *.tar 34 | *.zip 35 | *.tgz 36 | 37 | /bin/ 38 | *.classpath 39 | *.project 40 | */.settings/* 41 | 42 | # publishing secrets 43 | secrets/signing-key 44 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Dynomite-Manager 2 | 3 | Documentation for Dynomite is available in the following locations: 4 | - [wiki]( https://github.com/Netflix/dynomite-manager/wiki ) 5 | 6 | External Documentation: 7 | - Dynomite docs 8 | 9 | The Dynomite team is following the Gitflow workflow. The active development branch is [dev](https://github.com/Netflix/dynomite-manager/tree/dev), the stable branch is [master](https://github.com/Netflix/dynomite-manager/tree/master). 10 | 11 | Contributions will be accepted to the [dev](https://github.com/Netflix/dynomite-manager/tree/dev) only. 12 | 13 | 14 | 15 | 16 | ## How to provide a patch for a new feature 17 | 18 | 1. If it is a major feature, please create an [Issue]( https://github.com/Netflix/dynomite-manager/issues ) and discuss with the project leaders. 19 | 20 | 2. If in step 1 you get an acknowledge from the project leaders, use the 21 | following procedure to submit a patch: 22 | 23 | a. Fork Dynomite on github ( http://help.github.com/fork-a-repo/ ) 24 | 25 | b. Create a topic branch (git checkout -b my_branch) 26 | 27 | c. Push to your branch (git push origin my_branch) 28 | 29 | d. Initiate a pull request on github ( http://help.github.com/send-pull-requests/ ) 30 | 31 | e. Done :) 32 | 33 | For minor fixes just open a pull request to the [dev]( https://github.com/Netflix/dynomite-manager/tree/dev ) branch on Github. Make sure to add your name in the [Contributors](https://github.com/Netflix/dynomite-manager/blob/dev/CONTRIBUTORS.md) list. 34 | 35 | ## Questions 36 | 37 | If you have questions or want to report a bug please create an [Issue]( https://github.com/Netflix/dynomite-manager/issues ) or chat with us on [![Dev chat at https://gitter.im/Netflix/dynomite](https://badges.gitter.im/Netflix/dynomite.svg)](https://gitter.im/Netflix/dynomite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge). 38 | 39 | 40 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | # Dynomite Manager contributors (sorted alphabetically) 2 | 3 | Non-Netflix contributors: 4 | 5 | - [Akbar S. Ahmed](https://github.com/akbarahmed) 6 | - Archaius integration (configuration) 7 | - [Diego Pacheco](https://github.com/diegopacheco) 8 | 9 | [Full contributors list](https://github.com/Netflix/dynomite-manager/graphs/contributors) 10 | 11 | -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | 2 | buildscript { 3 | repositories { 4 | mavenCentral() 5 | maven { 6 | url = 'https://plugins.gradle.org/m2' 7 | } 8 | } 9 | dependencies { 10 | classpath "org.gretty:gretty:3.1.2" 11 | } 12 | } 13 | 14 | plugins { 15 | id "com.netflix.nebula.netflixoss" version "11.5.0" 16 | } 17 | 18 | ext.githubProjectName = rootProject.name // Change if github project name is not the same as the root project's name 19 | 20 | /*apply from: file('gradle/convention.gradle') 21 | apply from: file('gradle/maven.gradle') 22 | apply from: file('gradle/check.gradle') 23 | apply from: file('gradle/license.gradle') 24 | apply from: file('gradle/release.gradle') 25 | */ 26 | 27 | 28 | subprojects { 29 | apply plugin: 'nebula.netflixoss' 30 | apply plugin: 'java' 31 | apply plugin: 'idea' 32 | apply plugin: 'eclipse' 33 | 34 | group = "com.netflix.${githubProjectName}" 35 | 36 | 37 | sourceCompatibility = '1.8' 38 | targetCompatibility = '1.8' 39 | 40 | repositories { 41 | mavenCentral() 42 | } 43 | 44 | dependencies { 45 | //StringUtils 46 | implementation 'commons-lang:commons-lang:2.6' 47 | 48 | //Jersey & Servlet 49 | compileOnly 'javax.servlet:servlet-api:2.5' 50 | 51 | //Logging 52 | implementation 'org.slf4j:slf4j-api:1.6.1' 53 | implementation 'org.slf4j:slf4j-log4j12:1.6.1' 54 | 55 | // Archaius-2 56 | implementation group: 'com.netflix.archaius', name: 'archaius2-core', version: '2.1.11' 57 | 58 | //Unit Testing 59 | testImplementation 'org.jmockit:jmockit:1.19' 60 | testImplementation 'junit:junit:latest.release' 61 | 62 | //Google Injections 63 | implementation 'com.google.inject:guice:4.0' 64 | implementation 'com.google.inject.extensions:guice-multibindings:4.0' 65 | implementation 'javax.inject:javax.inject:1' 66 | 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /dependencies.lock: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } -------------------------------------------------------------------------------- /dynomitemanager-common/.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /bin/ 3 | -------------------------------------------------------------------------------- /dynomitemanager-common/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java-library' 2 | 3 | dependencies { 4 | // Archaius-2 5 | api group: 'com.netflix.archaius', name: 'archaius2-guice', version: '2.1.11' 6 | api group: 'com.netflix.archaius', name: 'archaius2-api', version: '2.1.11' 7 | 8 | api "com.google.inject:guice:3.0" 9 | api "com.google.guava:guava:19.0" 10 | 11 | api 'com.amazonaws:aws-java-sdk:1.11.172' 12 | api 'com.amazonaws:aws-java-sdk-core:1.11.172' 13 | api "org.quartz-scheduler:quartz:1.7.3" 14 | api 'javax.inject:javax.inject:1' 15 | api "org.slf4j:slf4j-log4j12:1.6.1" 16 | api 'commons-io:commons-io:2.5' 17 | api 'commons-cli:commons-cli:1.3.1' 18 | api "javax.ws.rs:jsr311-api:1.1.1" 19 | api "joda-time:joda-time:2.0" 20 | api "commons-configuration:commons-configuration:1.5" 21 | api('com.netflix.astyanax:astyanax:2.0.1') { 22 | exclude module: 'junit' 23 | } 24 | 25 | //Unit test 26 | testImplementation 'junit:junit:4.+' 27 | testImplementation 'org.mockito:mockito-all:1.+' 28 | testImplementation 'org.assertj:assertj-core:3.3.0' 29 | } 30 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/aws/AwsInstanceEnvIdentity.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.aws; 2 | 3 | import com.netflix.nfsidecar.identity.InstanceEnvIdentity; 4 | import com.netflix.nfsidecar.instance.InstanceDataRetriever; 5 | import com.netflix.nfsidecar.instance.VpcInstanceDataRetriever; 6 | 7 | /** 8 | * A means to determine if running instance is within classic, default vpc account, or non-default vpc account 9 | */ 10 | public class AwsInstanceEnvIdentity implements InstanceEnvIdentity { 11 | 12 | private Boolean isClassic = false, isDefaultVpc = false, isNonDefaultVpc = false; 13 | 14 | public AwsInstanceEnvIdentity() { 15 | String vpcId = getVpcId(); 16 | if (vpcId == null || vpcId.isEmpty()) { 17 | this.isClassic = true; 18 | } else { 19 | this.isNonDefaultVpc = true; // our instances run under a non 20 | // default ("persistence_*") AWS acct 21 | } 22 | } 23 | 24 | /* 25 | * @return the vpc id of the running instance, null if instance is not 26 | * running within vpc. 27 | */ 28 | private String getVpcId() { 29 | InstanceDataRetriever insDataRetriever = new VpcInstanceDataRetriever(); 30 | return insDataRetriever.getVpcId(); 31 | } 32 | 33 | @Override 34 | public Boolean isClassic() { 35 | return this.isClassic; 36 | } 37 | 38 | @Override 39 | public Boolean isDefaultVpc() { 40 | return this.isDefaultVpc; 41 | } 42 | 43 | @Override 44 | public Boolean isNonDefaultVpc() { 45 | return this.isNonDefaultVpc; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/aws/ClearCredential.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.aws; 17 | 18 | import java.io.FileInputStream; 19 | import java.io.IOException; 20 | import java.util.Properties; 21 | 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import com.amazonaws.auth.AWSCredentials; 26 | import com.amazonaws.auth.AWSCredentialsProvider; 27 | import com.amazonaws.auth.BasicAWSCredentials; 28 | 29 | /** 30 | * This is a basic implementation of ICredentials. User should prefer to 31 | * implement their own versions for more secured access. This class requires 32 | * clear AWS key and access. 33 | * 34 | * Set the following properties in "conf/awscredntial.properties" 35 | * 36 | */ 37 | public class ClearCredential implements ICredential { 38 | private static final Logger logger = LoggerFactory.getLogger(ClearCredential.class); 39 | private static final String CRED_FILE = "/etc/awscredential.properties"; 40 | private final Properties props; 41 | private final String AWS_ACCESS_ID; 42 | private final String AWS_KEY; 43 | 44 | public ClearCredential() { 45 | FileInputStream fis = null; 46 | try { 47 | fis = new FileInputStream(CRED_FILE); 48 | props = new Properties(); 49 | props.load(fis); 50 | AWS_ACCESS_ID = props.getProperty("AWSACCESSID") != null ? props.getProperty("AWSACCESSID").trim() : ""; 51 | AWS_KEY = props.getProperty("AWSKEY") != null ? props.getProperty("AWSKEY").trim() : ""; 52 | } catch (Exception e) { 53 | logger.error("Exception with credential file ", e); 54 | throw new RuntimeException("Problem reading credential file. Cannot start.", e); 55 | } finally { 56 | try { 57 | fis.close(); 58 | } catch (IOException e) { 59 | e.printStackTrace(); 60 | } 61 | } 62 | 63 | } 64 | 65 | public String getAccessKeyId() { 66 | return AWS_ACCESS_ID; 67 | } 68 | 69 | public String getSecretAccessKey() { 70 | return AWS_KEY; 71 | } 72 | 73 | public AWSCredentials getCredentials() { 74 | return new BasicAWSCredentials(getAccessKeyId(), getSecretAccessKey()); 75 | } 76 | 77 | @Override 78 | public AWSCredentialsProvider getAwsCredentialProvider() { 79 | return new AWSCredentialsProvider() { 80 | public AWSCredentials getCredentials() { 81 | return ClearCredential.this.getCredentials(); 82 | } 83 | 84 | @Override 85 | public void refresh() { 86 | // NOP 87 | } 88 | }; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/aws/IAMCredential.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.aws; 17 | 18 | import com.amazonaws.auth.AWSCredentialsProvider; 19 | import com.amazonaws.auth.InstanceProfileCredentialsProvider; 20 | 21 | public class IAMCredential implements ICredential 22 | { 23 | private final InstanceProfileCredentialsProvider iamCredProvider; 24 | 25 | public IAMCredential() 26 | { 27 | this.iamCredProvider = new InstanceProfileCredentialsProvider(); 28 | } 29 | 30 | public AWSCredentialsProvider getAwsCredentialProvider() 31 | { 32 | return iamCredProvider; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/aws/ICredential.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.aws; 17 | 18 | import com.amazonaws.auth.AWSCredentialsProvider; 19 | import com.google.inject.ImplementedBy; 20 | 21 | /** 22 | * Credential file interface for services supporting 23 | * Access ID and key authentication 24 | */ 25 | @ImplementedBy(ClearCredential.class) 26 | public interface ICredential 27 | { 28 | 29 | public AWSCredentialsProvider getAwsCredentialProvider(); 30 | } 31 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/aws/UpdateSecuritySettings.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.aws; 17 | 18 | import java.util.List; 19 | import java.util.Random; 20 | 21 | import com.google.common.collect.Lists; 22 | import com.google.inject.Inject; 23 | import com.google.inject.Singleton; 24 | import com.netflix.nfsidecar.config.CommonConfig; 25 | import com.netflix.nfsidecar.identity.AppsInstance; 26 | import com.netflix.nfsidecar.identity.IMembership; 27 | import com.netflix.nfsidecar.identity.InstanceIdentity; 28 | import com.netflix.nfsidecar.resources.env.IEnvVariables; 29 | import com.netflix.nfsidecar.scheduler.SimpleTimer; 30 | import com.netflix.nfsidecar.scheduler.Task; 31 | import com.netflix.nfsidecar.scheduler.TaskTimer; 32 | import com.netflix.nfsidecar.tokensdb.IAppsInstanceFactory; 33 | 34 | @Singleton 35 | public class UpdateSecuritySettings extends Task { 36 | public static final String JOBNAME = "Update_SG"; 37 | public static boolean firstTimeUpdated = false; 38 | 39 | private static final Random ran = new Random(); 40 | private final IMembership membership; 41 | private final IAppsInstanceFactory factory; 42 | private final IEnvVariables envVariables; 43 | private final CommonConfig config; 44 | 45 | @Inject 46 | public UpdateSecuritySettings(CommonConfig config, IMembership membership, IAppsInstanceFactory factory, 47 | IEnvVariables envVariables) { 48 | this.config = config; 49 | this.membership = membership; 50 | this.factory = factory; 51 | this.envVariables = envVariables; 52 | } 53 | 54 | @Override 55 | public void execute() { 56 | // if seed dont execute. 57 | int port = config.getDynomitePeerPort(); 58 | List acls = membership.listACL(port, port); 59 | List instances = factory.getAllIds(envVariables.getDynomiteClusterName()); 60 | 61 | // iterate to add... 62 | List add = Lists.newArrayList(); 63 | for (AppsInstance instance : factory.getAllIds(envVariables.getDynomiteClusterName())) { 64 | String range = instance.getHostIP() + "/32"; 65 | if (!acls.contains(range)) 66 | add.add(range); 67 | } 68 | if (add.size() > 0) { 69 | membership.addACL(add, port, port); 70 | firstTimeUpdated = true; 71 | } 72 | 73 | // just iterate to generate ranges. 74 | List currentRanges = Lists.newArrayList(); 75 | for (AppsInstance instance : instances) { 76 | String range = instance.getHostIP() + "/32"; 77 | currentRanges.add(range); 78 | } 79 | 80 | // iterate to remove... 81 | List remove = Lists.newArrayList(); 82 | for (String acl : acls) 83 | if (!currentRanges.contains(acl)) // if not found then remove.... 84 | remove.add(acl); 85 | if (remove.size() > 0) { 86 | membership.removeACL(remove, port, port); 87 | firstTimeUpdated = true; 88 | } 89 | } 90 | 91 | public static TaskTimer getTimer(InstanceIdentity id) { 92 | SimpleTimer return_; 93 | if (id.isSeed()) 94 | return_ = new SimpleTimer(JOBNAME, 120 * 1000 + ran.nextInt(120 * 1000)); 95 | else 96 | return_ = new SimpleTimer(JOBNAME); 97 | return return_; 98 | } 99 | 100 | @Override 101 | public String getName() { 102 | return JOBNAME; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/backup/Backup.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.backup; 2 | 3 | import java.io.File; 4 | 5 | import org.joda.time.DateTime; 6 | 7 | public interface Backup { 8 | boolean upload(File file, DateTime todayStart); 9 | } 10 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/backup/Restore.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.backup; 2 | 3 | public interface Restore { 4 | boolean restoreData(String dateString); 5 | } 6 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/config/AWSCommonConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.config; 17 | 18 | import com.netflix.archaius.api.annotations.Configuration; 19 | import com.netflix.archaius.api.annotations.DefaultValue; 20 | import com.netflix.archaius.api.annotations.PropertyName; 21 | 22 | @Configuration(prefix = "dbsidecar.aws") 23 | public interface AWSCommonConfig { 24 | 25 | // Dual Account 26 | /* 27 | * @return the Amazon Resource Name (ARN) for EC2 classic. 28 | */ 29 | @DefaultValue("null") 30 | @PropertyName(name = "ec2.roleassumption.arn") 31 | public String getClassicAWSRoleAssumptionArn(); 32 | 33 | /* 34 | * @return the Amazon Resource Name (ARN) for VPC. 35 | */ 36 | @DefaultValue("null") 37 | @PropertyName(name = "vpc.roleassumption.arn") 38 | public String getVpcAWSRoleAssumptionArn(); 39 | 40 | @DefaultValue("false") 41 | @PropertyName(name = "roleassumption.dualaccount") 42 | public boolean isDualAccount(); 43 | 44 | // Backup and Restore 45 | 46 | @DefaultValue("us-east-1.dynomite-backup-test") 47 | @PropertyName(name = "dyno.backup.bucket.name") // TODO: For a common 48 | // default value we probably 49 | // have to result to defined 50 | // FP 51 | public String getBucketName(); 52 | 53 | @DefaultValue("backup") 54 | @PropertyName(name = "dyno.backup.s3.base_dir") // TODO: For a common 55 | // default value we probably 56 | // have to result to defined 57 | // FP 58 | public String getBackupLocation(); 59 | 60 | } 61 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/config/CassCommonConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.config; 17 | 18 | import com.netflix.archaius.api.annotations.Configuration; 19 | import com.netflix.archaius.api.annotations.DefaultValue; 20 | import com.netflix.archaius.api.annotations.PropertyName; 21 | 22 | @Configuration(prefix = "dbsidecar.cass") 23 | public interface CassCommonConfig { 24 | 25 | /** 26 | * @return Bootstrap cluster name (depends on another cass cluster) 27 | */ 28 | @DefaultValue("cass_turtle") 29 | @PropertyName(name = "dyno.sidecore.clusterName") 30 | public String getCassandraClusterName(); 31 | 32 | /** 33 | * @return if Eureka is used to find the bootstrap cluster 34 | */ 35 | @DefaultValue("false") 36 | @PropertyName(name = "dyno.sidecore.eureka.enabled") 37 | public boolean isEurekaHostsSupplierEnabled(); 38 | 39 | /** 40 | * @return the port that the bootstrap cluster can be contacted 41 | */ 42 | @DefaultValue("7102") 43 | @PropertyName(name = "dyno.sidecore.port") 44 | public int getCassandraThriftPort(); 45 | 46 | @DefaultValue("127.0.0.1") 47 | @PropertyName(name = "dyno.sidecore.seeds") 48 | public String getCassandraSeeds(); 49 | 50 | /** 51 | * Get the name of the keyspace that stores tokens for the Dynomite cluster. 52 | * 53 | * @return the keyspace name 54 | */ 55 | @DefaultValue("dyno_bootstrap") 56 | @PropertyName(name = "metadata.keyspace") 57 | public String getCassandraKeyspaceName(); 58 | 59 | /** 60 | * @return the refresh interval in msecs for getting the tokens 61 | * 0 value means, do not cache the tokens. Every query to Dynomite-manager 62 | * to get tokens will be forwarded to the token store 63 | */ 64 | @DefaultValue("0") 65 | @PropertyName(name = "dyno.sidecore.tokenRefreshInterval") 66 | public long getTokenRefreshInterval(); 67 | 68 | } 69 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/config/CommonConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.config; 17 | 18 | import java.util.List; 19 | 20 | import com.netflix.archaius.api.annotations.Configuration; 21 | import com.netflix.archaius.api.annotations.DefaultValue; 22 | import com.netflix.archaius.api.annotations.PropertyName; 23 | 24 | @Configuration(prefix = "dbsidecar.common") 25 | public interface CommonConfig { 26 | 27 | /** 28 | * @return Get the Region name 29 | */ 30 | @DefaultValue("") 31 | @PropertyName(name = "region") 32 | public String getRegion(); 33 | 34 | @DefaultValue("") 35 | @PropertyName(name = "rack") 36 | public String getRack(); 37 | 38 | @PropertyName(name = "zones.available") 39 | public List getRacks(); 40 | 41 | 42 | /** 43 | * Get the security group associated with nodes in this cluster 44 | */ 45 | @PropertyName(name = "acl.groupname") 46 | public String getACLGroupName(); 47 | 48 | /*****************************************************************/ 49 | 50 | /** 51 | * Get the peer-to-peer port used by Dynomite to communicate with other 52 | * Dynomite nodes. 53 | * 54 | * @return the peer-to-peer port used for intra-cluster communication 55 | */ 56 | @DefaultValue("8101") 57 | @PropertyName(name = "dyno.peer.port") 58 | public int getDynomitePeerPort(); 59 | 60 | @DefaultValue("8102") 61 | @PropertyName(name = "dyno.port") 62 | public int getDynomitePort(); 63 | 64 | @DefaultValue("22222") 65 | @PropertyName(name = "dyno.stats.port") 66 | public int getDynomiteStatsPort(); 67 | 68 | @DefaultValue("true") 69 | @PropertyName(name = "dyno.stats.localhost.only") 70 | public boolean isDynomiteStatsLocalHostOnly(); 71 | 72 | // Default value of -1 means it is not having a secure port 73 | @DefaultValue("-1") 74 | @PropertyName(name = "dyno.secure.port") 75 | public int getDynomiteSecurePort(); 76 | 77 | // Default value of -1 means it is not having a secure port 78 | @DefaultValue("-1") 79 | @PropertyName(name = "dyno.secure.storage.port") 80 | public int getDynomiteSecureStoragePort(); 81 | 82 | @DefaultValue("false") 83 | @PropertyName(name = "dyno.backup.snapshot.enabled") 84 | public boolean isBackupEnabled(); 85 | 86 | @DefaultValue("false") 87 | @PropertyName(name = "dyno.backup.restore.enabled") 88 | public boolean isRestoreEnabled(); 89 | 90 | @DefaultValue("day") 91 | @PropertyName(name = "dyno.backup.schedule") 92 | public String getBackupSchedule(); 93 | 94 | @DefaultValue("12") 95 | @PropertyName(name = "dyno.backup.hour") 96 | public int getBackupHour(); 97 | 98 | @DefaultValue("20101010") 99 | @PropertyName(name = "dyno.backup.restore.date") 100 | public String getRestoreDate(); 101 | 102 | 103 | 104 | 105 | } 106 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/configSource/CompositeConfigSource.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.configSource; 2 | 3 | import java.util.Collection; 4 | 5 | import com.google.common.base.Preconditions; 6 | import com.google.common.collect.ImmutableCollection; 7 | import com.google.common.collect.ImmutableList; 8 | import com.google.common.collect.Iterables; 9 | import com.google.inject.Inject; 10 | import com.google.inject.Singleton; 11 | 12 | /** 13 | * A {@link IConfigSource} that delegates method calls to the underline sources. The order in which values are provided 14 | * depend on the {@link IConfigSource}s provided. If user asks for key 'foo', and this composite has three sources, it 15 | * will first check if the key is found in the first source, if not it will check the second and if not, the third, else 16 | * return null or false if {@link #contains(String)} was called. 17 | * 18 | * Implementation note: get methods with a default are implemented in {@link AbstractConfigSource}, if the underlying 19 | * source overrides one of these methods, then that implementation will be ignored. 20 | */ 21 | @Singleton 22 | public class CompositeConfigSource extends AbstractConfigSource 23 | { 24 | 25 | private final ImmutableCollection sources; 26 | 27 | @Inject 28 | public CompositeConfigSource(final ImmutableCollection sources) 29 | { 30 | Preconditions.checkArgument(!sources.isEmpty(), "Can not create a composite config source without config sources!"); 31 | this.sources = sources; 32 | } 33 | 34 | public CompositeConfigSource(final Collection sources) 35 | { 36 | this(ImmutableList.copyOf(sources)); 37 | } 38 | 39 | public CompositeConfigSource(final Iterable sources) 40 | { 41 | this(ImmutableList.copyOf(sources)); 42 | } 43 | 44 | public CompositeConfigSource(final IConfigSource... sources) 45 | { 46 | this(ImmutableList.copyOf(sources)); 47 | } 48 | 49 | @Override 50 | public void intialize(final String asgName, final String region) 51 | { 52 | for (final IConfigSource source : sources) 53 | { 54 | //TODO should this catch any potential exceptions? 55 | source.intialize(asgName, region); 56 | } 57 | } 58 | 59 | @Override 60 | public int size() 61 | { 62 | int size = 0; 63 | for (final IConfigSource c : sources) 64 | { 65 | size += c.size(); 66 | } 67 | return size; 68 | } 69 | 70 | @Override 71 | public boolean isEmpty() 72 | { 73 | return size() == 0; 74 | } 75 | 76 | @Override 77 | public boolean contains(final String key) 78 | { 79 | return get(key) != null; 80 | } 81 | 82 | @Override 83 | public String get(final String key) 84 | { 85 | Preconditions.checkNotNull(key); 86 | for (final IConfigSource c : sources) 87 | { 88 | final String value = c.get(key); 89 | if (value != null) 90 | { 91 | return value; 92 | } 93 | } 94 | return null; 95 | } 96 | 97 | @Override 98 | public void set(final String key, final String value) 99 | { 100 | Preconditions.checkNotNull(value, "Value can not be null for configurations."); 101 | final IConfigSource firstSource = Iterables.getFirst(sources, null); 102 | // firstSource shouldn't be null because the collection is immutable, and the collection is non empty. 103 | Preconditions.checkState(firstSource != null, "There was no IConfigSource found at the first location?"); 104 | firstSource.set(key, value); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/configSource/DefaultConfigSource.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.configSource; 2 | 3 | import javax.inject.Inject; 4 | 5 | 6 | public class DefaultConfigSource extends CompositeConfigSource { 7 | 8 | @Inject 9 | public DefaultConfigSource(final PropertiesConfigSource simpleDBConfigSource, 10 | final PropertiesConfigSource propertiesConfigSource, 11 | final SystemPropertiesConfigSource systemPropertiesConfigSource) { 12 | super(simpleDBConfigSource, propertiesConfigSource, systemPropertiesConfigSource); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/configSource/IConfigSource.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.configSource; 2 | 3 | import java.util.List; 4 | 5 | import com.google.inject.ImplementedBy; 6 | 7 | /** 8 | * Defines the configurations for an application. 9 | */ 10 | @ImplementedBy(DefaultConfigSource.class) 11 | public interface IConfigSource 12 | { 13 | 14 | /** 15 | * Must be called before any other method. This method will allow implementations to do any setup that they require 16 | * before being called. 17 | */ 18 | void intialize(String asgName, String region); 19 | 20 | /** 21 | * A non-negative integer indicating a count of elements. 22 | * 23 | * @return non-negative integer indicating a count of elements. 24 | */ 25 | int size(); 26 | 27 | /** 28 | * Returns {@code true} if the size is zero. May be more efficient than calculating size. 29 | * 30 | * @return {@code true} if the size is zero otherwise {@code false}. 31 | */ 32 | boolean isEmpty(); 33 | 34 | /** 35 | * Check if the given key can be found in the config. 36 | * 37 | * @param key to look up value. 38 | * @return if the key is present 39 | */ 40 | boolean contains(String key); 41 | 42 | /** 43 | * Get a String associated with the given configuration key. 44 | * 45 | * @param key to look up value. 46 | * @return value from config or null if not present. 47 | */ 48 | String get(String key); 49 | 50 | /** 51 | * Get a String associated with the given configuration key. 52 | * 53 | * @param key to look up value. 54 | * @param defaultValue if value is not present. 55 | * @return value from config or defaultValue if not present. 56 | */ 57 | String get(String key, String defaultValue); 58 | 59 | /** 60 | * Get a boolean associated with the given configuration key. 61 | * 62 | * @param key to look up value. 63 | * @param defaultValue if value is not present. 64 | * @return value from config or defaultValue if not present. 65 | */ 66 | boolean get(String key, boolean defaultValue); 67 | 68 | /** 69 | * Get a Class associated with the given configuration key. 70 | * 71 | * @param key to look up value. 72 | * @param defaultValue if value is not present. 73 | * @return value from config or defaultValue if not present. 74 | */ 75 | Class get(String key, Class defaultValue); 76 | 77 | /** 78 | * Get a Enum associated with the given configuration key. 79 | * 80 | * @param key to look up value. 81 | * @param defaultValue if value is not present. 82 | * @param enum type. 83 | * @return value from config or defaultValue if not present. 84 | */ 85 | > T get(String key, T defaultValue); 86 | 87 | /** 88 | * Get a int associated with the given configuration key. 89 | * 90 | * @param key to look up value. 91 | * @param defaultValue if value is not present. 92 | * @return value from config or defaultValue if not present. 93 | */ 94 | int get(String key, int defaultValue); 95 | 96 | /** 97 | * Get a long associated with the given configuration key. 98 | * 99 | * @param key to look up value. 100 | * @param defaultValue if value is not present. 101 | * @return value from config or defaultValue if not present. 102 | */ 103 | long get(String key, long defaultValue); 104 | 105 | /** 106 | * Get a float associated with the given configuration key. 107 | * 108 | * @param key to look up value. 109 | * @param defaultValue if value is not present. 110 | * @return value from config or defaultValue if not present. 111 | */ 112 | float get(String key, float defaultValue); 113 | 114 | /** 115 | * Get a double associated with the given configuration key. 116 | * 117 | * @param key to look up value. 118 | * @param defaultValue if value is not present. 119 | * @return value from config or defaultValue if not present. 120 | */ 121 | double get(String key, double defaultValue); 122 | 123 | /** 124 | * Get a list of strings associated with the given configuration key. 125 | * 126 | * @param key to look up value. 127 | * @return value from config or an immutable list if not present. 128 | */ 129 | List getList(String key); 130 | 131 | /** 132 | * Get a list of strings associated with the given configuration key. 133 | * 134 | * @param key to look up value. 135 | * @param defaultValue if value is not present. 136 | * @return value from config or defaultValue if not present. 137 | */ 138 | List getList(String key, List defaultValue); 139 | 140 | /** 141 | * Set the value for the given key. 142 | * 143 | * @param key to set value for. 144 | * @param value to set. 145 | */ 146 | void set(String key, String value); 147 | } 148 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/configSource/MemoryConfigSource.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.configSource; 2 | 3 | import com.google.common.collect.Maps; 4 | 5 | import java.util.Map; 6 | 7 | public final class MemoryConfigSource extends AbstractConfigSource 8 | { 9 | private final Map data = Maps.newConcurrentMap(); 10 | 11 | @Override 12 | public int size() 13 | { 14 | return data.size(); 15 | } 16 | 17 | @Override 18 | public String get(final String key) 19 | { 20 | return data.get(key); 21 | } 22 | 23 | @Override 24 | public void set(final String key, final String value) 25 | { 26 | data.put(key, value); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/configSource/PropertiesConfigSource.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.configSource; 2 | 3 | import com.google.common.annotations.VisibleForTesting; 4 | import com.google.common.base.Preconditions; 5 | import com.google.common.base.Strings; 6 | import com.google.common.collect.Maps; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.IOException; 12 | import java.net.URL; 13 | import java.util.Map; 14 | import java.util.Properties; 15 | 16 | import static com.google.common.base.Preconditions.checkNotNull; 17 | 18 | /** 19 | * Loads the 'florida.properties' file as a source. 20 | */ 21 | public class PropertiesConfigSource extends AbstractConfigSource 22 | { 23 | private static final Logger logger = LoggerFactory.getLogger(PropertiesConfigSource.class.getName()); 24 | 25 | private static final String DEFAULT_FLORIDA_PROPERTIES = "florida.properties"; 26 | 27 | private final Map data = Maps.newConcurrentMap(); 28 | private final String appsFile; 29 | 30 | public PropertiesConfigSource() 31 | { 32 | this.appsFile = DEFAULT_FLORIDA_PROPERTIES; 33 | } 34 | 35 | public PropertiesConfigSource(final Properties properties) 36 | { 37 | checkNotNull(properties); 38 | this.appsFile = DEFAULT_FLORIDA_PROPERTIES; 39 | clone(properties); 40 | } 41 | 42 | @VisibleForTesting 43 | PropertiesConfigSource(final String file) 44 | { 45 | this.appsFile = checkNotNull(file); 46 | } 47 | 48 | @Override 49 | public void intialize(final String asgName, final String region) 50 | { 51 | super.intialize(asgName, region); 52 | Properties properties = new Properties(); 53 | URL url = PropertiesConfigSource.class.getClassLoader().getResource(appsFile); 54 | if (url != null) 55 | { 56 | try 57 | { 58 | properties.load(url.openStream()); 59 | clone(properties); 60 | } 61 | catch (IOException e) 62 | { 63 | logger.info("No Dynomite.properties. Ignore!"); 64 | } 65 | } 66 | else 67 | { 68 | logger.info("No Dynomite.properties. Ignore!"); 69 | } 70 | } 71 | 72 | @Override 73 | public String get(final String prop) 74 | { 75 | return data.get(prop); 76 | } 77 | 78 | @Override 79 | public void set(final String key, final String value) 80 | { 81 | Preconditions.checkNotNull(value, "Value can not be null for configurations."); 82 | data.put(key, value); 83 | } 84 | 85 | 86 | @Override 87 | public int size() 88 | { 89 | return data.size(); 90 | } 91 | 92 | @Override 93 | public boolean contains(final String prop) 94 | { 95 | return data.containsKey(prop); 96 | } 97 | 98 | /** 99 | * Clones all the values from the properties. If the value is null, it will be ignored. 100 | * 101 | * @param properties to clone 102 | */ 103 | private void clone(final Properties properties) 104 | { 105 | if (properties.isEmpty()) return; 106 | 107 | synchronized (properties) 108 | { 109 | for (final String key : properties.stringPropertyNames()) 110 | { 111 | final String value = properties.getProperty(key); 112 | if (!Strings.isNullOrEmpty(value)) 113 | { 114 | data.put(key, value); 115 | } 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/configSource/SystemPropertiesConfigSource.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.configSource; 2 | 3 | import java.util.Map; 4 | import java.util.Properties; 5 | 6 | import com.google.common.base.Preconditions; 7 | import com.google.common.collect.Maps; 8 | 9 | /** 10 | * Loads {@link System#getProperties()} as a source. 11 | * 12 | * Implementation note: {@link #set(String, String)} does not write to system properties, but will write to a new map. 13 | * This means that setting values to this source has no effect on system properties or other instances of this class. 14 | */ 15 | public final class SystemPropertiesConfigSource extends AbstractConfigSource 16 | { 17 | private static final String BLANK = ""; 18 | 19 | private final Map data = Maps.newConcurrentMap(); 20 | 21 | @Override 22 | public void intialize(final String asgName, final String region) 23 | { 24 | super.intialize(asgName, region); 25 | 26 | Properties systemProps = System.getProperties(); 27 | 28 | for (final String key : systemProps.stringPropertyNames()) 29 | { 30 | final String value = systemProps.getProperty(key); 31 | if (value != null && !BLANK.equals(value)) 32 | { 33 | data.put(key, value); 34 | } 35 | } 36 | } 37 | 38 | @Override 39 | public int size() 40 | { 41 | return data.size(); 42 | } 43 | 44 | @Override 45 | public String get(final String key) 46 | { 47 | return data.get(key); 48 | } 49 | 50 | @Override 51 | public void set(final String key, final String value) 52 | { 53 | Preconditions.checkNotNull(value, "Value can not be null for configurations."); 54 | data.put(key, value); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/guice/GuiceContext.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.guice; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import com.google.inject.Injector; 7 | 8 | /** 9 | * A holder class around the Governator Guice {@code Injector} 10 | */ 11 | public class GuiceContext { 12 | private static final Logger logger = LoggerFactory.getLogger(GuiceContext.class); 13 | private static final GuiceContext INSTANCE = new GuiceContext(); 14 | private Injector injector; 15 | 16 | private GuiceContext(){} 17 | 18 | /* 19 | * IMPORTANT: must be invoked when the web app starts (@see PriamLifecycleListener.initialize()) 20 | */ 21 | public static void setInjector(Injector val) { 22 | if (INSTANCE.injector == null) { 23 | synchronized(GuiceContext.class) { 24 | if (INSTANCE.injector == null) { 25 | INSTANCE.injector = val; 26 | } 27 | } 28 | } 29 | 30 | } 31 | 32 | public static Injector getInjector() 33 | { 34 | if (INSTANCE.injector == null) { 35 | throw new IllegalStateException("The injector is null. It should have been set when the web app starts (in some listener such as PriamLifecycleListener.initialize()"); 36 | } 37 | 38 | logger.info("The injector provided has id: " + INSTANCE.injector.hashCode()); 39 | 40 | return INSTANCE.injector; 41 | } 42 | } -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/identity/AppsInstance.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.identity; 17 | 18 | import java.io.Serializable; 19 | import java.util.Map; 20 | 21 | public class AppsInstance implements Serializable 22 | { 23 | private static final long serialVersionUID = 5606412386974488659L; 24 | private String hostname; 25 | private int dynomitePort; 26 | private int dynomiteSecurePort; 27 | private int dynomiteSecureStoragePort; 28 | private int peerPort; 29 | private long updatetime; 30 | private boolean outOfService; 31 | 32 | private String app; 33 | private int Id; 34 | private String instanceId; 35 | private String availabilityZone; 36 | private String rack; 37 | private String publicip; 38 | private String location; 39 | private String token; 40 | //Handles Storage objects 41 | private Map volumes; 42 | 43 | public String getApp() 44 | { 45 | return app; 46 | } 47 | 48 | public void setApp(String app) 49 | { 50 | this.app = app; 51 | } 52 | 53 | public int getId() 54 | { 55 | return Id; 56 | } 57 | 58 | public void setId(int id) 59 | { 60 | Id = id; 61 | } 62 | 63 | public String getInstanceId() 64 | { 65 | return instanceId; 66 | } 67 | 68 | public void setInstanceId(String instanceId) 69 | { 70 | this.instanceId = instanceId; 71 | } 72 | 73 | public String getZone() 74 | { 75 | return availabilityZone; 76 | } 77 | 78 | public void setZone(String availabilityZone) 79 | { 80 | this.availabilityZone = availabilityZone; 81 | } 82 | 83 | public String getHostName() 84 | { 85 | return hostname; 86 | } 87 | 88 | public String getHostIP() 89 | { 90 | return publicip; 91 | } 92 | 93 | public void setHost(String hostname, String publicip) 94 | { 95 | this.hostname = hostname; 96 | this.publicip = publicip; 97 | } 98 | 99 | public void setHost(String hostname) 100 | { 101 | this.hostname = hostname; 102 | } 103 | 104 | public void setHostIP(String publicip) 105 | { 106 | this.publicip = publicip; 107 | } 108 | 109 | public String getToken() 110 | { 111 | return token; 112 | } 113 | 114 | public void setToken(String token) 115 | { 116 | this.token = token; 117 | } 118 | 119 | public Map getVolumes() 120 | { 121 | return volumes; 122 | } 123 | 124 | public void setVolumes(Map volumes) 125 | { 126 | this.volumes = volumes; 127 | } 128 | 129 | @Override 130 | public String toString() 131 | { 132 | return String.format("Hostname: %s, InstanceId: %s, APP_NAME: %s, RAC : %s Location %s, Id: %s: Token: %s", getHostName(), getInstanceId(), getApp(), getZone(), getDatacenter(), getId(), 133 | getToken()); 134 | } 135 | 136 | public String getDatacenter() 137 | { 138 | return location; 139 | } 140 | 141 | public void setDatacenter(String dc) 142 | { 143 | this.location = dc; 144 | } 145 | 146 | public long getUpdatetime() 147 | { 148 | return updatetime; 149 | } 150 | 151 | public void setUpdatetime(long updatetime) 152 | { 153 | this.updatetime = updatetime; 154 | } 155 | 156 | public boolean isOutOfService() 157 | { 158 | return outOfService; 159 | } 160 | 161 | public void setOutOfService(boolean outOfService) 162 | { 163 | this.outOfService = outOfService; 164 | } 165 | 166 | public String getRack() 167 | { 168 | return rack; 169 | } 170 | 171 | public void setRack(String rack) 172 | { 173 | this.rack = rack; 174 | } 175 | 176 | public void setDynomitePort(int port) { this.dynomitePort = port; } 177 | 178 | public int getDynomitePort() { return this.dynomitePort; } 179 | 180 | public void setDynomiteSecurePort(int port) { this.dynomiteSecurePort = port; } 181 | 182 | public int getDynomiteSecurePort() { return this.dynomiteSecurePort; } 183 | 184 | public void setDynomiteSecureStoragePort(int port) { this.dynomiteSecureStoragePort = port; } 185 | 186 | public int getDynomiteSecureStoragePort() { return this.dynomiteSecureStoragePort; } 187 | 188 | public void setPeerPort(int port) { this.peerPort = port; } 189 | 190 | public int getPeerPort() { return this.peerPort; } 191 | 192 | } 193 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/identity/IInstanceState.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.identity; 2 | 3 | public interface IInstanceState { 4 | 5 | public boolean isSideCarProcessAlive(); 6 | 7 | public boolean isBootstrapping(); 8 | 9 | public boolean getYmlWritten(); 10 | 11 | public void setYmlWritten(boolean b); 12 | 13 | public void setStorageProxyAlive(boolean isStorageProxyAlive); 14 | 15 | } -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/identity/IMembership.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.identity; 17 | 18 | import java.util.Collection; 19 | import java.util.List; 20 | 21 | /** 22 | * Interface to manage membership meta information such as size of RAC, list of 23 | * nodes in RAC etc. Also perform ACL updates used in multi-regional clusters 24 | */ 25 | public interface IMembership 26 | { 27 | /** 28 | * Get a list of Instances in the current RAC 29 | */ 30 | public List getRacMembership(); 31 | 32 | /** 33 | * @return Get a list of Instances if there is a cross-account RAC 34 | */ 35 | public List getCrossAccountRacMembership(); 36 | 37 | /** 38 | * @return Size of current RAC 39 | */ 40 | public int getRacMembershipSize(); 41 | 42 | /** 43 | * @return Size of the crossaccount RAC 44 | */ 45 | public int getCrossAccountRacMembershipSize(); 46 | 47 | /** 48 | * Add security group ACLs 49 | * 50 | * @param listIPs 51 | * @param from 52 | * @param to 53 | */ 54 | public void addACL(Collection listIPs, int from, int to); 55 | 56 | /** 57 | * Remove security group ACLs 58 | * 59 | * @param listIPs 60 | * @param from 61 | * @param to 62 | */ 63 | public void removeACL(Collection listIPs, int from, int to); 64 | 65 | /** 66 | * List all ACLs 67 | */ 68 | public List listACL(int from, int to); 69 | 70 | /** 71 | * Expand the membership size by 1. 72 | * 73 | * @param count 74 | */ 75 | public void expandRacMembership(int count); 76 | 77 | 78 | } -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/identity/InstanceEnvIdentity.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.identity; 2 | 3 | /* 4 | * A means to determine the environment for the running instance 5 | */ 6 | public interface InstanceEnvIdentity { 7 | /* 8 | * @return true if running instance is in "classic", false otherwise. 9 | */ 10 | public Boolean isClassic(); 11 | /* 12 | * @return true if running instance is in VPC, under your default AWS account, false otherwise. 13 | */ 14 | public Boolean isDefaultVpc(); 15 | /* 16 | * @return true if running instance is in VPC, under a specific AWS account, false otherwise. 17 | */ 18 | public Boolean isNonDefaultVpc(); 19 | 20 | public static enum InstanceEnvironent { 21 | CLASSIC, DEFAULT_VPC, NONDEFAULT_VPC 22 | }; 23 | } 24 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/instance/AwsInstanceDataRetriever.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.instance; 2 | 3 | import com.netflix.nfsidecar.utils.SystemUtils; 4 | 5 | /** 6 | * Calls AWS ec2 metadata to get info on the location of the running instance. 7 | * 8 | */ 9 | public class AwsInstanceDataRetriever implements InstanceDataRetriever { 10 | public String getRac() { 11 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/placement/availability-zone"); 12 | } 13 | 14 | public String getPublicHostname() { 15 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/public-hostname"); 16 | } 17 | 18 | public String getPublicIP() { 19 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/public-ipv4"); 20 | } 21 | 22 | public String getInstanceId() { 23 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/instance-id"); 24 | } 25 | 26 | public String getInstanceType() { 27 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/instance-type"); 28 | } 29 | 30 | @Override 31 | /* 32 | * @return id of the network interface for running instance 33 | */ 34 | public String getMac() { 35 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/network/interfaces/macs/").trim(); 36 | } 37 | 38 | @Override 39 | public String getVpcId() { 40 | throw new UnsupportedOperationException("Not applicable as running instance is in classic environment"); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/instance/InstanceDataRetriever.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.instance; 2 | 3 | public interface InstanceDataRetriever 4 | { 5 | String getRac(); 6 | String getPublicHostname(); 7 | String getPublicIP(); 8 | String getInstanceId(); 9 | String getInstanceType(); 10 | String getMac(); //fetch id of the network interface for running instance 11 | String getVpcId(); //the id of the vpc for running instance 12 | } 13 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/instance/LocalInstanceDataRetriever.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.instance; 2 | 3 | /** 4 | * Looks at local (system) properties for metadata about the running 'instance'. 5 | * Typically, this is used for locally-deployed testing. 6 | * 7 | * @author jason brown 8 | */ 9 | public class LocalInstanceDataRetriever implements InstanceDataRetriever 10 | { 11 | private static final String PREFIX = "florida.localInstance."; 12 | 13 | public String getRac() 14 | { 15 | return System.getProperty(PREFIX + "availabilityZone", ""); 16 | } 17 | 18 | public String getPublicHostname() 19 | { 20 | return System.getProperty(PREFIX + "publicHostname", ""); 21 | } 22 | 23 | public String getPublicIP() 24 | { 25 | return System.getProperty(PREFIX + "publicIp", ""); 26 | } 27 | 28 | public String getInstanceId() 29 | { 30 | return System.getProperty(PREFIX + "instanceId", ""); 31 | } 32 | 33 | public String getInstanceType() 34 | { 35 | return System.getProperty(PREFIX + "instanceType", ""); 36 | } 37 | 38 | public String getMac() { 39 | return System.getProperty(PREFIX + "instanceMac", ""); 40 | } 41 | 42 | @Override 43 | public String getVpcId() { 44 | throw new UnsupportedOperationException("Not applicable as running instance is in classic environment"); 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/instance/VpcInstanceDataRetriever.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.instance; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import com.netflix.nfsidecar.utils.SystemUtils; 7 | 8 | /** 9 | * Calls AWS ec2 metadata to get info on the location of the running instance in 10 | * VPC. Public Hostname will return local-hostname Public IP will return 11 | * local-ipv4 12 | */ 13 | public class VpcInstanceDataRetriever implements InstanceDataRetriever { 14 | private static final Logger logger = LoggerFactory.getLogger(VpcInstanceDataRetriever.class); 15 | 16 | public String getRac() { 17 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/placement/availability-zone"); 18 | } 19 | 20 | public String getPublicHostname() { 21 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/public-hostname"); 22 | } 23 | 24 | public String getPublicIP() { 25 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/public-ipv4"); 26 | } 27 | 28 | public String getInstanceId() { 29 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/instance-id"); 30 | } 31 | 32 | public String getInstanceType() { 33 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/instance-type"); 34 | } 35 | 36 | @Override 37 | /* 38 | * @return id of the network interface for running instance 39 | */ 40 | public String getMac() { 41 | return SystemUtils.getDataFromUrl("http://169.254.169.254/latest/meta-data/network/interfaces/macs/").trim(); 42 | } 43 | 44 | @Override 45 | /* 46 | * @return the id of the vpc account for running instance, null if does not 47 | * exist. 48 | */ 49 | public String getVpcId() { 50 | String nacId = getMac(); 51 | if (nacId == null || nacId.isEmpty()) 52 | return null; 53 | 54 | String vpcId = null; 55 | try { 56 | vpcId = SystemUtils 57 | .getDataFromUrl( 58 | "http://169.254.169.254/latest/meta-data/network/interfaces/macs/" + nacId + "vpc-id") 59 | .trim(); 60 | } catch (Exception e) { 61 | logger.info( 62 | "Vpc id does not exist for running instance, not fatal as running instance maybe not be in vpc. Msg: " 63 | + e.getLocalizedMessage()); 64 | } 65 | 66 | return vpcId; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/resources/SecurityGroupAdmin.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.resources; 2 | 3 | import java.util.Collections; 4 | 5 | import javax.ws.rs.DELETE; 6 | import javax.ws.rs.POST; 7 | import javax.ws.rs.Path; 8 | import javax.ws.rs.Produces; 9 | import javax.ws.rs.QueryParam; 10 | import javax.ws.rs.core.MediaType; 11 | import javax.ws.rs.core.Response; 12 | 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import com.google.inject.Inject; 17 | import com.netflix.nfsidecar.identity.IMembership; 18 | 19 | /** 20 | * This http endpoint allows direct updates (adding/removing) (CIDR) IP addresses and port 21 | * ranges to the security group for this app. 22 | */ 23 | @Path("/v1/secgroup") 24 | @Produces(MediaType.TEXT_PLAIN) 25 | public class SecurityGroupAdmin 26 | { 27 | private static final Logger log = LoggerFactory.getLogger(SecurityGroupAdmin.class); 28 | private static final String CIDR_TAG = "/32"; 29 | private final IMembership membership; 30 | 31 | @Inject 32 | public SecurityGroupAdmin(IMembership membership) 33 | { 34 | this.membership = membership; 35 | } 36 | 37 | @POST 38 | public Response addACL(@QueryParam("ip") String ipAddr, @QueryParam("fromPort") int fromPort, @QueryParam("toPort") int toPort) 39 | { 40 | if(!ipAddr.endsWith(CIDR_TAG)) 41 | ipAddr += CIDR_TAG; 42 | try 43 | { 44 | membership.addACL(Collections.singletonList(ipAddr), fromPort, toPort); 45 | } 46 | catch(Exception e) 47 | { 48 | log.error("Error while trying to add an ACL to a security group", e); 49 | return Response.serverError().build(); 50 | } 51 | return Response.ok().build(); 52 | } 53 | 54 | @DELETE 55 | public Response removeACL(@QueryParam("ip") String ipAddr, @QueryParam("fromPort") int fromPort, @QueryParam("toPort") int toPort) 56 | { 57 | if(!ipAddr.endsWith(CIDR_TAG)) 58 | ipAddr += CIDR_TAG; 59 | try 60 | { 61 | membership.removeACL(Collections.singletonList(ipAddr), fromPort, toPort); 62 | } 63 | catch(Exception e) 64 | { 65 | log.error("Error while trying to remove an ACL to a security group", e); 66 | return Response.serverError().build(); 67 | } 68 | return Response.ok().build(); 69 | } 70 | } 71 | 72 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/resources/env/IEnvVariables.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.resources.env; 2 | 3 | public interface IEnvVariables { 4 | 5 | public String getDynomiteClusterName(); 6 | 7 | public String getRegion(); 8 | 9 | public String getRack(); 10 | } 11 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/resources/env/InstanceEnvVariables.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.resources.env; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import com.google.inject.Inject; 8 | import com.netflix.nfsidecar.config.CommonConfig; 9 | 10 | public class InstanceEnvVariables implements IEnvVariables { 11 | 12 | private static final Logger logger = LoggerFactory.getLogger(InstanceEnvVariables.class); 13 | 14 | CommonConfig config; 15 | 16 | @Inject 17 | public InstanceEnvVariables(CommonConfig config) { 18 | this.config = config; 19 | } 20 | 21 | @Override 22 | public String getDynomiteClusterName() { 23 | String clusterName = System.getenv("NETFLIX_APP"); 24 | /* if (StringUtils.isBlank(clusterName)) { 25 | logger.warn("Cluster name variable not defined. Falling back to FP " + config.getDynomiteClusterName()); 26 | clusterName = config.getDynomiteClusterName(); 27 | } 28 | */ 29 | return clusterName; 30 | } 31 | 32 | @Override 33 | public String getRegion() { 34 | String region = System.getenv("EC2_REGION"); 35 | if (StringUtils.isBlank(region)) { 36 | logger.warn("Region environment variable not defined. Falling back to " + config.getRegion()); 37 | region = config.getRegion(); 38 | } 39 | return region; 40 | } 41 | 42 | @Override 43 | public String getRack() { 44 | String rack = System.getenv("NETFLIX_AUTO_SCALE_GROUP"); 45 | if (StringUtils.isBlank(rack)) { 46 | logger.error("Rack environment variable not defined. Falling back to " + config.getRack()); 47 | rack = config.getRack(); 48 | } 49 | return rack; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/BlockingSubmitThreadPoolExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.util.concurrent.*; 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | 24 | /** 25 | * {@link ThreadPoolExecutor} that will block in the {@code submit()} method 26 | * until the task can be successfully added to the queue. 27 | */ 28 | public class BlockingSubmitThreadPoolExecutor extends ThreadPoolExecutor 29 | { 30 | private static final long DEFAULT_SLEEP = 100; 31 | private static final long DEFAULT_KEEP_ALIVE = 100; 32 | private static final Logger logger = LoggerFactory.getLogger(BlockingSubmitThreadPoolExecutor.class); 33 | private BlockingQueue queue; 34 | private long giveupTime; 35 | private AtomicInteger active; 36 | 37 | public BlockingSubmitThreadPoolExecutor(int maximumPoolSize, BlockingQueue workQueue, long timeoutAdding) 38 | { 39 | super(maximumPoolSize, maximumPoolSize, DEFAULT_KEEP_ALIVE, TimeUnit.SECONDS, workQueue); 40 | this.queue = workQueue; 41 | this.giveupTime = timeoutAdding; 42 | this.active = new AtomicInteger(0); 43 | } 44 | 45 | /** 46 | * This is a thread safe way to avoid rejection exception... this is 47 | * implemented because we might want to hold the incoming requests till 48 | * there is a free thread. 49 | */ 50 | @Override 51 | public Future submit(Callable task) 52 | { 53 | synchronized (this) 54 | { 55 | active.incrementAndGet(); 56 | long timeout = 0; 57 | while (queue.remainingCapacity() == 0) 58 | { 59 | try 60 | { 61 | if (timeout <= giveupTime) 62 | { 63 | Thread.sleep(DEFAULT_SLEEP); 64 | timeout += DEFAULT_SLEEP; 65 | } 66 | else 67 | { 68 | throw new RuntimeException("Timed out because TPE is too busy..."); 69 | } 70 | } 71 | catch (InterruptedException e) 72 | { 73 | throw new RuntimeException(e); 74 | } 75 | } 76 | return super.submit(task); 77 | } 78 | } 79 | 80 | @Override 81 | protected void afterExecute(Runnable r, Throwable t) 82 | { 83 | super.afterExecute(r, t); 84 | active.decrementAndGet(); 85 | } 86 | 87 | /** 88 | * blocking call to test if the threads are done or not. 89 | */ 90 | public void sleepTillEmpty() 91 | { 92 | long timeout = 0; 93 | 94 | while (!queue.isEmpty() || (active.get() > 0)) 95 | { 96 | try 97 | { 98 | if (timeout <= giveupTime) 99 | { 100 | Thread.sleep(DEFAULT_SLEEP); 101 | timeout += DEFAULT_SLEEP; 102 | logger.debug("After Sleeping for empty: {}, Count: {}", +queue.size(), active.get()); 103 | } 104 | else 105 | { 106 | throw new RuntimeException("Timed out because TPE is too busy..."); 107 | } 108 | } 109 | catch (InterruptedException e) 110 | { 111 | throw new RuntimeException(e); 112 | } 113 | } 114 | 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/CronTimer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import java.text.ParseException; 19 | 20 | import org.quartz.CronTrigger; 21 | import org.quartz.Scheduler; 22 | import org.quartz.Trigger; 23 | 24 | /** 25 | * Runs jobs at the specified absolute time and frequency 26 | */ 27 | public class CronTimer implements TaskTimer 28 | { 29 | private String cronExpression; 30 | 31 | public enum DayOfWeek 32 | { 33 | SUN, MON, TUE, WED, THU, FRI, SAT 34 | } 35 | 36 | /** 37 | * Hourly cron. 38 | */ 39 | public CronTimer(int minute, int sec) 40 | { 41 | cronExpression = sec + " " + minute + " * * * ?"; 42 | } 43 | 44 | /** 45 | * Daily Cron 46 | */ 47 | public CronTimer(int hour, int minute, int sec) 48 | { 49 | cronExpression = sec + " " + minute + " " + hour + " * * ?"; 50 | } 51 | 52 | /** 53 | * Weekly cron jobs 54 | */ 55 | public CronTimer(DayOfWeek dayofweek, int hour, int minute, int sec) 56 | { 57 | cronExpression = sec + " " + minute + " " + hour + " * * " + dayofweek; 58 | } 59 | 60 | /** 61 | * Cron Expression. 62 | */ 63 | public CronTimer(String expression) 64 | { 65 | this.cronExpression = expression; 66 | } 67 | 68 | public Trigger getTrigger() throws ParseException 69 | { 70 | return new CronTrigger("CronTrigger", Scheduler.DEFAULT_GROUP, cronExpression); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/ExecutionException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | public class ExecutionException extends Exception 19 | { 20 | private static final long serialVersionUID = 1L; 21 | 22 | public ExecutionException(String msg, Throwable th) 23 | { 24 | super(msg, th); 25 | } 26 | 27 | public ExecutionException(String msg) 28 | { 29 | super(msg); 30 | } 31 | 32 | public ExecutionException(Exception ex) 33 | { 34 | super(ex); 35 | } 36 | 37 | public ExecutionException(Throwable th) 38 | { 39 | super(th); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/GuiceJobFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import org.quartz.Job; 19 | import org.quartz.JobDetail; 20 | import org.quartz.SchedulerException; 21 | import org.quartz.spi.JobFactory; 22 | import org.quartz.spi.TriggerFiredBundle; 23 | 24 | import com.google.inject.Inject; 25 | import com.google.inject.Injector; 26 | 27 | public class GuiceJobFactory implements JobFactory 28 | { 29 | public final Injector guice; 30 | 31 | @Inject 32 | public GuiceJobFactory(Injector guice) 33 | { 34 | this.guice = guice; 35 | } 36 | 37 | @Override 38 | public Job newJob(TriggerFiredBundle bundle) throws SchedulerException 39 | { 40 | JobDetail jobDetail = bundle.getJobDetail(); 41 | Class jobClass = jobDetail.getJobClass(); 42 | Job job = (Job) guice.getInstance(jobClass); 43 | guice.injectMembers(job); 44 | return job; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/NamedThreadPoolExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import java.util.concurrent.BlockingQueue; 19 | import java.util.concurrent.LinkedBlockingQueue; 20 | import java.util.concurrent.RejectedExecutionException; 21 | import java.util.concurrent.RejectedExecutionHandler; 22 | import java.util.concurrent.ThreadPoolExecutor; 23 | import java.util.concurrent.TimeUnit; 24 | 25 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 26 | 27 | public class NamedThreadPoolExecutor extends ThreadPoolExecutor 28 | { 29 | public NamedThreadPoolExecutor(int poolSize, String poolName) 30 | { 31 | this(poolSize, poolName, new LinkedBlockingQueue()); 32 | } 33 | 34 | public NamedThreadPoolExecutor(int poolSize, String poolName, BlockingQueue queue) 35 | { 36 | super(poolSize, poolSize, 1000, TimeUnit.MILLISECONDS, queue, 37 | new ThreadFactoryBuilder().setDaemon(true).setNameFormat(poolName + "-%d").build(), 38 | new LocalRejectedExecutionHandler(queue)); 39 | } 40 | 41 | private static class LocalRejectedExecutionHandler implements RejectedExecutionHandler 42 | { 43 | private final BlockingQueue queue; 44 | 45 | LocalRejectedExecutionHandler(BlockingQueue queue) 46 | { 47 | this.queue = queue; 48 | } 49 | 50 | public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) 51 | { 52 | while (true) 53 | { 54 | if (executor.isShutdown()) 55 | throw new RejectedExecutionException("ThreadPoolExecutor has shut down"); 56 | 57 | try 58 | { 59 | if (queue.offer(task, 1000, TimeUnit.MILLISECONDS)) 60 | break; 61 | } 62 | catch (InterruptedException e) 63 | { 64 | //NOP 65 | } 66 | } 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/SimpleTimer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import java.text.ParseException; 19 | import java.util.Date; 20 | 21 | import org.quartz.Scheduler; 22 | import org.quartz.SimpleTrigger; 23 | import org.quartz.Trigger; 24 | 25 | /** 26 | * SimpleTimer allows jobs to run starting from specified time occurring at 27 | * regular frequency's. Frequency of the execution timestamp since epoch. 28 | */ 29 | public class SimpleTimer implements TaskTimer 30 | { 31 | private SimpleTrigger trigger; 32 | 33 | public SimpleTimer(String name, long interval) 34 | { 35 | this.trigger = new SimpleTrigger(name, SimpleTrigger.REPEAT_INDEFINITELY, interval); 36 | } 37 | 38 | /** 39 | * Run once at given time... 40 | */ 41 | public SimpleTimer(String name, String group, long startTime) 42 | { 43 | this.trigger = new SimpleTrigger(name, group, new Date(startTime)); 44 | } 45 | 46 | /** 47 | * Run immediately and dont do that again. 48 | */ 49 | public SimpleTimer(String name) 50 | { 51 | this.trigger = new SimpleTrigger(name, Scheduler.DEFAULT_GROUP); 52 | } 53 | 54 | public Trigger getTrigger() throws ParseException 55 | { 56 | trigger.setMisfireInstruction(SimpleTrigger.MISFIRE_INSTRUCTION_FIRE_NOW); 57 | return trigger; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/Task.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import com.google.common.base.Throwables; 19 | 20 | import org.quartz.Job; 21 | import org.quartz.JobExecutionContext; 22 | import org.quartz.JobExecutionException; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import javax.management.MBeanServer; 27 | import javax.management.ObjectName; 28 | import java.lang.management.ManagementFactory; 29 | import java.util.concurrent.atomic.AtomicInteger; 30 | 31 | /** 32 | * Task class that should be implemented by all cron tasks. Jobconf will contain 33 | * any instance specific data 34 | * 35 | * NOTE: Constructor must not throw any exception. This will cause Quartz to set the job to failure 36 | */ 37 | public abstract class Task implements Job, TaskMBean 38 | { 39 | public STATE status = STATE.DONE; 40 | 41 | public static enum STATE 42 | { 43 | ERROR, RUNNING, DONE 44 | } 45 | 46 | private static final Logger logger = LoggerFactory.getLogger(Task.class); 47 | private final AtomicInteger errors = new AtomicInteger(); 48 | private final AtomicInteger executions = new AtomicInteger(); 49 | 50 | protected Task() 51 | { 52 | this(ManagementFactory.getPlatformMBeanServer()); 53 | } 54 | 55 | protected Task(MBeanServer mBeanServer) { 56 | // TODO: don't do mbean registration here 57 | String mbeanName = "com.netflix.florida.scheduler:type=" + this.getClass().getName(); 58 | try 59 | { 60 | mBeanServer.registerMBean(this, new ObjectName(mbeanName)); 61 | initialize(); 62 | } 63 | catch (Exception e) 64 | { 65 | throw Throwables.propagate(e); 66 | } 67 | } 68 | 69 | 70 | /** 71 | * This method has to be implemented and cannot thow any exception. 72 | */ 73 | public void initialize() throws ExecutionException 74 | { 75 | // nothing to initialize 76 | } 77 | 78 | public abstract void execute() throws Exception; 79 | 80 | /** 81 | * Main method to execute a task 82 | */ 83 | public void execute(JobExecutionContext context) throws JobExecutionException 84 | { 85 | executions.incrementAndGet(); 86 | try 87 | { 88 | if (status == STATE.RUNNING) 89 | return; 90 | status = STATE.RUNNING; 91 | execute(); 92 | 93 | } 94 | catch (Exception e) 95 | { 96 | status = STATE.ERROR; 97 | logger.error("Couldn't execute the task because of: " + e.getMessage(), e); 98 | errors.incrementAndGet(); 99 | } 100 | catch (Throwable e) 101 | { 102 | status = STATE.ERROR; 103 | logger.error("Couldnt execute the task because of: " + e.getMessage(), e); 104 | errors.incrementAndGet(); 105 | } 106 | if (status != STATE.ERROR) 107 | status = STATE.DONE; 108 | } 109 | 110 | public STATE state() 111 | { 112 | return status; 113 | } 114 | 115 | public int getErrorCount() 116 | { 117 | return errors.get(); 118 | } 119 | 120 | public int getExecutionCount() 121 | { 122 | return executions.get(); 123 | } 124 | 125 | public abstract String getName(); 126 | 127 | } 128 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/TaskMBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | /** 19 | * MBean to monitor Task executions. 20 | * 21 | */ 22 | public interface TaskMBean 23 | { 24 | public int getErrorCount(); 25 | public int getExecutionCount(); 26 | public String getName(); 27 | } 28 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/TaskScheduler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import java.text.ParseException; 19 | 20 | import org.quartz.JobDetail; 21 | import org.quartz.Scheduler; 22 | import org.quartz.SchedulerException; 23 | import org.quartz.SchedulerFactory; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import com.google.inject.Inject; 28 | import com.google.inject.Singleton; 29 | import com.netflix.nfsidecar.utils.Sleeper; 30 | 31 | /** 32 | * Scheduling class to schedule Florida tasks. Uses Quartz scheduler 33 | */ 34 | @Singleton 35 | public class TaskScheduler 36 | { 37 | private static final Logger logger = LoggerFactory.getLogger(TaskScheduler.class); 38 | private final Scheduler scheduler; 39 | private final GuiceJobFactory jobFactory; 40 | private final Sleeper sleeper; 41 | 42 | @Inject 43 | public TaskScheduler(SchedulerFactory factory, GuiceJobFactory jobFactory, Sleeper sleeper) 44 | { 45 | try 46 | { 47 | this.scheduler = factory.getScheduler(); 48 | this.scheduler.setJobFactory(jobFactory); 49 | this.jobFactory = jobFactory; 50 | } 51 | catch (SchedulerException e) 52 | { 53 | throw new RuntimeException(e); 54 | } 55 | this.sleeper = sleeper; 56 | } 57 | 58 | /** 59 | * Add a task to the scheduler 60 | */ 61 | public void addTask(String name, Class taskclass, TaskTimer timer) throws SchedulerException, ParseException 62 | { 63 | assert timer != null : "Cannot add scheduler task " + name + " as no timer is set"; 64 | JobDetail job = new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); 65 | scheduler.scheduleJob(job, timer.getTrigger()); 66 | } 67 | 68 | /** 69 | * Add a delayed task to the scheduler 70 | */ 71 | public void addTaskWithDelay(final String name, Class taskclass, final TaskTimer timer, final int delayInSeconds) throws SchedulerException, ParseException 72 | { 73 | assert timer != null : "Cannot add scheduler task " + name + " as no timer is set"; 74 | final JobDetail job = new JobDetail(name, Scheduler.DEFAULT_GROUP, taskclass); 75 | 76 | new Thread(new Runnable(){ 77 | public void run() 78 | { 79 | try 80 | { 81 | sleeper.sleepQuietly(delayInSeconds * 1000L); 82 | scheduler.scheduleJob(job, timer.getTrigger()); 83 | } 84 | catch (SchedulerException e) 85 | { 86 | logger.warn("problem occurred while scheduling a job with name " + name, e); 87 | } 88 | catch (ParseException e) 89 | { 90 | logger.warn("problem occurred while parsing a job with name " + name, e); 91 | } 92 | } 93 | }).start(); 94 | } 95 | 96 | public void runTaskNow(Class taskclass) throws Exception 97 | { 98 | jobFactory.guice.getInstance(taskclass).execute(null); 99 | } 100 | 101 | public void deleteTask(String name) throws SchedulerException, ParseException 102 | { 103 | scheduler.deleteJob(name, Scheduler.DEFAULT_GROUP); 104 | } 105 | 106 | public final Scheduler getScheduler() 107 | { 108 | return scheduler; 109 | } 110 | 111 | public void shutdown() 112 | { 113 | try 114 | { 115 | scheduler.shutdown(); 116 | } 117 | catch (SchedulerException e) 118 | { 119 | throw new RuntimeException(e); 120 | } 121 | } 122 | 123 | public void start() 124 | { 125 | try 126 | { 127 | scheduler.start(); 128 | } 129 | catch (SchedulerException ex) 130 | { 131 | throw new RuntimeException(ex); 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/scheduler/TaskTimer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.scheduler; 17 | 18 | import java.text.ParseException; 19 | 20 | import org.quartz.Trigger; 21 | 22 | /** 23 | * Interface to represent time/interval 24 | */ 25 | public interface TaskTimer 26 | { 27 | public Trigger getTrigger() throws ParseException; 28 | } 29 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/supplier/EurekaHostSupplier.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.supplier; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import com.google.common.base.Function; 11 | import com.google.common.base.Predicate; 12 | import com.google.common.base.Supplier; 13 | import com.google.common.collect.Collections2; 14 | import com.google.common.collect.Lists; 15 | import com.google.inject.Inject; 16 | import com.google.inject.Singleton; 17 | import com.netflix.appinfo.AmazonInfo; 18 | import com.netflix.appinfo.AmazonInfo.MetaDataKey; 19 | import com.netflix.appinfo.InstanceInfo; 20 | import com.netflix.astyanax.connectionpool.Host; 21 | import com.netflix.discovery.DiscoveryClient; 22 | import com.netflix.discovery.shared.Application; 23 | 24 | /** 25 | * Use Eureka to provide a list of Cassandra hosts that contain the complete 26 | * Dynomite topology. 27 | * 28 | * EurekaHostsSupplier provides a {@code Supplier>} via the 29 | * {DiscoveryManager} (i.e. the Eureka client). 30 | * 31 | * Note that the class needs the Eureka application name to discover all 32 | * instances for that application. 33 | */ 34 | @Singleton 35 | public class EurekaHostSupplier implements HostSupplier { 36 | 37 | private static final Logger LOG = LoggerFactory.getLogger(EurekaHostSupplier.class); 38 | 39 | // Eureka client 40 | private final DiscoveryClient discoveryClient; 41 | 42 | @Inject 43 | public EurekaHostSupplier(DiscoveryClient discoveryClient) { 44 | this.discoveryClient = discoveryClient; 45 | } 46 | 47 | /** 48 | * Get a list of Cassandra hosts that contain the complete Dynomite 49 | * topology. 50 | * 51 | * @param clusterName 52 | * name of the Dynomite cluster 53 | * @return a Supplier that returns a list of Cassandra hosts 54 | */ 55 | @Override 56 | public Supplier> getSupplier(final String clusterName) { 57 | return new Supplier>() { 58 | 59 | @Override 60 | public List get() { 61 | 62 | if (discoveryClient == null) { 63 | LOG.error("Eureka DiscoveryClient cannot be null"); 64 | throw new RuntimeException("EurekaHostsSupplier needs a non-null DiscoveryClient"); 65 | } 66 | 67 | LOG.debug("Fetching instance list for app: " + clusterName); 68 | 69 | Application app = discoveryClient.getApplication(clusterName.toUpperCase()); 70 | List hosts = new ArrayList(); 71 | 72 | if (app == null) { 73 | LOG.warn("Cluster '{}' not found in Eureka", clusterName); 74 | return hosts; 75 | } 76 | 77 | List ins = app.getInstances(); 78 | 79 | if (ins == null || ins.isEmpty()) { 80 | LOG.warn("Cluster '{}' found in Eureka but has no instances", clusterName); 81 | return hosts; 82 | } 83 | 84 | hosts = Lists 85 | .newArrayList(Collections2.transform(Collections2.filter(ins, new Predicate() { 86 | @Override 87 | public boolean apply(InstanceInfo input) { 88 | return input.getStatus() == InstanceInfo.InstanceStatus.UP; 89 | } 90 | }), new Function() { 91 | @Override 92 | public Host apply(InstanceInfo info) { 93 | String[] parts = StringUtils.split(StringUtils.split(info.getHostName(), ".")[0], '-'); 94 | 95 | Host host = new Host(info.getHostName(), info.getPort()) 96 | .addAlternateIpAddress(StringUtils 97 | .join(new String[] { parts[1], parts[2], parts[3], parts[4] }, ".")) 98 | .addAlternateIpAddress(info.getIPAddr()).setId(info.getId()); 99 | 100 | try { 101 | if (info.getDataCenterInfo() instanceof AmazonInfo) { 102 | AmazonInfo amazonInfo = (AmazonInfo) info.getDataCenterInfo(); 103 | host.setRack(amazonInfo.get(MetaDataKey.availabilityZone)); 104 | } 105 | } catch (Throwable t) { 106 | LOG.error("Error getting rack for host " + host.getName(), t); 107 | } 108 | 109 | return host; 110 | } 111 | })); 112 | 113 | LOG.debug("Found hosts in Eureka. Num hosts: " + hosts.size()); 114 | 115 | return hosts; 116 | } 117 | }; 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/supplier/HostSupplier.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.supplier; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.base.Supplier; 6 | import com.netflix.astyanax.connectionpool.Host; 7 | 8 | public interface HostSupplier { 9 | public Supplier> getSupplier(String clusterName); 10 | } -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/supplier/LocalHostSupplier.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.supplier; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import org.apache.commons.lang.StringUtils; 8 | 9 | import com.google.common.base.Supplier; 10 | import com.google.inject.Inject; 11 | import com.netflix.astyanax.connectionpool.Host; 12 | import com.netflix.nfsidecar.config.CassCommonConfig; 13 | 14 | /** 15 | * Use the {@code DM_CASSANDRA_CLUSTER_SEEDS} environment variable to provide a 16 | * list of Cassandra hosts that contain the complete Dynomite topology. 17 | */ 18 | public class LocalHostSupplier implements HostSupplier { 19 | 20 | private static final String errMsg = "DM_CASSANDRA_CLUSTER_SEEDS cannot be empty. It must contain one or more Cassandra hosts."; 21 | private CassCommonConfig config; 22 | 23 | @Inject 24 | public LocalHostSupplier(CassCommonConfig config) { 25 | this.config = config; 26 | } 27 | 28 | @Override 29 | public Supplier> getSupplier(String clusterName) { 30 | final List hosts = new ArrayList(); 31 | 32 | String bootCluster = config.getCassandraClusterName(); 33 | 34 | if (bootCluster.equals(clusterName)) { 35 | 36 | String seeds = System.getenv("DM_CASSANDRA_CLUSTER_SEEDS"); 37 | 38 | if (seeds == null || "".equals(seeds)) 39 | throw new RuntimeException(errMsg); 40 | 41 | List cassHostnames = new ArrayList(Arrays.asList(StringUtils.split(seeds, ","))); 42 | 43 | if (cassHostnames.size() == 0) 44 | throw new RuntimeException(errMsg); 45 | 46 | for (String cassHost : cassHostnames) { 47 | hosts.add(new Host(cassHost, 9160)); 48 | } 49 | 50 | } else { 51 | hosts.add(new Host("127.0.0.1", 9160).setRack("localdc")); 52 | } 53 | 54 | return new Supplier>() { 55 | @Override 56 | public List get() { 57 | return hosts; 58 | } 59 | }; 60 | 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/tokensdb/CassandraInstanceFactory.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.tokensdb; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.Comparator; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import com.google.inject.Inject; 14 | import com.google.inject.Singleton; 15 | import com.netflix.nfsidecar.config.CommonConfig; 16 | import com.netflix.nfsidecar.identity.AppsInstance; 17 | import com.netflix.nfsidecar.resources.env.IEnvVariables; 18 | 19 | /** 20 | * Factory to use Cassandra for managing instance data 21 | */ 22 | 23 | @Singleton 24 | public class CassandraInstanceFactory implements IAppsInstanceFactory 25 | { 26 | private static final Logger logger = LoggerFactory.getLogger(CassandraInstanceFactory.class); 27 | 28 | CommonConfig config; 29 | InstanceDataDAOCassandra dao; 30 | IEnvVariables envVariables; 31 | 32 | @Inject 33 | public CassandraInstanceFactory(CommonConfig config, InstanceDataDAOCassandra dao, IEnvVariables envVariables) { 34 | this.config = config; 35 | this.dao = dao; 36 | this.envVariables = envVariables; 37 | } 38 | 39 | public List getAllIds(String appName) 40 | { 41 | List return_ = new ArrayList(); 42 | for (AppsInstance instance : dao.getAllInstances(appName)) { 43 | return_.add(instance); 44 | } 45 | 46 | sort(return_); 47 | return return_; 48 | } 49 | 50 | public List getLocalDCIds(String appName, String region) 51 | { 52 | List return_ = new ArrayList(); 53 | for (AppsInstance instance : dao.getLocalDCInstances(appName, region)) { 54 | return_.add(instance); 55 | } 56 | 57 | sort(return_); 58 | return return_; 59 | } 60 | 61 | public void sort(List return_) 62 | { 63 | Comparator comparator = new Comparator() 64 | { 65 | 66 | @Override 67 | public int compare(AppsInstance o1, AppsInstance o2) 68 | { 69 | Integer c1 = o1.getId(); 70 | Integer c2 = o2.getId(); 71 | return c1.compareTo(c2); 72 | } 73 | }; 74 | Collections.sort(return_, comparator); 75 | } 76 | 77 | public AppsInstance create(String app, int id, String instanceID, String hostname, int dynomitePort, int dynomiteSecurePort, int dynomiteSecureStoragePort, int peerPort, String ip, String zone, Map volumes, String payload, String rack) 78 | { 79 | try { 80 | Map v = (volumes == null) ? new HashMap() : volumes; 81 | AppsInstance ins = new AppsInstance(); 82 | ins.setApp(app); 83 | ins.setZone(zone); 84 | ins.setRack(rack); 85 | ins.setHost(hostname); 86 | ins.setDynomitePort(dynomitePort); 87 | ins.setDynomiteSecurePort(dynomiteSecurePort); 88 | ins.setDynomiteSecureStoragePort(dynomiteSecureStoragePort); 89 | ins.setPeerPort(peerPort); 90 | ins.setHostIP(ip); 91 | ins.setId(id); 92 | ins.setInstanceId(instanceID); 93 | ins.setDatacenter(envVariables.getRegion()); 94 | ins.setToken(payload); 95 | ins.setVolumes(v); 96 | 97 | // remove old data node which are dead. 98 | //if (app.endsWith("-dead")) { 99 | // AppsInstance oldData = dao.getInstance(app, ins.getRack(), id); 100 | // clean up a very old data... 101 | //if (null != oldData) 102 | // dao.deleteInstanceEntry(oldData); 103 | //} 104 | dao.createInstanceEntry(ins); 105 | return ins; 106 | } 107 | catch (Exception e) { 108 | logger.error(e.getMessage()); 109 | throw new RuntimeException(e); 110 | } 111 | } 112 | 113 | public void delete(AppsInstance inst) 114 | { 115 | try { 116 | dao.deleteInstanceEntry(inst); 117 | } 118 | catch (Exception e) { 119 | throw new RuntimeException(e); 120 | } 121 | } 122 | 123 | public void update(AppsInstance inst) 124 | { 125 | try { 126 | dao.createInstanceEntry(inst); 127 | } 128 | catch (Exception e) { 129 | throw new RuntimeException(e); 130 | } 131 | } 132 | 133 | @Override 134 | public void attachVolumes(AppsInstance instance, String mountPath, String device) 135 | { 136 | throw new UnsupportedOperationException("Volumes not supported"); 137 | } 138 | 139 | @Override 140 | public AppsInstance getInstance(String appName, String dc, int id) 141 | { 142 | return dao.getInstance(appName, dc, id); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/tokensdb/IAppsInstanceFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.tokensdb; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | import com.netflix.nfsidecar.identity.AppsInstance; 22 | 23 | /** 24 | * Interface for managing Dynomite instance data. Provides functionality 25 | * to register, update, delete or list instances from the registry 26 | */ 27 | 28 | public interface IAppsInstanceFactory 29 | { 30 | /** 31 | * Return a list of all Dynomite server nodes registered. 32 | * @param appName the cluster name 33 | * @return a list of all nodes in {@code appName} 34 | */ 35 | public List getAllIds(String appName); 36 | 37 | 38 | /** 39 | * Return a list of Local Dynomite server nodes registered. 40 | * @param appName the cluster name 41 | * @param region the the region of the node 42 | * @return a list of nodes in {@code appName} and same Racks 43 | */ 44 | public List getLocalDCIds(String appName, String region); 45 | 46 | /** 47 | * Return the Dynomite server node with the given {@code id}. 48 | * @param appName the cluster name 49 | * @param id the node id 50 | * @return the node with the given {@code id}, or {@code null} if none found 51 | */ 52 | public AppsInstance getInstance(String appName, String dc, int id); 53 | 54 | /** 55 | * Create/Register an instance of the server with its info. 56 | * @param app 57 | * @param id 58 | * @param instanceID 59 | * @param hostname 60 | * @param ip 61 | * @param rac 62 | * @param volumes 63 | * @param token 64 | * @return the new node 65 | */ 66 | public AppsInstance create(String app, int id, String instanceID, String hostname, int dynomitePort, int dynomiteSecurePort, int dynomiteSecureStoragePort, int peerPort, String ip, String rac, 67 | Map volumes, String token, String datacenter); 68 | 69 | /** 70 | * Delete the server node from the registry 71 | * @param inst the node to delete 72 | */ 73 | public void delete(AppsInstance inst); 74 | 75 | /** 76 | * Update the details of the server node in registry 77 | * @param inst the node to update 78 | */ 79 | public void update(AppsInstance inst); 80 | 81 | /** 82 | * Sort the list by instance ID 83 | * @param return_ the list of nodes to sort 84 | */ 85 | public void sort(List return_); 86 | 87 | /** 88 | * Attach volumes if required 89 | * @param instance 90 | * @param mountPath 91 | * @param device 92 | */ 93 | public void attachVolumes(AppsInstance instance, String mountPath, String device); 94 | 95 | 96 | 97 | } -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/BoundedExponentialRetryCallable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import org.apache.commons.lang.exception.ExceptionUtils; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.util.concurrent.CancellationException; 23 | 24 | public abstract class BoundedExponentialRetryCallable extends RetryableCallable 25 | { 26 | public final static long MAX_SLEEP = 10000; 27 | public final static long MIN_SLEEP = 1000; 28 | public final static int MAX_RETRIES = 10; 29 | 30 | private static final Logger logger = LoggerFactory.getLogger(BoundedExponentialRetryCallable.class); 31 | private long max; 32 | private long min; 33 | private int maxRetries; 34 | private final ThreadSleeper sleeper = new ThreadSleeper(); 35 | 36 | public BoundedExponentialRetryCallable() 37 | { 38 | this.max = MAX_SLEEP; 39 | this.min = MIN_SLEEP; 40 | this.maxRetries = MAX_RETRIES; 41 | } 42 | 43 | public BoundedExponentialRetryCallable(long minSleep, long maxSleep, int maxNumRetries) 44 | { 45 | this.max = maxSleep; 46 | this.min = minSleep; 47 | this.maxRetries = maxNumRetries; 48 | } 49 | 50 | public T call() throws Exception { 51 | long delay = min;// ms 52 | int retry = 0; 53 | 54 | while (true) { 55 | try { 56 | return retriableCall(); 57 | } catch (CancellationException e) { 58 | throw e; 59 | } catch (Exception e) { 60 | retry++; 61 | 62 | if (delay < max && retry <= maxRetries) { 63 | delay *= 2; 64 | logger.error(String.format("Retry #%d for: %s", retry, e.getMessage())); 65 | sleeper.sleep(delay); 66 | } else if (delay >= max && retry <= maxRetries) { 67 | logger.error(String.format("Retry #%d for: %s", retry, ExceptionUtils.getFullStackTrace(e))); 68 | sleeper.sleep(max); 69 | } else { 70 | logger.info("Exception --> " + ExceptionUtils.getFullStackTrace(e)); 71 | throw e; 72 | } 73 | } finally { 74 | forEachExecution(); 75 | } 76 | } 77 | } 78 | 79 | public void setMax(long max) { 80 | this.max = max; 81 | } 82 | 83 | public void setMin(long min) { 84 | this.min = min; 85 | } 86 | 87 | public void setMaxRetries(int maxRetries) { 88 | this.maxRetries = maxRetries; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/ExponentialRetryCallable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import java.util.concurrent.CancellationException; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | public abstract class ExponentialRetryCallable extends RetryableCallable 24 | { 25 | public final static long MAX_SLEEP = 240000; 26 | public final static long MIN_SLEEP = 200; 27 | 28 | private static final Logger logger = LoggerFactory.getLogger(RetryableCallable.class); 29 | private long max; 30 | private long min; 31 | 32 | public ExponentialRetryCallable() 33 | { 34 | this.max = MAX_SLEEP; 35 | this.min = MIN_SLEEP; 36 | } 37 | 38 | public ExponentialRetryCallable(long minSleep, long maxSleep) 39 | { 40 | this.max = maxSleep; 41 | this.min = minSleep; 42 | } 43 | 44 | public T call() throws Exception 45 | { 46 | long delay = min;// ms 47 | while (true) 48 | { 49 | try 50 | { 51 | return retriableCall(); 52 | } 53 | catch (CancellationException e) 54 | { 55 | throw e; 56 | } 57 | catch (Exception e) 58 | { 59 | delay *= 2; 60 | if (delay > max) 61 | { 62 | throw e; 63 | } 64 | logger.error(e.getMessage()); 65 | Thread.sleep(delay); 66 | } 67 | finally 68 | { 69 | forEachExecution(); 70 | } 71 | } 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/FifoQueue.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import java.util.Comparator; 19 | import java.util.TreeSet; 20 | 21 | public class FifoQueue> extends TreeSet 22 | { 23 | private static final long serialVersionUID = -7388604551920505669L; 24 | private int capacity; 25 | 26 | public FifoQueue(int capacity) 27 | { 28 | super(new Comparator() 29 | { 30 | @Override 31 | public int compare(E o1, E o2) 32 | { 33 | return o1.compareTo(o2); 34 | } 35 | }); 36 | this.capacity = capacity; 37 | } 38 | 39 | public FifoQueue(int capacity, Comparator comparator) 40 | { 41 | super(comparator); 42 | this.capacity = capacity; 43 | } 44 | 45 | public synchronized void adjustAndAdd(E e) 46 | { 47 | add(e); 48 | if (capacity < size()) 49 | pollFirst(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/ITokenManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import com.google.inject.ImplementedBy; 19 | 20 | import java.math.BigInteger; 21 | import java.util.List; 22 | 23 | @ImplementedBy(TokenManager.class) 24 | public interface ITokenManager 25 | { 26 | String createToken(int mySlot, int racCount, int racSize, String region); 27 | 28 | String createToken(int mySlot, int totalCount, String region); 29 | 30 | BigInteger findClosestToken(BigInteger tokenToSearch, List tokenList); 31 | 32 | int regionOffset(String region); 33 | } 34 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/NFException.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.utils; 2 | 3 | 4 | public class NFException { 5 | 6 | private final String cfKey; 7 | private final String pathName; 8 | private final String stacktrace; 9 | 10 | public NFException(String cfKey,String pathName,String stacktrace) 11 | { 12 | this.cfKey = cfKey; 13 | this.pathName = pathName; 14 | this.stacktrace = stacktrace; 15 | } 16 | 17 | public String getCfKey() { 18 | return cfKey; 19 | } 20 | 21 | public String getPathName() { 22 | return pathName; 23 | } 24 | 25 | public String getStacktrace() { 26 | return stacktrace; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/ProcessTuner.java: -------------------------------------------------------------------------------- 1 | package com.netflix.nfsidecar.utils; 2 | 3 | import java.io.IOException; 4 | 5 | public interface ProcessTuner 6 | { 7 | void writeAllProperties(String yamlLocation) throws Exception; 8 | 9 | } 10 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/RetryableCallable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import org.apache.commons.lang.exception.ExceptionUtils; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.util.concurrent.Callable; 23 | import java.util.concurrent.CancellationException; 24 | 25 | public abstract class RetryableCallable implements Callable 26 | { 27 | private static final Logger logger = LoggerFactory.getLogger(RetryableCallable.class); 28 | public static final int DEFAULT_NUMBER_OF_RETRIES = 15; 29 | public static final long DEFAULT_WAIT_TIME = 100; 30 | private int retrys; 31 | private long waitTime; 32 | 33 | public RetryableCallable() 34 | { 35 | this(DEFAULT_NUMBER_OF_RETRIES, DEFAULT_WAIT_TIME); 36 | } 37 | 38 | public RetryableCallable(int retrys, long waitTime) 39 | { 40 | set(retrys, waitTime); 41 | } 42 | 43 | public void set(int retrys, long waitTime) 44 | { 45 | this.retrys = retrys; 46 | this.waitTime = waitTime; 47 | } 48 | 49 | public abstract T retriableCall() throws Exception; 50 | 51 | public T call() throws Exception 52 | { 53 | int retry = 0; 54 | int logCounter = 0; 55 | while (true) 56 | { 57 | try 58 | { 59 | return retriableCall(); 60 | } 61 | catch (CancellationException e) 62 | { 63 | throw e; 64 | } 65 | catch (Exception e) 66 | { 67 | retry++; 68 | if (retry == retrys) 69 | { 70 | throw e; 71 | } 72 | logger.error(String.format("Retry #%d for: %s",retry, e.getMessage())); 73 | 74 | if(++logCounter == 1) 75 | logger.error("Exception --> "+ExceptionUtils.getFullStackTrace(e)); 76 | Thread.sleep(waitTime); 77 | } 78 | finally 79 | { 80 | forEachExecution(); 81 | } 82 | } 83 | } 84 | 85 | public void forEachExecution() 86 | { 87 | // do nothing by default. 88 | } 89 | } -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/Sleeper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import com.google.inject.ImplementedBy; 19 | 20 | /** 21 | * An abstraction to {@link Thread#sleep(long)} so we can mock it in tests. 22 | */ 23 | @ImplementedBy(ThreadSleeper.class) 24 | public interface Sleeper 25 | { 26 | void sleep(long waitTimeMs) throws InterruptedException; 27 | void sleepQuietly(long waitTimeMs); 28 | } 29 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/SystemUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import java.io.ByteArrayOutputStream; 19 | import java.io.DataInputStream; 20 | import java.io.File; 21 | import java.io.FilterInputStream; 22 | import java.io.IOException; 23 | import java.net.HttpURLConnection; 24 | import java.net.URL; 25 | import java.util.List; 26 | 27 | import org.apache.commons.io.FileUtils; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import com.google.common.base.Charsets; 32 | 33 | 34 | public class SystemUtils 35 | { 36 | private static final Logger logger = LoggerFactory.getLogger(SystemUtils.class); 37 | 38 | /** 39 | * REST call 40 | * @param url 41 | * @return the response from the HTTP GET 42 | */ 43 | public static String getDataFromUrl(String url) 44 | { 45 | try 46 | { 47 | HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); 48 | conn.setConnectTimeout(1000); 49 | conn.setReadTimeout(1000); 50 | conn.setRequestMethod("GET"); 51 | if (conn.getResponseCode() != 200) 52 | { 53 | throw new RuntimeException("Unable to get data for URL " + url); 54 | } 55 | byte[] b = new byte[2048]; 56 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 57 | DataInputStream d = new DataInputStream((FilterInputStream) conn.getContent()); 58 | int c = 0; 59 | while ((c = d.read(b, 0, b.length)) != -1) 60 | bos.write(b, 0, c); 61 | String return_ = new String(bos.toByteArray(), Charsets.UTF_8); 62 | logger.info("Calling URL API: {} returns: {}", url, return_); 63 | conn.disconnect(); 64 | return return_; 65 | } 66 | catch (Exception ex) 67 | { 68 | throw new RuntimeException(ex); 69 | } 70 | 71 | } 72 | 73 | /** 74 | * delete all the files/dirs in the given Directory but dont delete the dir 75 | * itself. 76 | */ 77 | public static void cleanupDir(String dirPath, List childdirs) throws IOException 78 | { 79 | if (childdirs == null || childdirs.size() == 0) 80 | FileUtils.cleanDirectory(new File(dirPath)); 81 | else 82 | { 83 | for (String cdir : childdirs) 84 | FileUtils.cleanDirectory(new File(dirPath + "/" + cdir)); 85 | } 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/ThreadSleeper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | /** 19 | * Sleeper impl that delegates to Thread.sleep 20 | */ 21 | public class ThreadSleeper implements Sleeper 22 | { 23 | @Override 24 | public void sleep(long waitTimeMs) throws InterruptedException 25 | { 26 | Thread.sleep(waitTimeMs); 27 | } 28 | 29 | public void sleepQuietly(long waitTimeMs) 30 | { 31 | try 32 | { 33 | sleep(waitTimeMs); 34 | } 35 | catch (InterruptedException e) 36 | { 37 | //no-op 38 | } 39 | 40 | } 41 | 42 | 43 | } -------------------------------------------------------------------------------- /dynomitemanager-common/src/main/java/com/netflix/nfsidecar/utils/TokenManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.nfsidecar.utils; 17 | 18 | import java.math.BigInteger; 19 | import java.util.List; 20 | 21 | import com.google.common.annotations.VisibleForTesting; 22 | import com.google.common.base.Preconditions; 23 | import com.google.common.collect.Ordering; 24 | 25 | public class TokenManager implements ITokenManager 26 | { 27 | public static final BigInteger MINIMUM_TOKEN = BigInteger.ZERO; 28 | //Currently using murmur so max value is 2^32 - 1 29 | public static final BigInteger MAXIMUM_TOKEN = new BigInteger("2").pow(32).add(new BigInteger("-1")); 30 | 31 | /** 32 | * Calculate a token for the given position, evenly spaced from other size-1 nodes. See 33 | * 34 | * @param size number of slots by which the token space will be divided 35 | * @param position slot number, multiplier 36 | * @param offset added to token 37 | * @return MAXIMUM_TOKEN / size * position + offset, if <= MAXIMUM_TOKEN, otherwise wrap around the MINIMUM_TOKEN 38 | */ 39 | @VisibleForTesting BigInteger initialToken(int size, int position, int offset) 40 | { 41 | Preconditions.checkArgument(size > 0, "size must be > 0"); 42 | Preconditions.checkArgument(offset >= 0, "offset must be >= 0"); 43 | /* 44 | * TODO: Is this it valid to add "&& position < size" to the following precondition? This currently causes 45 | * unit test failures. 46 | */ 47 | Preconditions.checkArgument(position >= 0, "position must be >= 0"); 48 | return MAXIMUM_TOKEN.divide(BigInteger.valueOf(size)) 49 | .multiply(BigInteger.valueOf(position)) 50 | .add(BigInteger.valueOf(offset)).mod(MAXIMUM_TOKEN); 51 | } 52 | 53 | /** 54 | * Creates a token given the following parameter 55 | * 56 | * @param my_slot 57 | * -- Slot where this instance has to be. 58 | * @param rac_count 59 | * -- Rac count is the numeber of RAC's 60 | * @param rac_size 61 | * -- number of memberships in the rac 62 | * @param region 63 | * -- name of the DC where it this token is created. 64 | */ 65 | @Override 66 | public String createToken(int my_slot, int rac_count, int rac_size, String region) 67 | { 68 | int regionCount = rac_count * rac_size; 69 | return initialToken(regionCount, my_slot, regionOffset(region)).toString(); 70 | } 71 | 72 | @Override 73 | public String createToken(int my_slot, int totalCount, String region) 74 | { 75 | return initialToken(totalCount, my_slot, regionOffset(region)).toString(); 76 | } 77 | 78 | @Override 79 | public BigInteger findClosestToken(BigInteger tokenToSearch, List tokenList) 80 | { 81 | Preconditions.checkArgument(!tokenList.isEmpty(), "token list must not be empty"); 82 | List sortedTokens = Ordering.natural().sortedCopy(tokenList); 83 | int index = Ordering.natural().binarySearch(sortedTokens, tokenToSearch); 84 | if (index < 0) 85 | { 86 | int i = Math.abs(index) - 1; 87 | if ((i >= sortedTokens.size()) || (i > 0 && sortedTokens.get(i).subtract(tokenToSearch) 88 | .compareTo(tokenToSearch.subtract(sortedTokens.get(i - 1))) > 0)) 89 | --i; 90 | return sortedTokens.get(i); 91 | } 92 | return sortedTokens.get(index); 93 | } 94 | 95 | /** 96 | * Due to warm bootstrap feature, we make region offset to be the same for all DCs 97 | * and will support different offsets later 98 | */ 99 | @Override 100 | public int regionOffset(String dataCenter) 101 | { 102 | return Math.abs(reverse("Dynomite").hashCode()); 103 | //return Math.abs(reverse(dataCenter).hashCode()); 104 | } 105 | 106 | private String reverse(String s) 107 | { 108 | if (s == null) 109 | return null; 110 | 111 | StringBuilder sb = new StringBuilder(); 112 | for(int i=s.length()-1; i>=0; i--) { 113 | sb.append(s.charAt(i)); 114 | } 115 | 116 | return sb.toString(); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /dynomitemanager-core/.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /bin/ 3 | -------------------------------------------------------------------------------- /dynomitemanager-core/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java-library' 2 | 3 | dependencies { 4 | // Project 5 | api project(':dynomitemanager-common') 6 | 7 | api "com.netflix.runtime:health-api:latest.release" 8 | api group: 'com.netflix.archaius', name: 'archaius2-guice', version: '2.1.11' 9 | api "com.netflix.servo:servo-core:latest.release" 10 | 11 | //External 12 | api 'javax.inject:javax.inject:1' 13 | 14 | api group: 'commons-httpclient', name: 'commons-httpclient', version: '3.1' 15 | api "org.codehaus.jettison:jettison:latest.release" 16 | 17 | 18 | api "redis.clients:jedis:2.9.0" 19 | api "org.yaml:snakeyaml:1.17" 20 | api "com.googlecode.json-simple:json-simple:1.1" 21 | 22 | api "javax.ws.rs:jsr311-api:1.1.1" 23 | api "org.apache.cassandra:cassandra-thrift:1.2.11" 24 | api "org.quartz-scheduler:quartz:1.8.6" 25 | api "org.xerial.snappy:snappy-java:1.0.4.1" 26 | api "com.googlecode.json-simple:json-simple:1.1" 27 | api 'commons-io:commons-io:2.5' 28 | api 'commons-cli:commons-cli:1.3.1' 29 | api "javax.ws.rs:jsr311-api:1.1.1" 30 | api "joda-time:joda-time:2.0" 31 | api "commons-configuration:commons-configuration:1.5" 32 | api "net.java.dev.jna:jna:3.2.3" 33 | 34 | 35 | //Unit test 36 | testImplementation 'junit:junit:4.+' 37 | testImplementation 'org.mockito:mockito-all:1.+' 38 | testImplementation 'org.assertj:assertj-core:3.3.0' 39 | testImplementation 'com.netflix.governator:governator-test-junit:[1.15.3,)' 40 | 41 | 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/defaultimpl/test/FakeEnvVariables.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.defaultimpl.test; 2 | 3 | import com.netflix.florida.resources.env.IEnvVariables; 4 | 5 | public class FakeEnvVariables implements IEnvVariables { 6 | 7 | @Override 8 | public String getDynomiteClusterName() { 9 | return "Dynomite"; 10 | } 11 | 12 | @Override 13 | public String getRegion() { 14 | return "us-east-1"; 15 | } 16 | 17 | @Override 18 | public String getRack() { 19 | return "us-east-1c"; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/defaultimpl/test/FakeInstanceDataRetriever.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.defaultimpl.test; 2 | 3 | import com.netflix.florida.instance.InstanceDataRetriever; 4 | 5 | public class FakeInstanceDataRetriever implements InstanceDataRetriever { 6 | 7 | @Override 8 | public String getRac() { 9 | return "us-east-1"; 10 | } 11 | 12 | @Override 13 | public String getPublicHostname() { 14 | return "dynomite"; 15 | } 16 | 17 | @Override 18 | public String getPublicIP() { 19 | return "0.0.0.0"; 20 | } 21 | 22 | @Override 23 | public String getInstanceId() { 24 | return "i-abcdefg"; 25 | } 26 | 27 | @Override 28 | public String getInstanceType() { 29 | return "r3.2xlarge"; 30 | } 31 | 32 | @Override 33 | public String getMac() { 34 | return "00:00:00:00:00"; 35 | } 36 | 37 | @Override 38 | public String getVpcId() { 39 | return "no"; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/defaultimpl/test/FakeInstanceIdentity.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.defaultimpl.test; 2 | 3 | import java.net.UnknownHostException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import com.netflix.florida.identity.InstanceIdentity; 8 | 9 | public class FakeInstanceIdentity extends InstanceIdentity { 10 | 11 | public FakeInstanceIdentity() throws Exception { 12 | super(null, null, null, null, null, null, null, new FakeEnvVariables()); 13 | } 14 | 15 | @Override 16 | public void init() throws Exception { 17 | // overrides by-design so it forces not to init the InstanceIdentity. 18 | } 19 | 20 | @Override 21 | public String getTokens() { 22 | return "101134286"; 23 | } 24 | 25 | @Override 26 | public List getSeeds() throws UnknownHostException { 27 | List seeds = new ArrayList<>(); 28 | seeds.add("dynomite.us-west-2.prod.myaws.com:8101:us-west-2a:us-west-2:1383429731"); 29 | seeds.add("dynomite.us-west-2.prod.myaws.com:8101:us-west-2b:us-west-2:1383429731"); 30 | seeds.add("dynomite.us-west-2.prod.myaws.com:8101:us-west-2c:us-west-2:1383429731"); 31 | return seeds; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/defaultimpl/test/FakeInstanceState.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.defaultimpl.test; 2 | 3 | import com.netflix.florida.identity.IInstanceState; 4 | 5 | public class FakeInstanceState implements IInstanceState { 6 | 7 | @Override 8 | public boolean isSideCarProcessAlive() { 9 | return false; 10 | } 11 | 12 | @Override 13 | public boolean isBootstrapping() { 14 | return false; 15 | } 16 | 17 | @Override 18 | public boolean getYmlWritten() { 19 | return false; 20 | } 21 | 22 | @Override 23 | public void setYmlWritten(boolean b) { 24 | // TODO Auto-generated method stub 25 | 26 | } 27 | 28 | } -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/defaultimpl/test/FakeStorageProxy.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.defaultimpl.test; 2 | 3 | import java.io.IOException; 4 | 5 | import com.netflix.florida.sidecore.storage.Bootstrap; 6 | import com.netflix.florida.sidecore.storage.StorageProxy; 7 | 8 | public class FakeStorageProxy implements StorageProxy { 9 | 10 | @Override 11 | public boolean isAlive() { 12 | // TODO Auto-generated method stub 13 | return false; 14 | } 15 | 16 | @Override 17 | public long getUptime() { 18 | // TODO Auto-generated method stub 19 | return 0; 20 | } 21 | 22 | @Override 23 | public Bootstrap warmUpStorage(String[] peers) { 24 | // TODO Auto-generated method stub 25 | return null; 26 | } 27 | 28 | @Override 29 | public boolean resetStorage() { 30 | // TODO Auto-generated method stub 31 | return false; 32 | } 33 | 34 | @Override 35 | public boolean takeSnapshot() { 36 | // TODO Auto-generated method stub 37 | return false; 38 | } 39 | 40 | @Override 41 | public boolean loadingData() { 42 | // TODO Auto-generated method stub 43 | return false; 44 | } 45 | 46 | @Override 47 | public void stopPeerSync() { 48 | // TODO Auto-generated method stub 49 | 50 | } 51 | 52 | @Override 53 | public String getEngine() { 54 | // TODO Auto-generated method stub 55 | return null; 56 | } 57 | 58 | @Override 59 | public int getEngineNumber() { 60 | // TODO Auto-generated method stub 61 | return 0; 62 | } 63 | 64 | @Override 65 | public void updateConfiguration() throws IOException { 66 | // TODO Auto-generated method stub 67 | 68 | } 69 | 70 | @Override 71 | public String getStartupScript() { 72 | // TODO Auto-generated method stub 73 | return null; 74 | } 75 | 76 | @Override 77 | public String getStopScript() { 78 | // TODO Auto-generated method stub 79 | return null; 80 | } 81 | 82 | @Override 83 | public String getIpAddress() { 84 | // TODO Auto-generated method stub 85 | return null; 86 | } 87 | 88 | @Override 89 | public int getPort() { 90 | // TODO Auto-generated method stub 91 | return 0; 92 | } 93 | 94 | @Override 95 | public String getUnixPath() { 96 | return ""; 97 | } 98 | 99 | @Override 100 | public long getStoreMaxMem() { 101 | // TODO Auto-generated method stub 102 | return 0; 103 | } 104 | 105 | @Override 106 | public long getTotalAvailableSystemMemory() { 107 | // TODO Auto-generated method stub 108 | return 0; 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/defaultimpl/test/FloridaStandardTunerTest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.defaultimpl.test; 2 | 3 | import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; 4 | 5 | import java.io.File; 6 | import java.nio.file.Files; 7 | import java.nio.file.Paths; 8 | 9 | import org.apache.commons.io.FileUtils; 10 | import org.junit.Assert; 11 | import org.junit.Test; 12 | import com.netflix.florida.dynomite.DynomiteStandardTuner; 13 | 14 | public class FloridaStandardTunerTest { 15 | 16 | @Test 17 | public void testWriteAllProperties() throws Exception { 18 | DynomiteStandardTuner tuner = new DynomiteStandardTuner(new BlankConfiguration(), new FakeInstanceIdentity(), 19 | new FakeInstanceState(), new FakeStorageProxy(), new FakeEnvVariables(), new FakeInstanceDataRetriever()); 20 | 21 | String yamlPath = System.getProperty("java.io.tmpdir") + "/yaml-tunner.yaml"; 22 | String templateYamlPath = new File(".").getCanonicalPath() + "/src/test/resources/sample-yaml.yaml"; 23 | Files.copy(Paths.get(templateYamlPath), Paths.get(yamlPath), REPLACE_EXISTING); 24 | 25 | tuner.writeAllProperties(yamlPath); 26 | String result = FileUtils.readFileToString(new File(yamlPath),"UTF-8"); 27 | 28 | Assert.assertNotNull(result); 29 | Assert.assertTrue(result.contains("101134286")); 30 | Assert.assertTrue(result.contains("/apps/dynomite/conf/dynomite.pem")); 31 | Assert.assertTrue(result.contains(new FakeInstanceIdentity().getSeeds().get(0))); 32 | Assert.assertTrue(result.contains(new FakeInstanceIdentity().getSeeds().get(1))); 33 | Assert.assertTrue(result.contains(new FakeInstanceIdentity().getSeeds().get(2))); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/sidecore/monitoring/test/RedisInfoMetricsTaskTest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.sidecore.monitoring.test; 2 | 3 | import java.io.File; 4 | import java.nio.file.Files; 5 | import java.nio.file.Paths; 6 | 7 | import org.junit.Assert; 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | 11 | import com.netflix.florida.config.FloridaConfig; 12 | import com.netflix.florida.defaultimpl.test.BlankConfiguration; 13 | import com.netflix.florida.defaultimpl.test.FakeStorageProxy; 14 | import com.netflix.florida.monitoring.JedisFactory; 15 | import com.netflix.florida.monitoring.RedisInfoMetricsTask; 16 | import com.netflix.florida.sidecore.storage.StorageProxy; 17 | import com.netflix.servo.DefaultMonitorRegistry; 18 | 19 | import mockit.Expectations; 20 | import mockit.Mocked; 21 | import mockit.integration.junit4.JMockit; 22 | import redis.clients.jedis.Jedis; 23 | 24 | @RunWith(JMockit.class) 25 | public class RedisInfoMetricsTaskTest { 26 | 27 | @Mocked 28 | Jedis jedis; 29 | 30 | @Test 31 | public void executeTest() throws Exception { 32 | 33 | int metricsCountSampleRedisInfo = 26; 34 | 35 | File file = new File(new File(".").getCanonicalPath() + "/src/test/resources/redis_info.txt"); 36 | final String info = new String(Files.readAllBytes((Paths.get(file.getPath())))); 37 | 38 | new Expectations() { 39 | { 40 | jedis.connect(); 41 | jedis.info(); 42 | result = info; 43 | jedis.disconnect(); 44 | } 45 | }; 46 | 47 | JedisFactory jedisFactory = new JedisFactory() { 48 | 49 | @Override 50 | public Jedis newInstance(String hostname, int port) { 51 | return jedis; 52 | } 53 | }; 54 | 55 | StorageProxy storageProxy = new FakeStorageProxy(); 56 | 57 | RedisInfoMetricsTask mimt = new RedisInfoMetricsTask(storageProxy, jedisFactory); 58 | mimt.execute(); 59 | 60 | Assert.assertNotNull(DefaultMonitorRegistry.getInstance().getRegisteredMonitors()); 61 | Assert.assertEquals(metricsCountSampleRedisInfo, 62 | DefaultMonitorRegistry.getInstance().getRegisteredMonitors().size()); 63 | 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/utils/test/ArdbConfParserTest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.utils.test; 2 | 3 | import java.io.File; 4 | import java.util.Scanner; 5 | 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import com.netflix.florida.sidecore.storage.ArdbRocksDbRedisCompatible; 10 | 11 | public class ArdbConfParserTest { 12 | @Test 13 | public void test() throws Exception { 14 | 15 | /** 16 | * some random values 17 | */ 18 | int writeBufferSize = 128; 19 | int maxWriteBufferNumber = 16; 20 | int minWriteBufferToMerge = 4; 21 | long storeMaxMem = 10000000; 22 | 23 | String configPathName = "./src/test/resources/rocksdb.conf"; 24 | ArdbRocksDbRedisCompatible checkConf = new ArdbRocksDbRedisCompatible(storeMaxMem, writeBufferSize, 25 | maxWriteBufferNumber, minWriteBufferToMerge); 26 | checkConf.updateConfiguration(configPathName); 27 | 28 | String conf = new Scanner(new File(configPathName)).useDelimiter("\\Z").next(); 29 | 30 | final String bufSize = "write_buffer_size=" + writeBufferSize + "M;"; 31 | int occurrences = 0; 32 | int index = 0; 33 | while (index < conf.length() && (index = conf.indexOf(bufSize, index)) >= 0) { 34 | occurrences++; 35 | index += bufSize.length(); 36 | } 37 | Assert.assertTrue(occurrences == 1); 38 | 39 | final String bufNum = "max_write_buffer_number=" + maxWriteBufferNumber; 40 | occurrences = 0; 41 | index = 0; 42 | while (index < conf.length() && (index = conf.indexOf(bufNum, index)) >= 0) { 43 | occurrences++; 44 | index += bufNum.length(); 45 | } 46 | Assert.assertTrue(occurrences == 1); 47 | 48 | final String bufMerge = "min_write_buffer_number_to_merge=" + minWriteBufferToMerge;; 49 | occurrences = 0; 50 | index = 0; 51 | while (index < conf.length() && (index = conf.indexOf(bufMerge, index)) >= 0) { 52 | occurrences++; 53 | index += bufMerge.length(); 54 | } 55 | Assert.assertTrue(occurrences == 1); 56 | 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/utils/test/FakeSleeper.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.utils.test; 2 | 3 | import com.netflix.florida.sidecore.utils.Sleeper; 4 | 5 | public class FakeSleeper implements Sleeper 6 | { 7 | @Override 8 | public void sleep(long waitTimeMs) throws InterruptedException 9 | { 10 | // no-op 11 | } 12 | 13 | public void sleepQuietly(long waitTimeMs) 14 | { 15 | //no-op 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/utils/test/FloridaHealthCheckHandlerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Netflix, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use 3 | * this file except in compliance with the License. You may obtain a copy of the License at

4 | * http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software 5 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 6 | * either express or implied. See the License for the specific language governing permissions and limitations under the 7 | * License. 8 | */ 9 | package com.netflix.florida.utils.test; 10 | 11 | import org.junit.Assert; 12 | import org.junit.Test; 13 | 14 | import com.netflix.florida.config.InstanceState; 15 | import com.netflix.florida.resources.FloridaHealthCheckHandler; 16 | 17 | /** 18 | * FloridaHealthCheckHandler unit tests 19 | * 20 | */ 21 | public class FloridaHealthCheckHandlerTest { 22 | 23 | @Test 24 | public void testHandlerBootstrapping() { 25 | InstanceState state = new InstanceState() { 26 | public boolean isBootstrapping() { 27 | return true; 28 | } 29 | 30 | ; 31 | }; 32 | FloridaHealthCheckHandler fhc = new FloridaHealthCheckHandler(state); 33 | Assert.assertEquals(503, fhc.getStatus()); 34 | } 35 | 36 | @Test 37 | public void testHandlerNotHealthy() { 38 | InstanceState state = new InstanceState() { 39 | public boolean isHealthy() { 40 | return false; 41 | } 42 | 43 | ; 44 | }; 45 | FloridaHealthCheckHandler fhc = new FloridaHealthCheckHandler(state); 46 | Assert.assertEquals(503, fhc.getStatus()); 47 | } 48 | 49 | @Test 50 | public void testHandlerOK() { 51 | InstanceState state = new InstanceState() { 52 | public boolean isHealthy() { 53 | return true; 54 | } 55 | 56 | ; 57 | }; 58 | FloridaHealthCheckHandler fhc = new FloridaHealthCheckHandler(state); 59 | Assert.assertEquals(200, fhc.getStatus()); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/utils/test/ServoMetricsTastTest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida.utils.test; 2 | 3 | import java.util.concurrent.ConcurrentHashMap; 4 | 5 | import junit.framework.Assert; 6 | 7 | import org.junit.Test; 8 | 9 | import com.netflix.florida.monitoring.ServoMetricsTask; 10 | import com.netflix.servo.monitor.NumericMonitor; 11 | 12 | public class ServoMetricsTastTest { 13 | 14 | @Test 15 | public void test() throws Exception { 16 | 17 | String json = "{\"service\":\"dynomite\", \"source\":\"florida-i-16ca1846\", \"version\":\"0.3.1\", \"uptime\":40439, \"timestamp\":1399064677, \"datacenter\":\"DC1\"," 18 | + "\"dyn_o_mite\":" 19 | + "{\"client_eof\":0, \"client_err\":0, \"client_connections\":3, \"server_ejects\":0, \"forward_error\":0, \"fragments\":0, \"stats_count\":22," 20 | + "\"127.0.0.1\":" 21 | + "{\"server_eof\":0, \"server_err\":0, \"server_timedout\":11, \"server_connections\":3, \"server_ejected_at\":0, \"requests\":20000," 22 | + "\"request_bytes\":0, \"responses\":5, \"response_bytes\":0, \"in_queue\":0, \"in_queue_bytes\":0, \"out_queue\":0," 23 | + "\"out_queue_bytes\":0" + "}" + "}" + "}"; 24 | 25 | ServoMetricsTask impl = new ServoMetricsTask(null); 26 | impl.processJsonResponse(json); 27 | 28 | ConcurrentHashMap> metricMap = impl.getMetricsMap(); 29 | 30 | testCounterValue("dynomite__client_eof", 0, metricMap); 31 | testCounterValue("dynomite__client_err", 0, metricMap); 32 | testCounterValue("dynomite__client_connections", 3, metricMap); 33 | testCounterValue("dynomite__server_ejects", 0, metricMap); 34 | testCounterValue("dynomite__forward_error", 0, metricMap); 35 | testCounterValue("dynomite__fragments", 0, metricMap); 36 | testCounterValue("dynomite__stats_count", 22, metricMap); 37 | testCounterValue("dynomite__127.0.0.1__server_eof", 0, metricMap); 38 | testCounterValue("dynomite__127.0.0.1__server_err", 0, metricMap); 39 | testCounterValue("dynomite__127.0.0.1__server_timedout", 11, metricMap); 40 | testCounterValue("dynomite__127.0.0.1__server_connections", 3, metricMap); 41 | testCounterValue("dynomite__127.0.0.1__server_ejected_at", 0, metricMap); 42 | testCounterValue("dynomite__127.0.0.1__requests", 20000, metricMap); 43 | testCounterValue("dynomite__127.0.0.1__request_bytes", 0, metricMap); 44 | testCounterValue("dynomite__127.0.0.1__responses", 5, metricMap); 45 | testCounterValue("dynomite__127.0.0.1__response_bytes", 0, metricMap); 46 | testCounterValue("dynomite__127.0.0.1__in_queue", 0, metricMap); 47 | testCounterValue("dynomite__127.0.0.1__in_queue_bytes", 0, metricMap); 48 | testCounterValue("dynomite__127.0.0.1__out_queue", 0, metricMap); 49 | testCounterValue("dynomite__127.0.0.1__out_queue_bytes", 0, metricMap); 50 | } 51 | 52 | private void testCounterValue(String name, int expectedValue, 53 | ConcurrentHashMap> metricMap) throws Exception { 54 | 55 | NumericMonitor metric = metricMap.get(name); 56 | Assert.assertNotNull(metric); 57 | Assert.assertEquals(expectedValue, metric.getValue().intValue()); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /dynomitemanager-core/java/com/netflix/florida/utils/test/TokenManagerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Netflix, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use 3 | * this file except in compliance with the License. You may obtain a copy of the License at

4 | * http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software 5 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 6 | * either express or implied. See the License for the specific language governing permissions and limitations under the 7 | * License. 8 | */ 9 | package com.netflix.florida.utils.test; 10 | 11 | import org.junit.Assert; 12 | import org.junit.Test; 13 | 14 | import com.netflix.florida.sidecore.utils.TokenManager; 15 | 16 | public class TokenManagerTest { 17 | 18 | @Test 19 | public void createTokenTest() { 20 | TokenManager tm = new TokenManager(); 21 | String token = tm.createToken(0, 1, "us-west-2"); 22 | Assert.assertNotNull(token); 23 | Assert.assertTrue(!"".equals(token)); 24 | Assert.assertEquals("1383429731", token); 25 | } 26 | 27 | @Test 28 | public void createToken2Test() { 29 | TokenManager tm = new TokenManager(); 30 | String token = tm.createToken(1, 2, "us-west-2"); 31 | Assert.assertNotNull(token); 32 | Assert.assertTrue(!"".equals(token)); 33 | Assert.assertEquals("3530913378", token); 34 | } 35 | 36 | @Test 37 | public void createTokenRackAndSizeTest() { 38 | TokenManager tm = new TokenManager(); 39 | String token = tm.createToken(1, 2, 2, "us-west-2"); 40 | Assert.assertNotNull(token); 41 | Assert.assertTrue(!"".equals(token)); 42 | Assert.assertEquals("2457171554", token); 43 | } 44 | 45 | @Test(expected = IllegalArgumentException.class) 46 | public void createTokenWorngCountTest() { 47 | TokenManager tm = new TokenManager(); 48 | tm.createToken(0, -1, "us-west-2"); 49 | } 50 | 51 | @Test(expected = IllegalArgumentException.class) 52 | public void createTokenWorngSlotTest() { 53 | TokenManager tm = new TokenManager(); 54 | tm.createToken(-1, 0, "us-west-2"); 55 | } 56 | 57 | @Test(expected = IllegalArgumentException.class) 58 | public void createTokenWorngRackCountTest() { 59 | TokenManager tm = new TokenManager(); 60 | tm.createToken(1, -1, 2, "us-west-2"); 61 | } 62 | 63 | @Test(expected = IllegalArgumentException.class) 64 | public void createTokenWorngSizeTest() { 65 | TokenManager tm = new TokenManager(); 66 | tm.createToken(1, 1, -1, "us-west-2"); 67 | } 68 | 69 | @Test 70 | public void createRegionOffSet() { 71 | TokenManager tm = new TokenManager(); 72 | tm.createToken(0, 2, "us-west-2"); 73 | int offSet = tm.regionOffset("us-west-2"); 74 | Assert.assertTrue(offSet >= 1); 75 | Assert.assertEquals(1383429731, offSet); 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/backup/RestoreTask.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.backup; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.joda.time.DateTime; 6 | 7 | import com.google.inject.Inject; 8 | import com.google.inject.Singleton; 9 | import com.netflix.dynomitemanager.config.InstanceState; 10 | import com.netflix.dynomitemanager.dynomite.IDynomiteProcess; 11 | import com.netflix.dynomitemanager.storage.StorageProcessManager; 12 | import com.netflix.dynomitemanager.storage.StorageProxy; 13 | import com.netflix.nfsidecar.backup.Restore; 14 | import com.netflix.nfsidecar.config.CommonConfig; 15 | import com.netflix.nfsidecar.scheduler.Task; 16 | import com.netflix.nfsidecar.utils.Sleeper; 17 | 18 | /** 19 | * Task for restoring snapshots from object storage 20 | */ 21 | 22 | @Singleton 23 | public class RestoreTask extends Task { 24 | public static final String TaskName = "RestoreTask"; 25 | private static final Logger logger = LoggerFactory.getLogger(RestoreTask.class); 26 | private final InstanceState state; 27 | private final StorageProxy storageProxy; 28 | private final IDynomiteProcess dynProcess; 29 | private final Sleeper sleeper; 30 | private final Restore restore; 31 | private StorageProcessManager storageProcessMgr; 32 | private final CommonConfig config; 33 | 34 | @Inject 35 | public RestoreTask(CommonConfig config, InstanceState state, StorageProxy storageProxy, 36 | IDynomiteProcess dynProcess, Sleeper sleeper, Restore restore, StorageProcessManager storageProcessMgr) { 37 | this.config = config; 38 | this.state = state; 39 | this.storageProxy = storageProxy; 40 | this.dynProcess = dynProcess; 41 | this.sleeper = sleeper; 42 | this.restore = restore; 43 | this.storageProcessMgr = storageProcessMgr; 44 | } 45 | 46 | public void execute() throws Exception { 47 | this.state.setRestoring(true); 48 | this.state.setFirstRestore(false); 49 | /** 50 | * Set the status of the restore to "false" every time we start a 51 | * restore. This will ensure that prior to restore we recapture the 52 | * status of the restore. 53 | */ 54 | this.state.setRestoreStatus(false); 55 | 56 | /* stop dynomite process */ 57 | this.dynProcess.stop(); 58 | 59 | // stop storage process 60 | this.storageProcessMgr.stop(); 61 | 62 | /* restore from Object Storage */ 63 | if (restore.restoreData(config.getRestoreDate())) { 64 | /* start storage process and load data */ 65 | logger.info("Restored successful: Starting storage process with loading data."); 66 | this.storageProcessMgr.start(); 67 | if (!this.storageProxy.loadingData()) { 68 | logger.error("Restore not successful: Restore failed because of Redis."); 69 | } 70 | logger.info("Restore Completed, sleeping 5 seconds before starting Dynomite!"); 71 | 72 | sleeper.sleepQuietly(5000); 73 | this.dynProcess.start(); 74 | logger.info("Dynomite started"); 75 | this.state.setRestoreStatus(true); 76 | } else { 77 | /* start storage process without loading data */ 78 | logger.error("Restore not successful: Starting storage process without loading data."); 79 | } 80 | this.state.setRestoring(false); 81 | this.state.setRestoreTime(DateTime.now()); 82 | } 83 | 84 | @Override 85 | public String getName() { 86 | return TaskName; 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/dualAccount/AwsRoleAssumptionCredential.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.dualAccount; 2 | 3 | 4 | import com.amazonaws.auth.AWSCredentialsProvider; 5 | import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; 6 | import com.google.inject.Inject; 7 | import com.netflix.nfsidecar.aws.ICredential; 8 | import com.netflix.nfsidecar.config.AWSCommonConfig; 9 | import com.netflix.nfsidecar.identity.InstanceEnvIdentity; 10 | 11 | public class AwsRoleAssumptionCredential implements ICredential { 12 | private static final String AWS_ROLE_ASSUMPTION_SESSION_NAME = "AwsRoleAssumptionSession"; 13 | private ICredential cred; 14 | private InstanceEnvIdentity insEnvIdentity; 15 | private AWSCredentialsProvider stsSessionCredentialsProvider; 16 | private AWSCommonConfig config; 17 | 18 | @Inject 19 | public AwsRoleAssumptionCredential(ICredential cred, AWSCommonConfig config, InstanceEnvIdentity insEnvIdentity) { 20 | this.cred = cred; 21 | this.config = config; 22 | this.insEnvIdentity = insEnvIdentity; 23 | } 24 | 25 | @Override 26 | public AWSCredentialsProvider getAwsCredentialProvider() { 27 | if (this.stsSessionCredentialsProvider == null) { 28 | synchronized (this) { 29 | if (this.config.isDualAccount() || this.stsSessionCredentialsProvider == null) { 30 | 31 | String roleArn = null; 32 | /** 33 | * Create the assumed IAM role based on the environment. For 34 | * example, if the current environment is VPC, then the 35 | * assumed role is for EC2 classic, and vice versa. 36 | */ 37 | if (this.insEnvIdentity.isClassic()) { 38 | roleArn = this.config.getVpcAWSRoleAssumptionArn(); // Env 39 | // is 40 | // EC2 41 | // classic 42 | // --> 43 | // IAM 44 | // assumed 45 | // role 46 | // for 47 | // VPC 48 | // created 49 | } else { 50 | roleArn = this.config.getClassicAWSRoleAssumptionArn(); // Env 51 | // is 52 | // VPC 53 | // --> 54 | // IAM 55 | // assumed 56 | // role 57 | // for 58 | // EC2 59 | // classic 60 | // created 61 | } 62 | 63 | // 64 | if (roleArn == null || roleArn.isEmpty()) 65 | throw new NullPointerException( 66 | "Role ARN is null or empty probably due to missing config entry"); 67 | 68 | /** 69 | * Get handle to an implementation that uses AWS Security 70 | * Token Service (STS) to create temporary, short-lived 71 | * session with explicit refresh for session/token 72 | * expiration. 73 | */ 74 | try { 75 | this.stsSessionCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider( 76 | this.cred.getAwsCredentialProvider(), roleArn, AWS_ROLE_ASSUMPTION_SESSION_NAME); 77 | 78 | } catch (Exception ex) { 79 | throw new IllegalStateException( 80 | "Exception in getting handle to AWS Security Token Service (STS). Msg: " 81 | + ex.getLocalizedMessage(), 82 | ex); 83 | } 84 | 85 | } 86 | 87 | } 88 | } 89 | 90 | return this.stsSessionCredentialsProvider; 91 | 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/dynomite/DynomiteRest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.dynomite; 2 | 3 | import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler; 4 | import org.apache.commons.httpclient.HttpClient; 5 | import org.apache.commons.httpclient.methods.GetMethod; 6 | import org.apache.commons.httpclient.params.HttpMethodParams; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import com.netflix.config.DynamicPropertyFactory; 11 | import com.netflix.config.DynamicStringProperty; 12 | 13 | public class DynomiteRest { 14 | 15 | private static final Logger logger = LoggerFactory.getLogger(DynomiteRest.class); 16 | 17 | 18 | public static boolean sendCommand(String cmd) { 19 | DynamicStringProperty adminUrl = 20 | DynamicPropertyFactory.getInstance().getStringProperty("florida.metrics.url", "http://localhost:22222"); 21 | 22 | String url = adminUrl.get() + cmd; 23 | HttpClient client = new HttpClient(); 24 | client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, 25 | new DefaultHttpMethodRetryHandler()); 26 | 27 | GetMethod get = new GetMethod(url); 28 | try { 29 | int statusCode = client.executeMethod(get); 30 | if (!(statusCode == 200)) { 31 | logger.error("Got non 200 status code from " + url); 32 | return false; 33 | } 34 | 35 | String response = get.getResponseBodyAsString(); 36 | //logger.info("Received response from " + url + "\n" + response); 37 | 38 | if (!response.isEmpty()) { 39 | logger.info("Received response from " + url + "\n" + response); 40 | } else { 41 | logger.error("Cannot parse empty response from " + url); 42 | return false; 43 | } 44 | 45 | } catch (Exception e) { 46 | logger.error("Failed to sendCommand and invoke url: " + url, e); 47 | return false; 48 | } 49 | logger.info("Dynomite REST completed succesfully: " + url); 50 | 51 | return true; 52 | } 53 | 54 | 55 | } 56 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/dynomite/DynomiteYamlTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.dynomitemanager.dynomite; 17 | 18 | import com.google.inject.Inject; 19 | import com.google.inject.Singleton; 20 | import com.netflix.dynomitemanager.config.FloridaConfig; 21 | import com.netflix.nfsidecar.scheduler.SimpleTimer; 22 | import com.netflix.nfsidecar.scheduler.Task; 23 | import com.netflix.nfsidecar.scheduler.TaskTimer; 24 | import com.netflix.nfsidecar.utils.ProcessTuner; 25 | 26 | @Singleton 27 | public class DynomiteYamlTask extends Task 28 | { 29 | public static final String JOBNAME = "Tune-Task"; 30 | 31 | private final ProcessTuner tuner; 32 | private final FloridaConfig config; 33 | 34 | @Inject 35 | public DynomiteYamlTask(FloridaConfig config, ProcessTuner tuner) 36 | { 37 | this.config = config; 38 | this.tuner = tuner; 39 | } 40 | 41 | public void execute() throws Exception 42 | { 43 | tuner.writeAllProperties(config.getDynomiteYaml()); 44 | } 45 | 46 | @Override 47 | public String getName() 48 | { 49 | return "Tune-Task"; 50 | } 51 | 52 | // update the YML every 60 seconds. 53 | public static TaskTimer getTimer() 54 | { 55 | return new SimpleTimer(JOBNAME, 60L * 1000); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/dynomite/IDynomiteProcess.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.dynomite; 2 | 3 | import java.io.IOException; 4 | 5 | 6 | /** 7 | * Interface to aid in starting and stopping Dynomite. 8 | * 9 | */ 10 | public interface IDynomiteProcess 11 | { 12 | void start() throws IOException; 13 | 14 | void stop() throws IOException; 15 | 16 | boolean dynomiteCheck(); 17 | 18 | boolean dynomiteProcessCheck(); 19 | } 20 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/dynomite/ProxyAndStorageResetTask.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.dynomite; 2 | 3 | import java.io.IOException; 4 | 5 | import com.google.inject.Inject; 6 | import com.google.inject.Singleton; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import redis.clients.jedis.Jedis; 12 | 13 | import com.netflix.dynomitemanager.config.FloridaConfig; 14 | import com.netflix.dynomitemanager.storage.StorageProxy; 15 | import com.netflix.nfsidecar.scheduler.Task; 16 | import com.netflix.nfsidecar.utils.Sleeper; 17 | 18 | @Singleton 19 | public class ProxyAndStorageResetTask extends Task { 20 | public static final String JOBNAME = "ProxyResetTask-Task"; 21 | private static final Logger logger = LoggerFactory.getLogger(ProxyAndStorageResetTask.class); 22 | 23 | private final IDynomiteProcess dynProcess; 24 | private final StorageProxy storageProxy; 25 | private final Sleeper sleeper; 26 | private final FloridaConfig config; 27 | 28 | @Inject 29 | public ProxyAndStorageResetTask(FloridaConfig config, IDynomiteProcess dynProcess, StorageProxy storageProxy, 30 | Sleeper sleeper) { 31 | this.config = config; 32 | this.storageProxy = storageProxy; 33 | this.dynProcess = dynProcess; 34 | this.sleeper = sleeper; 35 | } 36 | 37 | public void execute() throws IOException { 38 | storageProxy.resetStorage(); 39 | dynomiteCheck(); 40 | setConsistency(); 41 | } 42 | 43 | @Override 44 | public String getName() { 45 | return JOBNAME; 46 | } 47 | 48 | private void setConsistency() { 49 | logger.info("Setting the consistency level for the cluster"); 50 | if (!DynomiteRest.sendCommand("/set_consistency/read/" + config.getDynomiteReadConsistency())) 51 | logger.error("REST call to Dynomite for read consistency failed --> using the default"); 52 | 53 | if (!DynomiteRest.sendCommand("/set_consistency/write/" + config.getDynomiteWriteConsistency())) 54 | logger.error("REST call to Dynomite for write consistency failed --> using the default"); 55 | } 56 | 57 | private void dynomiteCheck() { 58 | Jedis dynomiteJedis = new Jedis(config.getDynomiteLocalAddress(), config.getDynomiteClientPort(), 5000); 59 | logger.info("Checking Dynomite's status"); 60 | try { 61 | dynomiteJedis.connect(); 62 | if (dynomiteJedis.ping().equals("PONG") == false) { 63 | logger.warn("Pinging Dynomite failed ---> trying again after 1 sec"); 64 | sleeper.sleepQuietly(1000); 65 | if (dynomiteJedis.ping().equals("PONG") == false) { 66 | try { 67 | this.dynProcess.stop(); 68 | sleeper.sleepQuietly(1000); 69 | this.dynProcess.start(); 70 | } catch (IOException e) { 71 | logger.error("Dynomite cannot be restarted --> Requires manual restart" + e.getMessage()); 72 | } 73 | } else { 74 | logger.info("Dynomite is up and running"); 75 | } 76 | } else { 77 | logger.info("Dynomite is up and running"); 78 | } 79 | } catch (Exception e) { 80 | logger.warn("Unable to connect to Dynomite --> restarting: " + e.getMessage()); 81 | try { 82 | this.dynProcess.stop(); 83 | sleeper.sleepQuietly(1000); 84 | this.dynProcess.start(); 85 | } catch (IOException e1) { 86 | logger.error("Dynomite cannot be restarted --> Requires manual restart" + e1.getMessage()); 87 | } 88 | } 89 | 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/monitoring/JedisFactory.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.monitoring; 2 | 3 | import redis.clients.jedis.Jedis; 4 | 5 | public interface JedisFactory { 6 | public Jedis newInstance(String hostname, int port); 7 | } 8 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/monitoring/RedisInfoMetricsTask.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.monitoring; 2 | 3 | import java.io.ByteArrayInputStream; 4 | import java.io.InputStreamReader; 5 | import java.util.HashSet; 6 | import java.util.Map; 7 | import java.util.Set; 8 | import java.util.concurrent.ConcurrentHashMap; 9 | 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import redis.clients.jedis.Jedis; 14 | 15 | import com.google.inject.Inject; 16 | import com.google.inject.Singleton; 17 | import com.netflix.dynomitemanager.storage.RedisInfoParser; 18 | import com.netflix.dynomitemanager.storage.StorageProxy; 19 | import com.netflix.nfsidecar.scheduler.SimpleTimer; 20 | import com.netflix.nfsidecar.scheduler.Task; 21 | import com.netflix.nfsidecar.scheduler.TaskTimer; 22 | import com.netflix.servo.DefaultMonitorRegistry; 23 | import com.netflix.servo.monitor.Counter; 24 | import com.netflix.servo.monitor.LongGauge; 25 | import com.netflix.servo.monitor.MonitorConfig; 26 | import com.netflix.servo.monitor.Monitors; 27 | import com.netflix.servo.monitor.NumericMonitor; 28 | 29 | @Singleton 30 | public class RedisInfoMetricsTask extends Task { 31 | 32 | private static final Logger Logger = LoggerFactory.getLogger(RedisInfoMetricsTask.class); 33 | public static final String TaskName = "Redis-Info-Task"; 34 | 35 | private static final Set COUNTER_LIST = new HashSet(); 36 | 37 | static { 38 | COUNTER_LIST.add("Redis_Stats_instantaneous_ops_per_sec"); 39 | } 40 | 41 | private final ConcurrentHashMap redisInfoGaugeMetrics = new ConcurrentHashMap(); 42 | private final ConcurrentHashMap> redisInfoCounterMap = new ConcurrentHashMap>(); 43 | 44 | private JedisFactory jedisFactory; 45 | private StorageProxy storageProxy; 46 | 47 | /** 48 | * Default constructor 49 | * 50 | * @param storageProxy 51 | * @param jedisFactory 52 | */ 53 | @Inject 54 | public RedisInfoMetricsTask(StorageProxy storageProxy, JedisFactory jedisFactory) { 55 | this.jedisFactory = jedisFactory; 56 | this.storageProxy = storageProxy; 57 | } 58 | 59 | @Override 60 | public void execute() throws Exception { 61 | 62 | Jedis jedis = jedisFactory.newInstance(storageProxy.getIpAddress(),storageProxy.getPort()); 63 | try { 64 | jedis.connect(); 65 | String s = jedis.info(); 66 | 67 | InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(s.getBytes())); 68 | RedisInfoParser infoParser = new RedisInfoParser(); 69 | Map metrics = infoParser.parse(reader); 70 | 71 | processMetrics(metrics); 72 | 73 | } catch (Exception e) { 74 | Logger.error("Could not get jedis info metrics", e); 75 | } finally { 76 | jedis.disconnect(); 77 | } 78 | } 79 | 80 | private void processMetrics(Map metrics) { 81 | 82 | for (String key : metrics.keySet()) { 83 | 84 | Long value = metrics.get(key); 85 | 86 | if (COUNTER_LIST.contains(key)) { 87 | processCounterMetric(key, value); 88 | } else { 89 | processGaugeMetric(key, value); 90 | } 91 | } 92 | } 93 | 94 | private void processGaugeMetric(String key, Long value) { 95 | if (Logger.isDebugEnabled()) { 96 | Logger.debug("Process gauge: " + key + " " + value); 97 | } 98 | 99 | LongGauge oldGauge = redisInfoGaugeMetrics.get(key); 100 | if (oldGauge != null) { 101 | oldGauge.set(value); 102 | return; 103 | } 104 | 105 | // create a new long gauge 106 | LongGauge newGauge = new LongGauge(MonitorConfig.builder(key).build()); 107 | 108 | oldGauge = redisInfoGaugeMetrics.putIfAbsent(key, newGauge); 109 | if (oldGauge == null) { 110 | newGauge.set(value); 111 | DefaultMonitorRegistry.getInstance().register(newGauge); 112 | } else { 113 | // someone else beat us to it. just use the oldGauge 114 | oldGauge.set(value); 115 | } 116 | } 117 | 118 | private void processCounterMetric(String counterName, Long val) { 119 | 120 | if (Logger.isDebugEnabled()) { 121 | Logger.debug("Process counter: " + counterName + " " + val); 122 | } 123 | 124 | NumericMonitor counter = redisInfoCounterMap.get(counterName); 125 | if (counter != null) { 126 | long increment = val - counter.getValue().longValue(); 127 | ((Counter) counter).increment(increment); 128 | return; 129 | } 130 | 131 | counter = Monitors.newCounter(counterName); 132 | NumericMonitor oldCounter = redisInfoCounterMap.putIfAbsent(counterName, counter); 133 | 134 | if (oldCounter == null) { 135 | // this is the 1st time 136 | DefaultMonitorRegistry.getInstance().register(counter); 137 | } else { 138 | // someone beat us to it, take their obj instead 139 | counter = oldCounter; 140 | } 141 | 142 | long increment = val - counter.getValue().longValue(); 143 | ((Counter) counter).increment(increment); 144 | 145 | } 146 | 147 | @Override 148 | public String getName() { 149 | return TaskName; 150 | } 151 | 152 | public static TaskTimer getTimer() { 153 | // run once every 30 seconds 154 | return new SimpleTimer(TaskName, 30 * 1000); 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/monitoring/SimpleJedisFactory.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.monitoring; 2 | 3 | import redis.clients.jedis.Jedis; 4 | 5 | public class SimpleJedisFactory implements JedisFactory { 6 | 7 | public SimpleJedisFactory() { 8 | } 9 | 10 | @Override 11 | public Jedis newInstance(String hostname, int port) { 12 | return new Jedis(hostname, port); 13 | } 14 | 15 | } -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/storage/Bootstrap.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.storage; 2 | 3 | public enum Bootstrap { 4 | NOT_STARTED, CANNOT_CONNECT_FAIL, WARMUP_ERROR_FAIL, RETRIES_FAIL, EXPIRED_BOOTSTRAPTIME_FAIL, IN_SYNC_SUCCESS, 5 | } 6 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/storage/JedisUtils.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.storage; 2 | 3 | import com.netflix.config.DynamicIntProperty; 4 | import com.netflix.config.DynamicLongProperty; 5 | import com.netflix.config.DynamicPropertyFactory; 6 | import com.netflix.nfsidecar.utils.BoundedExponentialRetryCallable; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import redis.clients.jedis.Jedis; 11 | 12 | /** 13 | * Useful utilities to connect to storage or storage proxy via Jedis. 14 | * 15 | * @author Monal Daxini 16 | * @author ipapapa 17 | */ 18 | public class JedisUtils { 19 | private static final Logger logger = LoggerFactory.getLogger(JedisUtils.class); 20 | 21 | private static final DynamicLongProperty minRetryMs = DynamicPropertyFactory.getInstance() 22 | .getLongProperty("florida.storage.isAlive.retry.min.ms", 3000L); 23 | 24 | private static final DynamicLongProperty maxRetryMs = DynamicPropertyFactory.getInstance() 25 | .getLongProperty("florida.storage.isAlive.retry.max.ms", 30000L); 26 | 27 | private static final DynamicIntProperty jedisConnectTimeoutMs = DynamicPropertyFactory.getInstance() 28 | .getIntProperty("florida.storage.jedis.connect.timeout.ms", 30000); 29 | 30 | /** 31 | * The caller is responsible for invoking 32 | * {@link redis.clients.jedis.Jedis#disconnect()}. 33 | * 34 | * @return a Jedis object connected to the specified host and port. 35 | */ 36 | public static Jedis connect(final String host, final int port) { 37 | Jedis jedis; 38 | try { 39 | jedis = new Jedis(host, port, jedisConnectTimeoutMs.getValue()); 40 | jedis.connect(); 41 | return jedis; 42 | } catch (Exception e) { 43 | logger.warn("Unable to connect to host:" + host + " port: " + port); 44 | } 45 | 46 | return null; 47 | } 48 | 49 | /** 50 | * Sends a SETEX with an expire after 1 sec to Redis at the specified port 51 | * 52 | * @return an OK response if everything was written properly 53 | */ 54 | public static boolean isWritableWithRetry(final String host, final int port) { 55 | BoundedExponentialRetryCallable jedisRetryCallable = new BoundedExponentialRetryCallable() { 56 | Jedis jedis = null; 57 | 58 | @Override 59 | public Boolean retriableCall() throws Exception { 60 | jedis = connect(host, port); 61 | /* 62 | * check 1: write a SETEX key (single write) and auto-expire 63 | */ 64 | String status = jedis.setex("ignore_dyno", 1, "dynomite"); 65 | if (!status.equalsIgnoreCase("OK")) { 66 | jedis.disconnect(); 67 | return false; 68 | } 69 | return true; 70 | 71 | } 72 | 73 | @Override 74 | public void forEachExecution() { 75 | jedis.disconnect(); 76 | } 77 | }; 78 | 79 | jedisRetryCallable.setMin(minRetryMs.getValue()); 80 | jedisRetryCallable.setMax(maxRetryMs.getValue()); 81 | 82 | try { 83 | return jedisRetryCallable.call(); 84 | } catch (Exception e) { 85 | logger.warn(String.format("All retries to SETEX to host:%s port:%s failed.", host, port)); 86 | return false; 87 | } 88 | 89 | } 90 | 91 | /** 92 | * Sends a PING and INFO to Dynomite and Redis at the specified host and 93 | * port. 94 | * 95 | * @return true if a PONG or found "master" in the role, else false. 96 | */ 97 | public static boolean isAliveWithRetry(final String host, final int port) { 98 | 99 | BoundedExponentialRetryCallable jedisRetryCallable = new BoundedExponentialRetryCallable() { 100 | Jedis jedis = null; 101 | 102 | @Override 103 | public Boolean retriableCall() throws Exception { 104 | jedis = connect(host, port); 105 | /* check 1: perform a ping */ 106 | if (jedis.ping() == null) { 107 | jedis.disconnect(); 108 | return false; 109 | } 110 | jedis.disconnect(); 111 | return true; 112 | } 113 | 114 | @Override 115 | public void forEachExecution() { 116 | jedis.disconnect(); 117 | } 118 | }; 119 | 120 | jedisRetryCallable.setMin(minRetryMs.getValue()); 121 | jedisRetryCallable.setMax(maxRetryMs.getValue()); 122 | 123 | try { 124 | return jedisRetryCallable.call(); 125 | } catch (Exception e) { 126 | logger.warn(String.format("All retries to PING host:%s port:%s failed.", host, port)); 127 | return false; 128 | } 129 | 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/storage/MemcachedStorageProxy.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.storage; 2 | 3 | import java.io.IOException; 4 | 5 | public class MemcachedStorageProxy implements StorageProxy { 6 | 7 | private static final String DYNO_MEMCACHED = "memcached"; 8 | private static final int MEMCACHE_PORT = 11211; 9 | private static final String MEMCACHE_ADDRESS = "127.0.0.1"; 10 | 11 | private final String DEFAULT_MEMCACHED_START_SCRIPT = "/apps/memcached/bin/memcached"; 12 | private final String DEFAULT_MEMCACHED_STOP_SCRIPT = "/usr/bin/pkill memcached"; 13 | 14 | 15 | @Override 16 | public String getEngine() { 17 | return DYNO_MEMCACHED; 18 | } 19 | 20 | @Override 21 | public int getEngineNumber() { 22 | return 1; 23 | } 24 | 25 | @Override 26 | public boolean isAlive() { 27 | return false; 28 | } 29 | 30 | @Override 31 | public long getUptime() { 32 | return 0; 33 | } 34 | 35 | @Override 36 | public Bootstrap warmUpStorage(String[] peers) { 37 | return Bootstrap.IN_SYNC_SUCCESS; 38 | } 39 | 40 | @Override 41 | public boolean resetStorage() { 42 | return true; 43 | } 44 | 45 | @Override 46 | public boolean takeSnapshot() { 47 | return false; 48 | } 49 | 50 | @Override 51 | public boolean loadingData() { 52 | return false; 53 | } 54 | 55 | @Override 56 | public void stopPeerSync() { 57 | 58 | } 59 | 60 | @Override 61 | public void updateConfiguration() throws IOException { 62 | // TODO Auto-generated method stub 63 | 64 | } 65 | 66 | @Override 67 | public String getStartupScript() { 68 | return DEFAULT_MEMCACHED_START_SCRIPT; 69 | } 70 | 71 | @Override 72 | public String getStopScript() { 73 | return DEFAULT_MEMCACHED_STOP_SCRIPT; 74 | } 75 | 76 | @Override 77 | public String getIpAddress() { 78 | return MEMCACHE_ADDRESS; 79 | 80 | } 81 | 82 | @Override 83 | public int getPort() { 84 | return MEMCACHE_PORT; 85 | } 86 | 87 | @Override 88 | public long getStoreMaxMem() { 89 | return 0; 90 | } 91 | 92 | @Override 93 | public long getTotalAvailableSystemMemory() { 94 | return 0; 95 | } 96 | 97 | @Override 98 | public String getUnixPath() { return ""; } 99 | } 100 | -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/storage/StorageProcessManager.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.storage; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.File; 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.util.LinkedList; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | import org.apache.commons.lang.StringUtils; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import com.google.common.collect.Lists; 16 | import com.google.inject.Inject; 17 | import com.google.inject.Singleton; 18 | import com.netflix.dynomitemanager.config.InstanceState; 19 | import com.netflix.nfsidecar.utils.Sleeper; 20 | 21 | /** 22 | * Start or stop the storage engine, such as Redis or Memcached. 23 | */ 24 | @Singleton 25 | public class StorageProcessManager { 26 | private static final Logger logger = LoggerFactory.getLogger(StorageProcessManager.class); 27 | private static final String SUDO_STRING = "/usr/bin/sudo"; 28 | private static final int SCRIPT_EXECUTE_WAIT_TIME_MS = 5000; 29 | private final Sleeper sleeper; 30 | private final InstanceState instanceState; 31 | private final StorageProxy storageProxy; 32 | 33 | @Inject 34 | public StorageProcessManager(Sleeper sleeper, InstanceState instanceState, StorageProxy storageProxy) { 35 | this.sleeper = sleeper; 36 | this.instanceState = instanceState; 37 | this.storageProxy = storageProxy; 38 | } 39 | 40 | protected void setStorageEnv(Map env) { 41 | env.put("FLORIDA_STORAGE", String.valueOf(this.storageProxy.getEngine())); 42 | } 43 | 44 | /** 45 | * Start the storage engine (Redis, Memcached). 46 | * 47 | * @throws IOException 48 | */ 49 | public void start() throws IOException { 50 | logger.info(String.format("Starting Storage process")); 51 | ProcessBuilder startBuilder = process(getStartCommand()); 52 | setStorageEnv(startBuilder.environment()); 53 | Process starter = startBuilder.start(); 54 | 55 | try { 56 | sleeper.sleepQuietly(SCRIPT_EXECUTE_WAIT_TIME_MS); 57 | int code = starter.exitValue(); 58 | if (code == 0) { 59 | logger.info("Storage process has been started"); 60 | instanceState.setStorageProxyAlive(true); 61 | } else { 62 | logger.error("Unable to start Storage process. Error code: {}", code); 63 | } 64 | 65 | logProcessOutput(starter); 66 | } catch (Exception e) { 67 | logger.warn("Starting Storage process has an error", e); 68 | } 69 | } 70 | 71 | 72 | /** 73 | * A common class to initialize a ProcessBuilder 74 | * @param executeCommand 75 | * @return the process to start 76 | * @throws IOException 77 | */ 78 | private ProcessBuilder process(List executeCommand) throws IOException { 79 | List command = Lists.newArrayList(); 80 | if (!"root".equals(System.getProperty("user.name"))) { 81 | command.add(SUDO_STRING); 82 | command.add("-n"); 83 | command.add("-E"); 84 | } 85 | command.addAll(executeCommand); 86 | ProcessBuilder actionStorage = new ProcessBuilder(command); 87 | actionStorage.directory(new File("/")); 88 | actionStorage.redirectErrorStream(true); 89 | 90 | return actionStorage; 91 | } 92 | 93 | /** 94 | * Getting the start command 95 | * @return 96 | */ 97 | private List getStartCommand() { 98 | List startCmd = new LinkedList(); 99 | for (String param : storageProxy.getStartupScript().split(" ")) { 100 | if (StringUtils.isNotBlank(param)) 101 | startCmd.add(param); 102 | } 103 | return startCmd; 104 | } 105 | 106 | /** 107 | * Getting the stop command 108 | * @return 109 | */ 110 | private List getStopCommand() { 111 | List stopCmd = new LinkedList(); 112 | for (String param : storageProxy.getStopScript().split(" ")) { 113 | if (StringUtils.isNotBlank(param)) 114 | stopCmd.add(param); 115 | } 116 | return stopCmd; 117 | } 118 | 119 | private void logProcessOutput(Process p) { 120 | try { 121 | final String stdOut = readProcessStream(p.getInputStream()); 122 | final String stdErr = readProcessStream(p.getErrorStream()); 123 | logger.info("std_out: {}", stdOut); 124 | logger.info("std_err: {}", stdErr); 125 | } catch (IOException ioe) { 126 | logger.warn("Failed to read the std out/err streams", ioe); 127 | } 128 | } 129 | 130 | String readProcessStream(InputStream inputStream) throws IOException { 131 | final byte[] buffer = new byte[512]; 132 | final ByteArrayOutputStream baos = new ByteArrayOutputStream(buffer.length); 133 | int cnt; 134 | while ((cnt = inputStream.read(buffer)) != -1) 135 | baos.write(buffer, 0, cnt); 136 | return baos.toString(); 137 | } 138 | 139 | /** 140 | * Stop the storage engine (Redis, Memcached). 141 | * 142 | * @throws IOException 143 | */ 144 | public void stop() throws IOException { 145 | logger.info("Stopping storage process..."); 146 | ProcessBuilder stopBuilder = process(getStopCommand()); 147 | Process stopper = stopBuilder.start(); 148 | 149 | sleeper.sleepQuietly(SCRIPT_EXECUTE_WAIT_TIME_MS); 150 | try { 151 | int code = stopper.exitValue(); 152 | if (code == 0) { 153 | logger.info("Storage process has been stopped"); 154 | instanceState.setStorageProxyAlive(false); 155 | } else { 156 | logger.error("Unable to stop storage process. Error code: {}", code); 157 | logProcessOutput(stopper); 158 | } 159 | } catch (Exception e) { 160 | logger.warn("Could not shut down storage process correctly: ", e); 161 | } 162 | } 163 | } -------------------------------------------------------------------------------- /dynomitemanager-core/src/main/java/com/netflix/dynomitemanager/storage/StorageProxy.java: -------------------------------------------------------------------------------- 1 | package com.netflix.dynomitemanager.storage; 2 | 3 | import java.io.IOException; 4 | 5 | 6 | public interface StorageProxy { 7 | 8 | boolean isAlive(); 9 | 10 | long getUptime(); 11 | 12 | Bootstrap warmUpStorage(String[] peers); 13 | 14 | boolean resetStorage(); 15 | 16 | boolean takeSnapshot(); 17 | 18 | boolean loadingData(); 19 | 20 | void stopPeerSync(); 21 | 22 | String getEngine(); 23 | 24 | int getEngineNumber(); 25 | 26 | void updateConfiguration() throws IOException; 27 | 28 | String getStartupScript(); 29 | 30 | String getStopScript(); 31 | 32 | String getIpAddress(); 33 | 34 | int getPort(); 35 | 36 | String getUnixPath(); 37 | 38 | long getStoreMaxMem(); 39 | 40 | long getTotalAvailableSystemMemory(); 41 | 42 | 43 | } -------------------------------------------------------------------------------- /dynomitemanager-web/.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | /build/ 3 | -------------------------------------------------------------------------------- /dynomitemanager-web/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'war' 2 | apply plugin: "org.gretty" 3 | apply plugin: 'jacoco' 4 | 5 | 6 | // Gretty allows us to run tomcat from the command line using the "tomcatRun" task. 7 | if (!gradle.startParameter.taskRequests.isEmpty()) { 8 | Properties development_properties = new Properties() 9 | development_properties.load(new FileInputStream(file("src/main/resources/laptop.properties"))) 10 | gretty { 11 | jacocoEnabled = false 12 | httpPort = development_properties.getProperty("netflix.appinfo.port", "8080").toInteger() 13 | contextPath = '/' 14 | servletContainer = 'tomcat85' 15 | systemProperties = development_properties 16 | scanDirs = ['**/src/main/resources/**'] 17 | scanDependencies = true 18 | // More properties can be found here: 19 | // http://akhikhl.github.io/gretty-doc/Gretty-configuration.html 20 | } 21 | // Required exclusions to work around a bug in gretty: log4j-over-slf4j is accidentally included 22 | configurations.grettyRunnerTomcat85 { 23 | exclude group: 'org.slf4j', module: 'log4j-over-slf4j' 24 | } 25 | } 26 | 27 | dependencies { 28 | implementation project(':dynomitemanager-common') 29 | implementation project(':dynomitemanager-core') 30 | 31 | implementation 'com.google.inject.extensions:guice-servlet:4.0' 32 | implementation 'com.google.inject:guice:4.0' 33 | implementation group: 'com.owlike', name: 'genson', version: '1.4' 34 | 35 | implementation "com.netflix.runtime:health-guice:latest.release" 36 | 37 | // Governator wires up all of our dependencies in dependency injection fashion. 38 | implementation "com.netflix.governator:governator-core:latest.release" 39 | implementation "com.netflix.governator:governator-servlet:latest.release" 40 | implementation "com.netflix.governator:governator-jersey:latest.release" 41 | implementation "com.sun.jersey.contribs:jersey-guice:1.19" 42 | 43 | // This is here because it's the only reliable way to make IDEs pick up the proper dependencies when running 44 | // a main() function while also not packaging the dependencies to be deployed on EC2. 45 | if (gradle.startParameter.taskRequests.isEmpty()) { 46 | implementation ("com.netflix.governator:governator-jetty:latest.release") { transitive = false } 47 | implementation "org.eclipse.jetty:jetty-servlet:latest.release" 48 | implementation "org.eclipse.jetty:jetty-webapp:latest.release" 49 | implementation "javax.servlet:javax.servlet-api:latest.release" 50 | } else { 51 | compileOnly("com.netflix.governator:governator-jetty:latest.release") { transitive = false } 52 | compileOnly "org.eclipse.jetty:jetty-servlet:latest.release" 53 | compileOnly "org.eclipse.jetty:jetty-webapp:latest.release" 54 | compileOnly "javax.servlet:javax.servlet-api:latest.release" 55 | } 56 | 57 | 58 | } 59 | 60 | war.archiveFileName = "dynomite-manager.war" 61 | -------------------------------------------------------------------------------- /dynomitemanager-web/src/main/java/com/netflix/florida/startup/Florida.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.florida.startup; 14 | 15 | import com.google.inject.Injector; 16 | import com.netflix.archaius.guice.ArchaiusModule; 17 | import com.netflix.governator.InjectorBuilder; 18 | import com.netflix.governator.guice.jetty.Archaius2JettyModule; 19 | import com.netflix.governator.guice.servlet.WebApplicationInitializer; 20 | 21 | /** 22 | * The "main" class that boots up the service. When it's deployed within a servlet container such 23 | * as Tomcat, only the createInjector() is called. For local testing one simply invokes the 24 | * main() method as if running a normal Java app. 25 | * 26 | * @author This file is auto-generated by runtime@netflix.com. Feel free to modify. 27 | */ 28 | public class Florida implements WebApplicationInitializer { 29 | 30 | public static void main(String[] args) throws Exception { 31 | InjectorBuilder.fromModules( 32 | new FloridaModule(), 33 | new Archaius2JettyModule(), 34 | new ArchaiusModule() { 35 | @Override 36 | protected void configureArchaius() { 37 | bindApplicationConfigurationOverrideResource("laptop"); 38 | } 39 | }).createInjector().awaitTermination(); 40 | } 41 | 42 | @Override 43 | public Injector createInjector() { 44 | return InjectorBuilder.fromModules(new FloridaModule()).createInjector(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dynomitemanager-web/src/main/java/com/netflix/florida/startup/JerseyModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with 5 | * the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on 10 | * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the 11 | * specific language governing permissions and limitations under the License. 12 | */ 13 | package com.netflix.florida.startup; 14 | 15 | import java.io.IOException; 16 | import java.util.HashMap; 17 | import java.util.Map; 18 | 19 | import javax.inject.Singleton; 20 | import javax.servlet.Filter; 21 | import javax.servlet.FilterChain; 22 | import javax.servlet.FilterConfig; 23 | import javax.servlet.ServletException; 24 | import javax.servlet.ServletRequest; 25 | import javax.servlet.ServletResponse; 26 | import javax.servlet.http.HttpServletResponse; 27 | 28 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 29 | import com.fasterxml.jackson.databind.DeserializationFeature; 30 | import com.fasterxml.jackson.databind.ObjectMapper; 31 | import com.google.inject.Provides; 32 | import com.sun.jersey.api.core.PackagesResourceConfig; 33 | import com.sun.jersey.api.core.ResourceConfig; 34 | import com.sun.jersey.guice.JerseyServletModule; 35 | import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; 36 | 37 | /** 38 | * 39 | * @author Viren 40 | * 41 | */ 42 | public final class JerseyModule extends JerseyServletModule { 43 | 44 | @Override 45 | protected void configureServlets() { 46 | 47 | filter("/*").through(apiOriginFilter()); 48 | 49 | Map jerseyParams = new HashMap<>(); 50 | jerseyParams.put("com.sun.jersey.config.feature.FilterForwardOn404", "true"); 51 | jerseyParams.put("com.sun.jersey.config.property.WebPageContentRegex", 52 | "/(((webjars|api-docs|swagger-ui/docs|manage)/.*)|(favicon\\.ico))"); 53 | jerseyParams.put(PackagesResourceConfig.PROPERTY_PACKAGES, 54 | "com.netflix.dynomitemanager.resources;io.swagger.jaxrs.json;io.swagger.jaxrs.listing"); 55 | jerseyParams.put(ResourceConfig.FEATURE_DISABLE_WADL, "false"); 56 | serve("/api/*").with(GuiceContainer.class, jerseyParams); 57 | } 58 | 59 | @Provides 60 | @Singleton 61 | public ObjectMapper objectMapper() { 62 | final ObjectMapper om = new ObjectMapper(); 63 | om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 64 | om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); 65 | om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); 66 | om.setSerializationInclusion(Include.NON_NULL); 67 | om.setSerializationInclusion(Include.NON_EMPTY); 68 | return om; 69 | } 70 | 71 | 72 | @Provides 73 | @Singleton 74 | public Filter apiOriginFilter() { 75 | return new Filter() { 76 | 77 | @Override 78 | public void init(FilterConfig filterConfig) throws ServletException { 79 | } 80 | 81 | @Override 82 | public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) 83 | throws IOException, ServletException { 84 | HttpServletResponse res = (HttpServletResponse) response; 85 | if (!res.containsHeader("Access-Control-Allow-Origin")) { 86 | res.setHeader("Access-Control-Allow-Origin", "*"); 87 | } 88 | res.addHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT"); 89 | res.addHeader("Access-Control-Allow-Headers", "Content-Type, api_key, Authorization"); 90 | 91 | chain.doFilter(request, response); 92 | } 93 | 94 | @Override 95 | public void destroy() { 96 | } 97 | 98 | }; 99 | } 100 | 101 | @Override 102 | public boolean equals(Object obj) { 103 | return obj != null && getClass().equals(obj.getClass()); 104 | } 105 | 106 | @Override 107 | public int hashCode() { 108 | return getClass().hashCode(); 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /dynomitemanager-web/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | ## Standard logging configuration. 2 | # By default Keystone/Chukwa is not enabled as an appender, please enable it if you wish. 3 | # Chukwa is our Data Pipeline logger that you can write to using the Keystone API (go/keystone). 4 | # To enable, replace the following line with this: 5 | # log4j.rootCategory=INFO,CONSOLE,CUSTOM_EVENTS,REQUEST_BASED_LOGGING 6 | 7 | log4j.rootCategory=INFO,CONSOLE 8 | 9 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 10 | log4j.appender.CONSOLE.layout.ConversionPattern=%d %-5p %X{requestId} %C:%L [%t] [%M] %m%n 11 | log4j.appender.CONSOLE.layout=com.netflix.logging.log4jAdapter.NFPatternLayout 12 | 13 | log4j.appender.CHUKWA=com.netflix.chukwa.client.log4j.ChukwaLog4jStreamingAppender 14 | log4j.appender.CHUKWA.layout.ConversionPattern=%d %-5p %C:%L [%t] [%M] %m%n 15 | log4j.appender.CHUKWA.layout=com.netflix.logging.log4jAdapter.NFPatternLayout 16 | 17 | log4j.appender.FILE=org.apache.log4j.FileAppender 18 | log4j.appender.FILE.layout.ConversionPattern=%d %-5p %X{requestId} %C:%L [%t] [%M] %m%n 19 | log4j.appender.FILE.layout=com.netflix.logging.log4jAdapter.NFPatternLayout 20 | 21 | log4j.appender.CUSTOM_EVENTS=com.netflix.logging.CustomEventsMessageAppender 22 | log4j.appender.CUSTOM_EVENTS.layout.ConversionPattern=%d %-5p %C:%L [%t] [%M] %m%n 23 | log4j.appender.CUSTOM_EVENTS.layout=com.netflix.logging.log4jAdapter.NFPatternLayout 24 | log4j.appender.CUSTOM_EVENTS.threshold=WARN 25 | 26 | log4j.appender.REQUEST_BASED_LOGGING=com.netflix.logging.RequestLevelLoggingEventBusAppender 27 | log4j.appender.REQUEST_BASED_LOGGING.layout.ConversionPattern=%d %-5p %C:%L [%t] [%M] %m%n 28 | log4j.appender.REQUEST_BASED_LOGGING.layout=com.netflix.logging.log4jAdapter.NFPatternLayout 29 | 30 | log4j.appender.FILE_DETAILED=org.apache.log4j.ConsoleAppender 31 | log4j.appender.FILE_DETAILED.layout=com.netflix.logging.log4jAdapter.NFPatternLayout 32 | 33 | ## Quieting noisy 3rd party and legacy Netflix platform loggers by default. 34 | log4j.logger.httpclient=WARN 35 | log4j.logger.com.netflix.discovery=WARN 36 | log4j.logger.com.netflix.config=WARN 37 | log4j.logger.com.netflix.monitoring=WARN 38 | log4j.logger.com.netflix.server.base.epic.RequestStats=OFF,CONSOLE 39 | log4j.logger.com.netflix.server.base.RequestStats=WARN 40 | log4j.logger.com.netflix.eventbus.impl=WARN 41 | log4j.logger.com.netflix.atlas.plugin=WARN 42 | 43 | ## Our service logger configurations. 44 | log4j.logger.com.netflix.florida=INFO 45 | 46 | # Cassandra loggers 47 | log4j.logger.com.datastax.driver=ERROR 48 | log4j.logger.com.netflix.aeneas=ERROR 49 | 50 | ### Netflix Application configurations required for deployment. 51 | 52 | # AWS environment: test or prod? The default if not set is "test". 53 | # On deployment it's set to the "baked" NETFLIX_ENVIRONMENT env variable. Please don't modify. 54 | netflix.environment=${NETFLIX_ENVIRONMENT} 55 | # AWS region. We default if not set is "us-east-1" which is where we typically test. 56 | # On deployment it's set to the "baked" EC2_REGION env variable. Please don't modify. 57 | netflix.region=${EC2_REGION} 58 | # The netflix stack your application is running in. Default is no stack. 59 | netflix.stack=${NETFLIX_STACK} 60 | # The name of your service. This name will also match your Spinnaker generated deployment name. 61 | # That's the name that you will be known in Discovery. If you modify it you will also need to 62 | # modify your Spinnaker config. 63 | netflix.appinfo.name=${NETFLIX_APP:florida} 64 | # VIP address has been traditionally used as a lookup "rulebook" for a service that wants 65 | # to discover another service. Below you are saying to anyone looking for your service: 66 | # "You can find my instances first at the same stack you're deployed at 67 | # but if I'm not deployed there then look at the general service name under the environment". 68 | # Typical stack names are "test", "dev", "staging". They are decided offline between service owners. 69 | netflix.appinfo.vipAddress=${netflix.appinfo.name}${NETFLIX_STACK:} 70 | 71 | ## Application info 72 | netflix.appinfo.statusPageUrlPath=/REST/v1/admin/Status 73 | netflix.appinfo.homePageUrlPath=/REST/v1/admin/Status 74 | netflix.appinfo.healthCheckUrlPath=/healthcheck 75 | netflix.appinfo.port=8080 76 | 77 | ## Standard URLs that Discovery will add to your service metadata and make it easy to check up on 78 | ## deployed instances. 79 | # Service home page. You can put anything you like there and rename the path. 80 | netflix.appinfo.homePageUrlPath=/index.html 81 | # URL to check if an instance is healthy. It is tied to your healthcheck indicators. 82 | #The path is expected to be "/healthcheck". 83 | # Please don't modify. 84 | netflix.appinfo.healthCheckUrlPath=/healthcheck 85 | 86 | ## Jersey configurations 87 | # Let's add our endpoints that are in the /REST path. 88 | com.sun.jersey.config.property.packages=com.netflix.florida 89 | 90 | ### Our own service configuration 91 | 92 | 93 | 94 | ### Cassandra Aeneas Cluster config. 95 | 96 | # Properties must use a prefix that matches our cluster name in the CassandraModule. 97 | # As you add more clusters you can have multiple sets of properties prefixed by the cluster name. 98 | # Here is an example property that you can uncomment and try: 99 | # cass_sandbox.aeneas.nativePort=7104 100 | # For a full list of properties please visit: http://go/aeneas 101 | -------------------------------------------------------------------------------- /dynomitemanager-web/src/main/resources/laptop.properties: -------------------------------------------------------------------------------- 1 | # Do not try to register with discovery since we don't want others to discover our laptop. 2 | netflix.discovery.registration.enabled=false 3 | # Since we are not on AWS we don't have an instance ID and 4 | # we don't want Eureka to check for that and fail initialization. 5 | netflix.appinfo.validateInstanceId=false 6 | # Since we are not in AWS let's not make a call to EC2 to ask for EC2 node specific info. 7 | netflix.appinfo.doNotInitWithAmazonInfo=true 8 | 9 | netflix.atlas.plugin.enabled=false 10 | 11 | # These settings are required to prevent AtlasMainPoller NullPointerException when running locally 12 | netflix.atlas.plugin.sendToEpic=false 13 | epic.plugin.enabled=false 14 | 15 | ## Application info 16 | netflix.appinfo.statusPageUrlPath=/REST/v1/admin/Status 17 | netflix.appinfo.homePageUrlPath=/REST/v1/admin/Status 18 | netflix.appinfo.healthCheckUrlPath=/healthcheck 19 | netflix.appinfo.port=8080 20 | 21 | # Needed for running locally behind VPN 22 | swagger.hostname=localhost 23 | 24 | governator.jetty.embedded.port=${netflix.appinfo.port} 25 | 26 | -------------------------------------------------------------------------------- /dynomitemanager-web/src/main/webapp/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/dynomite-manager/f92b9e55e9c185fb5196dcbcbb982df93fe5d1bb/dynomitemanager-web/src/main/webapp/favicon.ico -------------------------------------------------------------------------------- /dynomitemanager-web/src/main/webapp/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Netflix florida 6 | 7 | 21 | 22 | 23 | 24 |

25 |
26 | Netflix florida 27 |
28 |
29 | 30 | 31 | -------------------------------------------------------------------------------- /dynomitemanager-web/src/smokeTest/java/com/netflix/florida/SmokeTest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.florida; 2 | 3 | import static io.restassured.RestAssured.*; 4 | import static org.hamcrest.Matchers.*; 5 | 6 | import java.io.IOException; 7 | 8 | import javax.inject.Named; 9 | 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | 13 | import com.google.inject.Inject; 14 | import com.netflix.archaius.test.TestPropertyOverride; 15 | import com.netflix.governator.guice.jetty.Archaius2JettyModule; 16 | import com.netflix.governator.guice.test.ModulesForTesting; 17 | import com.netflix.governator.guice.test.junit4.GovernatorJunit4ClassRunner; 18 | import com.netflix.florida.startup.FloridaModule; 19 | 20 | 21 | /** 22 | * This is the one and only one integration test for the whole service. 23 | * We leverage the governator-test-junit library to run the test for us. 24 | * 25 | * We don't do any deep testing here. Our unit tests are supposed to do that. We keep this test simple and we check that 26 | * everything is wired well together and that all of our endpoints are up. Testing the actual content returned 27 | * or that our POSTs work etc. is done in the unit tests. Our unit test do not reach out to the network so it's faster 28 | * to test all possible input/output scenarios there and gain confidence that our business logic works. 29 | * 30 | * @author This file is auto-generated by runtime@netflix.com. Feel free to modify. 31 | */ 32 | public class SmokeTest { 33 | 34 | @Test 35 | public void testRestEndpoint() throws IOException { 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /dynomitemanager-web/src/test/testing-guidelines-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/dynomite-manager/f92b9e55e9c185fb5196dcbcbb982df93fe5d1bb/dynomitemanager-web/src/test/testing-guidelines-diagram.png -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/dynomite-manager/f92b9e55e9c185fb5196dcbcbb982df93fe5d1bb/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name='dynomitemanager' 2 | include 'dynomitemanager-core','dynomitemanager-common','dynomitemanager-web' 3 | --------------------------------------------------------------------------------