├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── build.gradle ├── cache ├── build.gradle └── src │ ├── dist │ └── config │ │ ├── development.yml │ │ └── standalone.yml │ ├── main │ ├── java │ │ └── com │ │ │ └── uber │ │ │ └── buckcache │ │ │ ├── AuthenticationConfig.java │ │ │ ├── BuckCacheApplication.java │ │ │ ├── BuckCacheConfiguration.java │ │ │ ├── CacheInstanceMode.java │ │ │ ├── IgniteConfig.java │ │ │ ├── StatsdConfig.java │ │ │ ├── auth │ │ │ └── HttpHeaderAuthFilter.java │ │ │ ├── datastore │ │ │ ├── CacheAdapter.java │ │ │ ├── CacheEntry.java │ │ │ ├── DataStoreProvider.java │ │ │ ├── DataStoreProviderConfig.java │ │ │ ├── DataStoreProviderFactory.java │ │ │ ├── exceptions │ │ │ │ ├── CacheException.java │ │ │ │ ├── DatastoreUnavailableException.java │ │ │ │ └── EntryNotFoundException.java │ │ │ └── impl │ │ │ │ └── ignite │ │ │ │ ├── IgniteConfigurationBuilder.java │ │ │ │ ├── IgniteConstants.java │ │ │ │ ├── IgniteDataStoreProvider.java │ │ │ │ ├── IgniteInstance.java │ │ │ │ └── LocalCacheEventListener.java │ │ │ ├── health │ │ │ └── DataStoreHealthChecker.java │ │ │ ├── resources │ │ │ ├── RootResource.java │ │ │ └── buckcache │ │ │ │ ├── BuckCacheResource.java │ │ │ │ ├── HealthResource.java │ │ │ │ └── PutCacheEntry.java │ │ │ └── utils │ │ │ ├── BytesRateLimiter.java │ │ │ ├── MetricsRegistry.java │ │ │ ├── StatsDClient.java │ │ │ └── StatsDClientFactory.java │ └── resources │ │ ├── banner.txt │ │ └── log4j.properties │ └── test │ ├── java │ └── com │ │ └── uber │ │ └── buckcache │ │ ├── auth │ │ └── HttpHeaderAuthFilterTest.java │ │ ├── datastore │ │ └── impl │ │ │ └── ignite │ │ │ └── IgniteConfigurationBuilderTest.java │ │ └── integration │ │ ├── SingleServerIntegrationTest.java │ │ └── TestUtils.java │ └── resources │ ├── cache_data │ ├── 1473551747919 │ ├── 1473551747939 │ ├── 1473551747950 │ ├── 1473551747956 │ ├── 1473551747961 │ ├── 1473551747965 │ ├── 1473551747969 │ ├── 1473551747973 │ ├── 1473551747980 │ ├── 1473551747983 │ ├── 1473551747987 │ ├── 1473551747991 │ ├── 1473551748673 │ ├── 1473551748681 │ ├── 1473551748688 │ ├── 1473551748707 │ ├── 1473551748733 │ ├── 1473551748747 │ ├── 1473551748787 │ ├── 1473551748849 │ ├── 1473551748852 │ ├── 1473551748856 │ ├── 1473551748863 │ ├── 1473551749391 │ ├── 1473551749402 │ ├── 1473551749410 │ ├── 1473551749414 │ ├── 1473551749423 │ ├── 1473551749428 │ ├── 1473551749433 │ ├── 1473551749436 │ ├── 1473551749440 │ ├── 1473551749443 │ ├── 1473551749448 │ ├── 1473551749463 │ ├── 1473551749467 │ ├── 1473551749470 │ ├── 1473551749473 │ ├── 1473551749479 │ ├── 1473551749482 │ ├── 1473551749485 │ ├── 1473551749490 │ ├── 1473551749529 │ ├── 1473551749533 │ ├── 1473551749537 │ ├── 1473551749540 │ ├── 1473551749544 │ ├── 1473551749550 │ ├── 1473551749553 │ ├── 1473551749557 │ ├── 1473551749560 │ ├── 1473551749566 │ ├── 1473551749571 │ ├── 1473551749575 │ ├── 1473551749578 │ ├── 1473551749582 │ ├── 1473551749585 │ ├── 1473551749589 │ ├── 1473551749595 │ ├── 1473551749598 │ ├── 1473551749602 │ ├── 1473551749605 │ ├── 1473551749610 │ ├── 1473551749655 │ ├── 1473551749953 │ ├── 1473551749957 │ ├── 1473551749965 │ ├── 1473551749972 │ ├── 1473551749977 │ ├── 1473551749982 │ ├── 1473551749991 │ ├── 1473551749994 │ ├── 1473551749997 │ ├── 1473551750001 │ ├── 1473551750004 │ ├── 1473551750008 │ ├── 1473551750012 │ ├── 1473551750016 │ ├── 1473551750019 │ ├── 1473551750025 │ ├── 1473551750033 │ ├── 1473551750037 │ ├── 1473551750048 │ ├── 1473551750051 │ ├── 1473551750056 │ ├── 1473551750061 │ ├── 1473551750066 │ ├── 1473551750082 │ ├── 1473551750086 │ ├── 1473551750097 │ ├── 1473551750116 │ ├── 1473551750130 │ ├── 1473551750145 │ ├── 1473551750153 │ ├── 1473551750222 │ ├── 1473551750431 │ ├── 1473551750451 │ ├── 1473551750483 │ └── 1473551750498 │ ├── config │ └── test-server-1.yml │ └── log4j.properties ├── classes └── production │ └── cache │ └── banner.txt ├── dependencies.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── run_buck_cache_client.sh └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | local.properties 3 | .idea 4 | *.iml 5 | build 6 | .DS_Store 7 | checkstyle*.xml 8 | configuration.yml 9 | target/ 10 | 11 | .classpath 12 | .settings/ 13 | .loadpath 14 | 15 | 16 | */.classpath 17 | */.gitignore 18 | */.project 19 | */.settings/ 20 | 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8 2 | 3 | MAINTAINER Dhaval Patel dhaval@uber.com; 4 | 5 | RUN git clone https://github.com/uber/buck-http-cache.git 6 | 7 | WORKDIR buck-http-cache 8 | 9 | RUN ./run_buck_cache_client.sh standalone 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Uber Technologies, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Buck HTTP Cache As a service. 2 | 3 | （This project is deprecated and not maintained any more.） 4 | 5 | An Implementation of [Buck's HTTP Cache API](https://buckbuild.com/concept/http_cache_api.html) as a distributed cache service. 6 | 7 | ## Quick start 8 | 9 | ### Standalone mode 10 | 11 | Standalone mode starts this in a single node mode. For most usecases, this should work. Lets say you only have 1 project, with tens of developers and a few CI tasks, then this setup should be more than enough. 12 | 13 | #### From Source 14 | 15 | ```bash 16 | # make sure that JAVA_HOME is set. (please use jdk8) 17 | git clone https://github.com/uber/buck-http-cache.git 18 | cd buck-http-cache 19 | ./run_buck_cache_client.sh development 20 | ``` 21 | 22 | #### From Distribution 23 | 24 | ```bash 25 | # make sure that JAVA_HOME is set. (please use jdk8) 26 | wget https://github.com/uber/buck-http-cache/raw/dist/releases/cache-1.0.0.zip 27 | unzip cache-1.0.0.zip 28 | ./bin/cache server config/development.yml 29 | ``` 30 | 31 | ### Cluster mode 32 | If you have many users, projects that will use the cache, then chances are that standalone mode won't work for you. In this case you will need to setup a distributed cache. Lets say you want to deploy the cache to three machines (IP1, IP2, IP3). Update config file standalone.yml and add the following. 33 | 34 | ```yml 35 | multicastPort: 6734 36 | hostIPs: 37 | - 38 | - 39 | - 40 | ``` 41 | 42 | Now you are ready to start the cache in all three hosts. The data will be equally distributed across all three nodes, and you can query any node to fetch the data. Each node acts as a broker and as a in-memory data base. 43 | We will enable zookeeper based node discovery soon. At which point we won't have to manually add IPs to the config. Just pointing to a common zk cluster should do the trick. 44 | 45 | ## Usage 46 | 47 | The cache server by default runs on port `6457`. Under the hood it uses [Apache Ignite](https://ignite.apache.org/) as the cache data grid. In order to use this http cache in your project, add the following to your `buckconfig`: 48 | 49 | ```ini 50 | [cache] 51 | mode = http 52 | http_url = http://your-server-address:6457 53 | http_mode = readwrite 54 | ``` 55 | 56 | This is the most basic way of starting the cache locally. For additional options, see [buckconfig](https://buckbuild.com/concept/buckconfig.html#cache). This works in standlone mode. If you are running the cache in cluster mode, then you may want to setup a loadbalancer in front of the nodes. 57 | 58 | As an alternative, if you use [Okbuck](https://github.com/uber/okbuck) to build your gradle project with buck, you can modify `buckw` to randomly choose a server on every invocation 59 | 60 | ```bash 61 | CACHE_SERVER=$[ (( $RANDOM % 3 )) + 1 ] 62 | CACHE_URL="buck-cache-server${CACHE_SERVER}:6457" 63 | EXTRA_BUCK_CONFIG="--config cache.http_url=http://${CACHE_URL}" 64 | exec "$BUCK_BINARY" "$@" $EXTRA_BUCK_CONFIG 65 | ``` 66 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | jcenter() 4 | } 5 | 6 | apply from: rootProject.file('dependencies.gradle') 7 | 8 | dependencies { 9 | classpath deps.build.oneJar 10 | classpath deps.build.cobertura 11 | } 12 | } 13 | 14 | allprojects { project -> 15 | project.apply from: rootProject.file('dependencies.gradle') 16 | repositories { 17 | jcenter() 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /cache/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'gradle-one-jar' 3 | apply plugin: 'distribution' 4 | apply plugin: 'application' 5 | apply plugin: 'eclipse' 6 | apply plugin: 'cobertura' 7 | 8 | group = 'com.uber' 9 | version = '1.0.0' 10 | 11 | sourceCompatibility = 1.8 12 | targetCompatibility = 1.8 13 | 14 | mainClassName = 'com.uber.buckcache.BuckCacheApplication' 15 | 16 | dependencies { 17 | compile deps.external.dropwizard 18 | compile deps.external.dropwizardAuth 19 | compile deps.external.ignite 20 | compile deps.external.junit 21 | compile deps.external.httpClient 22 | compile deps.external.jackson 23 | compile deps.external.uuidGen 24 | compile deps.external.httpCore 25 | compile deps.external.statsdClient 26 | compile deps.external.dns 27 | testCompile 'org.mockito:mockito-all:1.9.5' 28 | } 29 | 30 | task distJar(type: OneJar) { 31 | mainClass = mainClassName 32 | additionalDir = file('src/main/resources/') 33 | } 34 | 35 | artifacts { 36 | distJar 37 | } 38 | 39 | build.dependsOn distJar 40 | -------------------------------------------------------------------------------- /cache/src/dist/config/development.yml: -------------------------------------------------------------------------------- 1 | server: 2 | applicationConnectors: 3 | - type: http 4 | port: 6457 5 | adminConnectors: 6 | - type: http 7 | port: 6735 8 | 9 | mode: DUAL 10 | 11 | # per instance, per second payload throttle limit 12 | # available units, b,k,m & g 13 | throttleLimit: 500m 14 | 15 | storeProviderKlass: com.uber.buckcache.datastore.impl.ignite.IgniteDataStoreProvider 16 | 17 | storeProviderConfig: 18 | config: 19 | multicastIP: 228.10.10.157 20 | multicastPort: 6734 21 | hostIPs: 22 | - 127.0.0.1 23 | cacheMode: PARTITIONED 24 | cacheBackupCount: 1 25 | expirationTimeUnit: DAYS 26 | expirationTimeValue: 7 27 | atomicSequenceReserveSize: 10000 28 | offHeapStorageSize: 40g # 40GB 29 | cacheMemoryMode : OFFHEAP_VALUES #https://ignite.apache.org/releases/1.9.0/javadoc/ 30 | 31 | authenticationConfig: 32 | tokens: 33 | 34 | statsd: 35 | enabled: false 36 | 37 | -------------------------------------------------------------------------------- /cache/src/dist/config/standalone.yml: -------------------------------------------------------------------------------- 1 | server: 2 | applicationConnectors: 3 | - type: http 4 | port: 6457 5 | adminConnectors: 6 | - type: http 7 | port: 6735 8 | 9 | mode: DUAL 10 | 11 | # per instance, per second payload throttle limit 12 | # available units, b,k,m & g 13 | throttleLimit: 500m 14 | 15 | storeProviderKlass: com.uber.buckcache.datastore.impl.ignite.IgniteDataStoreProvider 16 | 17 | storeProviderConfig: 18 | config: 19 | multicastIP: 228.10.10.157 20 | multicastPort: 6734 21 | hostIPs: 22 | dnsLookupAddress: 23 | cacheMode: PARTITIONED 24 | cacheBackupCount: 1 25 | expirationTimeUnit: DAYS 26 | expirationTimeValue: 2 27 | atomicSequenceReserveSize: 10000 28 | offHeapStorageSize: 40g # 40GB 29 | cacheMemoryMode : OFFHEAP_VALUES #https://ignite.apache.org/releases/1.9.0/javadoc/ 30 | 31 | authenticationConfig: 32 | tokens: 33 | 34 | statsd: 35 | enabled: true 36 | host: localhost 37 | port: 4744 38 | prefix: buck-cache-client.prod 39 | sampleRate: 1.0 40 | 41 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/AuthenticationConfig.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache; 2 | 3 | import java.util.List; 4 | import javax.annotation.Nonnull; 5 | 6 | import com.fasterxml.jackson.annotation.JsonCreator; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | 9 | public class AuthenticationConfig { 10 | @Nonnull 11 | private final List tokens; 12 | 13 | @JsonCreator 14 | public AuthenticationConfig(@Nonnull @JsonProperty("tokens") List tokens) { 15 | this.tokens = tokens; 16 | } 17 | 18 | public List getTokens() { 19 | return tokens; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/BuckCacheApplication.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache; 2 | 3 | import com.uber.buckcache.auth.HttpHeaderAuthFilter; 4 | import io.dropwizard.auth.AuthDynamicFeature; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import com.uber.buckcache.datastore.DataStoreProvider; 9 | import com.uber.buckcache.datastore.DataStoreProviderFactory; 10 | import com.uber.buckcache.health.DataStoreHealthChecker; 11 | import com.uber.buckcache.resources.RootResource; 12 | import com.uber.buckcache.resources.buckcache.BuckCacheResource; 13 | import com.uber.buckcache.resources.buckcache.HealthResource; 14 | import com.uber.buckcache.utils.BytesRateLimiter; 15 | import com.uber.buckcache.utils.StatsDClient; 16 | 17 | import io.dropwizard.Application; 18 | import io.dropwizard.setup.Bootstrap; 19 | import io.dropwizard.setup.Environment; 20 | 21 | /** 22 | * A REST service that implements Buck's HTTP Cache API 23 | * (https://buckbuild.com/concept/http_cache_api.html) 24 | * backed by Ignite. 25 | */ 26 | public class BuckCacheApplication extends Application { 27 | 28 | private static final Logger logger = LoggerFactory.getLogger(BuckCacheApplication.class); 29 | 30 | private static final String APPLICATION_NAME = "com.Uber.BuckCache"; 31 | private static final String IGNITE_HEALTH_CHECK_NAME = "healthCheck.Ignite"; 32 | 33 | public static void main(String[] args) throws Exception { 34 | new BuckCacheApplication().run(args); 35 | } 36 | 37 | @Override 38 | public String getName() { 39 | return APPLICATION_NAME; 40 | } 41 | 42 | @Override 43 | public void initialize(Bootstrap bootstrap) { 44 | 45 | } 46 | 47 | @Override 48 | public void run(BuckCacheConfiguration configuration, Environment environment) throws Exception { 49 | logger.info("**********************************************"); 50 | logger.info("starting with config : {}", configuration); 51 | logger.info("**********************************************"); 52 | 53 | StatsDClient.init(configuration.getStatsd()); 54 | 55 | BytesRateLimiter rateLimiter = new BytesRateLimiter(configuration.getThrottleLimit()); 56 | DataStoreProvider dataStoreProvider = DataStoreProviderFactory.get(configuration.getStoreProviderKlass(), 57 | configuration.getStoreProviderConfig(), configuration.getMode()); 58 | 59 | dataStoreProvider.start(); 60 | environment.lifecycle().manage(dataStoreProvider); 61 | 62 | environment.healthChecks().register(IGNITE_HEALTH_CHECK_NAME, new DataStoreHealthChecker(dataStoreProvider)); 63 | 64 | if (configuration.getMode() != CacheInstanceMode.DATABASE_ONLY) { 65 | 66 | final BuckCacheResource buckCacheResource = new BuckCacheResource(dataStoreProvider, rateLimiter); 67 | environment.jersey().register(buckCacheResource); 68 | environment.jersey().register(new HealthResource()); 69 | environment.jersey().register(BuckCacheResource.CacheResultBodyReader.class); 70 | environment.jersey().register(BuckCacheResource.CacheResultBodyWriter.class); 71 | 72 | final RootResource rootResource = new RootResource(); 73 | environment.jersey().register(rootResource); 74 | 75 | environment.jersey().register(new AuthDynamicFeature( 76 | new HttpHeaderAuthFilter( 77 | configuration.getAuthenticationConfig().getTokens()))); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/BuckCacheConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | import com.fasterxml.jackson.annotation.JsonCreator; 6 | import org.apache.commons.lang3.builder.ToStringBuilder; 7 | 8 | import com.fasterxml.jackson.annotation.JsonProperty; 9 | import com.uber.buckcache.datastore.DataStoreProviderConfig; 10 | 11 | import io.dropwizard.Configuration; 12 | 13 | public final class BuckCacheConfiguration extends Configuration { 14 | 15 | @Nonnull 16 | private final CacheInstanceMode mode; 17 | @Nonnull 18 | private final StatsdConfig statsd; 19 | @Nonnull 20 | private final String throttleLimit; 21 | @Nonnull 22 | private final String storeProviderKlass; 23 | @Nonnull 24 | private final DataStoreProviderConfig storeProviderConfig; 25 | @Nonnull 26 | private final AuthenticationConfig authenticationConfig; 27 | 28 | @JsonCreator 29 | public BuckCacheConfiguration( 30 | @Nonnull @JsonProperty("mode") CacheInstanceMode mode, 31 | @Nonnull @JsonProperty("statsd") StatsdConfig statsd, 32 | @Nonnull @JsonProperty("throttleLimit") String throttleLimit, 33 | @Nonnull @JsonProperty("storeProviderKlass") String storeProviderKlass, 34 | @Nonnull @JsonProperty("storeProviderConfig") DataStoreProviderConfig storeProviderConfig, 35 | @Nonnull @JsonProperty("authenticationConfig") AuthenticationConfig authenticationConfig) { 36 | this.mode = mode; 37 | this.statsd = statsd; 38 | this.throttleLimit = throttleLimit; 39 | this.storeProviderKlass = storeProviderKlass; 40 | this.storeProviderConfig = storeProviderConfig; 41 | this.authenticationConfig = authenticationConfig; 42 | } 43 | 44 | public String getStoreProviderKlass() { 45 | return storeProviderKlass; 46 | } 47 | 48 | public DataStoreProviderConfig getStoreProviderConfig() { 49 | return storeProviderConfig; 50 | } 51 | 52 | public AuthenticationConfig getAuthenticationConfig() { 53 | return authenticationConfig; 54 | } 55 | 56 | public String getThrottleLimit() { 57 | return throttleLimit; 58 | } 59 | 60 | public StatsdConfig getStatsd() { 61 | return statsd; 62 | } 63 | 64 | public CacheInstanceMode getMode() { 65 | return mode; 66 | } 67 | 68 | public String toString() { 69 | return ToStringBuilder.reflectionToString(this); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/CacheInstanceMode.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache; 2 | 3 | public enum CacheInstanceMode { 4 | 5 | CLIENT, 6 | 7 | DATABASE_ONLY, 8 | 9 | DUAL; 10 | } 11 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/IgniteConfig.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.TimeUnit; 5 | 6 | import javax.annotation.Nonnull; 7 | 8 | import com.fasterxml.jackson.annotation.JsonCreator; 9 | import com.fasterxml.jackson.annotation.JsonProperty; 10 | import org.apache.commons.lang3.builder.ToStringBuilder; 11 | import org.apache.ignite.cache.CacheMemoryMode; 12 | import org.apache.ignite.cache.CacheMode; 13 | 14 | public class IgniteConfig { 15 | 16 | @Nonnull 17 | private final String multicastIP; 18 | @Nonnull 19 | private final Integer multicastPort; 20 | @Nonnull 21 | private final CacheMode cacheMode; 22 | @Nonnull 23 | private final Integer cacheBackupCount; 24 | @Nonnull 25 | private final TimeUnit expirationTimeUnit; 26 | @Nonnull 27 | private final Long expirationTimeValue; 28 | @Nonnull 29 | private final Integer atomicSequenceReserveSize; 30 | @Nonnull 31 | private final String offHeapStorageSize; 32 | @Nonnull 33 | private final List hostIPs; 34 | @Nonnull 35 | private final String dnsLookupAddress; 36 | @Nonnull 37 | private final CacheMemoryMode cacheMemoryMode; 38 | 39 | @JsonCreator 40 | public IgniteConfig( 41 | @Nonnull @JsonProperty("multicastIP") String multicastIP, 42 | @Nonnull @JsonProperty("multicastPort") Integer multicastPort, 43 | @Nonnull @JsonProperty("cacheMode") CacheMode cacheMode, 44 | @Nonnull @JsonProperty("cacheBackupCount") Integer cacheBackupCount, 45 | @Nonnull @JsonProperty("expirationTimeUnit") TimeUnit expirationTimeUnit, 46 | @Nonnull @JsonProperty("expirationTimeValue") Long expirationTimeValue, 47 | @Nonnull @JsonProperty("atomicSequenceReserveSize") Integer atomicSequenceReserveSize, 48 | @Nonnull @JsonProperty("offHeapStorageSize") String offHeapStorageSize, 49 | @Nonnull @JsonProperty("hostIPs") List hostIPs, 50 | @Nonnull @JsonProperty("dnsLookupAddress") String dnsLookupAddress, 51 | @Nonnull @JsonProperty("cacheMemoryMode") CacheMemoryMode cacheMemoryMode) { 52 | this.multicastIP = multicastIP; 53 | this.multicastPort = multicastPort; 54 | this.cacheMode = cacheMode; 55 | this.cacheBackupCount = cacheBackupCount; 56 | this.expirationTimeUnit = expirationTimeUnit; 57 | this.expirationTimeValue = expirationTimeValue; 58 | this.atomicSequenceReserveSize = atomicSequenceReserveSize; 59 | this.offHeapStorageSize = offHeapStorageSize; 60 | this.hostIPs = hostIPs; 61 | this.dnsLookupAddress = dnsLookupAddress; 62 | this.cacheMemoryMode = cacheMemoryMode; 63 | } 64 | 65 | public List getHostIPs() { 66 | return hostIPs; 67 | } 68 | 69 | public String getOffHeapStorageSize() { 70 | return offHeapStorageSize; 71 | } 72 | 73 | public String getMulticastIP() { 74 | return multicastIP; 75 | } 76 | 77 | public Integer getMulticastPort() { 78 | return multicastPort; 79 | } 80 | 81 | public CacheMode getCacheMode() { 82 | return cacheMode; 83 | } 84 | 85 | public Integer getCacheBackupCount() { 86 | return cacheBackupCount; 87 | } 88 | 89 | public TimeUnit getExpirationTimeUnit() { 90 | return expirationTimeUnit; 91 | } 92 | 93 | public Long getExpirationTimeValue() { 94 | return expirationTimeValue; 95 | } 96 | 97 | public Integer getAtomicSequenceReserveSize() { 98 | return atomicSequenceReserveSize; 99 | } 100 | 101 | public CacheMemoryMode getCacheMemoryMode() { 102 | return cacheMemoryMode; 103 | } 104 | 105 | public String getDnsLookupAddress() { 106 | return dnsLookupAddress; 107 | } 108 | public String toString() { 109 | return ToStringBuilder.reflectionToString(this); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/StatsdConfig.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import org.apache.commons.lang3.builder.ToStringBuilder; 6 | 7 | public class StatsdConfig { 8 | private final boolean enabled; 9 | private final String prefix; 10 | private final String host; 11 | private final int port; 12 | private final double sampleRate; 13 | 14 | @JsonCreator 15 | public StatsdConfig( 16 | @JsonProperty("enabled") boolean enabled, 17 | @JsonProperty("prefix") String prefix, 18 | @JsonProperty("host") String host, 19 | @JsonProperty("port") int port, 20 | @JsonProperty("sampleRate") double sampleRate) { 21 | this.enabled = enabled; 22 | this.prefix = prefix; 23 | this.host = host; 24 | this.port = port; 25 | this.sampleRate = sampleRate; 26 | } 27 | 28 | public double getSampleRate() { 29 | return sampleRate; 30 | } 31 | 32 | public String getPrefix() { 33 | return prefix; 34 | } 35 | 36 | public boolean isEnabled() { 37 | return enabled; 38 | } 39 | 40 | public String getHost() { 41 | return host; 42 | } 43 | 44 | public int getPort() { 45 | return port; 46 | } 47 | 48 | public String toString() { 49 | return ToStringBuilder.reflectionToString(this); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/auth/HttpHeaderAuthFilter.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.auth; 2 | 3 | import io.dropwizard.auth.AuthFilter; 4 | 5 | import javax.annotation.Priority; 6 | import javax.ws.rs.NotAuthorizedException; 7 | import javax.ws.rs.Priorities; 8 | import javax.ws.rs.container.ContainerRequestContext; 9 | import javax.ws.rs.core.HttpHeaders; 10 | import java.io.IOException; 11 | import java.util.List; 12 | 13 | import static java.util.Collections.emptyList; 14 | 15 | @Priority(Priorities.AUTHENTICATION) 16 | public class HttpHeaderAuthFilter extends AuthFilter { 17 | 18 | private final List tokens; 19 | 20 | public HttpHeaderAuthFilter() { 21 | this(emptyList()); 22 | } 23 | 24 | public HttpHeaderAuthFilter(List tokens) { 25 | this.tokens = tokens == null ? emptyList() : tokens; 26 | } 27 | 28 | @Override 29 | public void filter(final ContainerRequestContext requestContext) throws IOException { 30 | if (tokens.isEmpty()) { 31 | return ; 32 | } 33 | 34 | String token = requestContext.getHeaders().getFirst(HttpHeaders.AUTHORIZATION); 35 | 36 | if (token == null || !tokens.contains(token)) { 37 | throw new NotAuthorizedException("Unauthorized"); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/CacheAdapter.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | import com.uber.buckcache.datastore.exceptions.DatastoreUnavailableException; 6 | import com.uber.buckcache.datastore.exceptions.EntryNotFoundException; 7 | 8 | /** 9 | * An adapter to the underlying cache. 10 | */ 11 | public interface CacheAdapter { 12 | @Nonnull 13 | CacheEntry get(String key) throws DatastoreUnavailableException, EntryNotFoundException; 14 | 15 | void put(@Nonnull String[] keys, @Nonnull CacheEntry cacheEntry) throws DatastoreUnavailableException; 16 | 17 | int cachedArtifactCount() throws DatastoreUnavailableException; 18 | int keysCount() throws DatastoreUnavailableException; 19 | } 20 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/CacheEntry.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | import org.apache.commons.lang3.builder.ToStringBuilder; 6 | 7 | /** 8 | * An entry in the build cache. 9 | */ 10 | public class CacheEntry { 11 | @Nonnull 12 | byte[] buckData; 13 | @Nonnull 14 | private final int bytes; 15 | 16 | public CacheEntry(byte[] buckData) { 17 | this.buckData = buckData; 18 | this.bytes = this.buckData.length; 19 | } 20 | 21 | public byte[] getBuckData() { 22 | return buckData; 23 | } 24 | 25 | public void setBuckData(byte[] buckData) { 26 | this.buckData = buckData; 27 | } 28 | 29 | public int getBytes() { 30 | return bytes; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/DataStoreProvider.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore; 2 | 3 | import com.codahale.metrics.health.HealthCheck; 4 | import com.uber.buckcache.CacheInstanceMode; 5 | 6 | import io.dropwizard.lifecycle.Managed; 7 | 8 | import java.util.concurrent.TimeUnit; 9 | 10 | public interface DataStoreProvider extends Managed { 11 | 12 | /** 13 | * CacheProviderConfig : passes all the configs that are provider in the config.yml file 14 | * CacheInstanceMode : this tells the cache provider whether to only come up as 15 | * cacheClient or server or both. 16 | * @param cacheProviderConfig 17 | */ 18 | public void init(DataStoreProviderConfig cacheProviderConfig, CacheInstanceMode mode) throws Exception; 19 | 20 | /** 21 | * returns cache entry associated with the key 22 | * @param key 23 | * @return 24 | */ 25 | public CacheEntry getData(String key) throws Exception; 26 | 27 | /** 28 | * @param keys : all the keys that associated with this given Cache Entry 29 | * @param cacheData : data to store/cache. 30 | */ 31 | public void putData(String[] keys, CacheEntry cacheData) throws Exception; 32 | 33 | /** 34 | * @param keys : all the keys that associated with this given Cache Entry 35 | * @param cacheData : data to store/cache 36 | * @param expirationTimeUnit : the time unit of expiration for the data 37 | * @param expirationTimeValue : the time value of expiration for the data 38 | */ 39 | public void putData(String[] keys, CacheEntry cacheData, TimeUnit expirationTimeUnit, Long expirationTimeValue) throws Exception; 40 | 41 | /** 42 | * return the current key count of the cache 43 | * @return 44 | */ 45 | public int getNumberOfKeys() throws Exception; 46 | 47 | /** 48 | * return the current value count of the cache 49 | * @return 50 | */ 51 | public int getNumberOfValues() throws Exception; 52 | 53 | /** 54 | * this is to monitor the health of the data store. 55 | * @return 56 | * @throws Exception 57 | */ 58 | public HealthCheck.Result check() throws Exception; 59 | 60 | } 61 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/DataStoreProviderConfig.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore; 2 | 3 | import java.util.Map; 4 | 5 | import com.fasterxml.jackson.annotation.JsonCreator; 6 | import com.fasterxml.jackson.annotation.JsonProperty; 7 | 8 | public class DataStoreProviderConfig { 9 | 10 | private final Map config; 11 | 12 | @JsonCreator 13 | public DataStoreProviderConfig(@JsonProperty("config") Map config) { 14 | this.config = config; 15 | } 16 | 17 | public Map getConfig() { 18 | return config; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/DataStoreProviderFactory.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore; 2 | 3 | import com.uber.buckcache.CacheInstanceMode; 4 | 5 | public class DataStoreProviderFactory { 6 | 7 | 8 | /** 9 | * returns the correct cache provider impl. 10 | * @param datastoreType 11 | * @param providerConfig 12 | * @param mode 13 | * @return 14 | * @throws Exception 15 | */ 16 | public static DataStoreProvider get(String klass, DataStoreProviderConfig providerConfig, CacheInstanceMode mode) throws Exception { 17 | DataStoreProvider cacheProvider = (DataStoreProvider) Class.forName(klass).newInstance(); 18 | cacheProvider.init(providerConfig, mode); 19 | return cacheProvider; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/exceptions/CacheException.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.exceptions; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | /** 6 | * Generic exception class for cache. 7 | */ 8 | class CacheException extends Exception { 9 | @Nonnull private final String message; 10 | 11 | CacheException(@Nonnull String message) { 12 | this.message = message; 13 | } 14 | 15 | 16 | @Override 17 | @Nonnull 18 | public String getMessage() { 19 | return message; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/exceptions/DatastoreUnavailableException.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.exceptions; 2 | 3 | /** 4 | * The datastore is offline. 5 | */ 6 | public class DatastoreUnavailableException extends CacheException { 7 | public DatastoreUnavailableException(String message) { 8 | super(message); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/exceptions/EntryNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.exceptions; 2 | 3 | /** 4 | * The specified entry was not found. 5 | */ 6 | public class EntryNotFoundException extends CacheException { 7 | public EntryNotFoundException(String message) { 8 | super(message); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/impl/ignite/IgniteConfigurationBuilder.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.impl.ignite; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | import javax.cache.expiry.Duration; 8 | import javax.cache.expiry.TouchedExpiryPolicy; 9 | 10 | import com.google.common.annotations.VisibleForTesting; 11 | import com.spotify.dns.DnsSrvResolvers; 12 | import org.apache.commons.lang3.StringUtils; 13 | import org.apache.ignite.cache.CacheMemoryMode; 14 | import org.apache.ignite.cache.CacheMode; 15 | import org.apache.ignite.configuration.AtomicConfiguration; 16 | import org.apache.ignite.configuration.CacheConfiguration; 17 | import org.apache.ignite.configuration.IgniteConfiguration; 18 | import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; 19 | import org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder; 20 | import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; 21 | 22 | import com.google.common.base.Strings; 23 | import com.spotify.dns.DnsSrvResolver; 24 | import com.spotify.dns.LookupResult; 25 | import com.uber.buckcache.utils.BytesRateLimiter.BIT_UNIT; 26 | 27 | public class IgniteConfigurationBuilder { 28 | 29 | private IgniteConfiguration igniteConfiguration; 30 | private final DnsSrvResolver dnsResolver; 31 | 32 | public IgniteConfigurationBuilder() { 33 | this(DnsSrvResolvers.newBuilder() 34 | .retainingDataOnFailures(true) 35 | .build()); 36 | igniteConfiguration = new IgniteConfiguration(); 37 | } 38 | 39 | protected IgniteConfigurationBuilder(DnsSrvResolver dnsResolver) { 40 | this.dnsResolver = dnsResolver; 41 | igniteConfiguration = new IgniteConfiguration(); 42 | } 43 | 44 | public IgniteConfigurationBuilder addMulticastBasedDiscrovery(String multicastIP, Integer multicastPort, List hostIPs, String dnsLookupAddress) { 45 | if (hostIPs == null) { 46 | hostIPs = new ArrayList(); 47 | } 48 | 49 | TcpDiscoverySpi spi = new TcpDiscoverySpi(); 50 | 51 | TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryMulticastIpFinder(); 52 | ((TcpDiscoveryMulticastIpFinder) ipFinder).setMulticastGroup(multicastIP); 53 | ((TcpDiscoveryMulticastIpFinder) ipFinder).setMulticastPort(multicastPort); 54 | 55 | boolean hasDNSLookupAddress = !Strings.isNullOrEmpty(dnsLookupAddress); 56 | 57 | if (hasDNSLookupAddress) { 58 | hostIPs.addAll(this.resolveAddressByDNS(dnsLookupAddress)); 59 | } 60 | ((TcpDiscoveryMulticastIpFinder) ipFinder).setAddresses(hostIPs); 61 | spi.setIpFinder(ipFinder); 62 | 63 | // Override default discovery SPI. 64 | igniteConfiguration.setDiscoverySpi(spi); 65 | return this; 66 | } 67 | 68 | public IgniteConfigurationBuilder addCacheConfiguration(CacheMode cacheMode, Integer backupCount, 69 | TimeUnit expirationTimeUnit, Long expirationTimeValue, String offHeapMaxSize, CacheMemoryMode cacheMemoryMode, String ...caches) { 70 | 71 | CacheConfiguration[] cacheConfigs = new CacheConfiguration[caches.length]; 72 | 73 | for (int i = 0; i < caches.length; i++) { 74 | CacheConfiguration cacheConfiguration = new CacheConfiguration(caches[i]); 75 | cacheConfiguration.setCacheMode(cacheMode); 76 | cacheConfiguration.setMemoryMode(cacheMemoryMode); 77 | cacheConfiguration.setStatisticsEnabled(true); 78 | 79 | String multiplier = offHeapMaxSize.substring(0, offHeapMaxSize.length() - 1); 80 | String unit = offHeapMaxSize.substring(offHeapMaxSize.length() - 1, offHeapMaxSize.length()); 81 | 82 | long offHeapMemoryMax = 83 | Long.parseLong(multiplier) * BIT_UNIT.valueOf(StringUtils.lowerCase(unit)).getNumberOfBytes(); 84 | cacheConfiguration.setOffHeapMaxMemory(offHeapMemoryMax); 85 | 86 | cacheConfiguration.setBackups(backupCount); 87 | cacheConfiguration 88 | .setExpiryPolicyFactory(TouchedExpiryPolicy.factoryOf(new Duration(expirationTimeUnit, expirationTimeValue))); 89 | cacheConfigs[i] = cacheConfiguration; 90 | } 91 | 92 | igniteConfiguration.setCacheConfiguration(cacheConfigs); 93 | return this; 94 | } 95 | 96 | public IgniteConfigurationBuilder addAtomicSequenceConfig(Integer reserveSize) { 97 | AtomicConfiguration atomicCfg = new AtomicConfiguration(); 98 | atomicCfg.setAtomicSequenceReserveSize(reserveSize); 99 | igniteConfiguration.setAtomicConfiguration(atomicCfg); 100 | return this; 101 | } 102 | 103 | public IgniteConfiguration build() { 104 | return igniteConfiguration; 105 | } 106 | 107 | private List resolveAddressByDNS(String dnsLookupAddress) { 108 | List address = new ArrayList(); 109 | for (LookupResult node : dnsResolver.resolve(dnsLookupAddress)) { 110 | address.add(node.host()); 111 | } 112 | return address; 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/impl/ignite/IgniteConstants.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.impl.ignite; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.ignite.events.EventType; 7 | 8 | public class IgniteConstants { 9 | 10 | public static final String KEYS_CACHE_NAME = "keys"; 11 | public static final String KEYS_REVERSE_CACHE_NAME = "keys-reverse"; 12 | public static final String METADATA_CACHE_NAME = "metadata"; 13 | public static final String ARTIFACT_CACHE_NAME = "artifacts"; 14 | public static final String UNDERLYING_KEY_SEQUENCE_NAME = "underlyingArtifactKeys"; 15 | 16 | public static final Map EVENT_TYPE_TO_NAME_MAP = new HashMap() { 17 | { 18 | put(EventType.EVT_CACHE_OBJECT_EXPIRED, "EXPIRED"); 19 | put(EventType.EVT_CACHE_OBJECT_REMOVED, "REMOVED"); 20 | put(EventType.EVT_CACHE_ENTRY_EVICTED, "EVICTED"); 21 | } 22 | }; 23 | } 24 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/impl/ignite/IgniteDataStoreProvider.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.impl.ignite; 2 | 3 | import static com.uber.buckcache.utils.MetricsRegistry.IGNITE_CACHE_GET_CALL_COUNT; 4 | import static com.uber.buckcache.utils.MetricsRegistry.IGNITE_CACHE_GET_CALL_TIME; 5 | import static com.uber.buckcache.utils.MetricsRegistry.IGNITE_CACHE_PUT_CALL_COUNT; 6 | import static com.uber.buckcache.utils.MetricsRegistry.IGNITE_CACHE_PUT_CALL_TIME; 7 | 8 | import java.io.IOException; 9 | 10 | import java.util.Optional; 11 | import java.util.concurrent.TimeUnit; 12 | import javax.cache.expiry.CreatedExpiryPolicy; 13 | import javax.cache.expiry.Duration; 14 | import javax.cache.expiry.ExpiryPolicy; 15 | import org.slf4j.Logger; 16 | 17 | import com.codahale.metrics.health.HealthCheck; 18 | import com.codahale.metrics.health.HealthCheck.Result; 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | import com.uber.buckcache.CacheInstanceMode; 21 | import com.uber.buckcache.IgniteConfig; 22 | import com.uber.buckcache.datastore.CacheEntry; 23 | import com.uber.buckcache.datastore.DataStoreProvider; 24 | import com.uber.buckcache.datastore.DataStoreProviderConfig; 25 | import com.uber.buckcache.datastore.exceptions.EntryNotFoundException; 26 | import com.uber.buckcache.utils.StatsDClient; 27 | 28 | public class IgniteDataStoreProvider implements DataStoreProvider { 29 | private static final ObjectMapper om = new ObjectMapper(); 30 | private static final Logger logger = org.slf4j.LoggerFactory.getLogger(IgniteDataStoreProvider.class); 31 | 32 | private CacheInstanceMode mode; 33 | private DataStoreProviderConfig cacheProviderConfig; 34 | private IgniteInstance igniteInstance; 35 | private IgniteConfig config; 36 | 37 | @Override 38 | public void init(DataStoreProviderConfig cacheProviderConfig, CacheInstanceMode mode) throws IOException { 39 | this.cacheProviderConfig = cacheProviderConfig; 40 | this.mode = mode; 41 | String configs = om.writeValueAsString(this.cacheProviderConfig.getConfig()); 42 | config = om.readValue(configs, IgniteConfig.class); 43 | logger.info("initialized IgniteCacheProvider using CacheProviderConfig {} , and Ignite Config {}", 44 | cacheProviderConfig, config); 45 | igniteInstance = new IgniteInstance(mode, config); 46 | igniteInstance.start(); 47 | } 48 | 49 | @Override 50 | public void start() throws Exception { 51 | logger.info("ignite cache provider startup sequence complete"); 52 | // nothing to do here 53 | } 54 | 55 | @Override 56 | public void stop() throws Exception { 57 | igniteInstance.stop(); 58 | logger.info("ignite cache provider shutdown sequence complete"); 59 | } 60 | 61 | @Override 62 | public CacheEntry getData(String key) throws EntryNotFoundException { 63 | StatsDClient.get().count(IGNITE_CACHE_GET_CALL_COUNT, 1L); 64 | long start = System.currentTimeMillis(); 65 | 66 | Long underlyingId = igniteInstance.getCacheKeys().get(key); 67 | 68 | if (underlyingId != null) { 69 | byte[] buckCacheData = igniteInstance.getBuckDataCache().get(underlyingId); 70 | 71 | if (buckCacheData != null) { 72 | StatsDClient.get().recordExecutionTimeToNow(IGNITE_CACHE_GET_CALL_TIME, start); 73 | return new CacheEntry(buckCacheData); 74 | } 75 | } 76 | 77 | throw new EntryNotFoundException("EntryNotFoundException"); 78 | } 79 | 80 | @Override 81 | public void putData(String[] keys, CacheEntry cacheData) { 82 | putData(keys, cacheData, Optional.empty()); 83 | } 84 | 85 | @Override 86 | public void putData(String[] keys, CacheEntry cacheData, TimeUnit expirationTimeUnit, Long expirationTimeValue) throws Exception { 87 | ExpiryPolicy customExpiryPolicy = new CreatedExpiryPolicy(new Duration(expirationTimeUnit, expirationTimeValue)); 88 | putData(keys, cacheData, Optional.of(customExpiryPolicy)); 89 | } 90 | 91 | @Override 92 | public int getNumberOfKeys() throws Exception { 93 | return igniteInstance.getCacheKeys().metrics().getKeySize(); 94 | } 95 | 96 | @Override 97 | public int getNumberOfValues() throws Exception { 98 | return igniteInstance.getBuckDataCache().metrics().getKeySize(); 99 | } 100 | 101 | @Override 102 | public Result check() throws Exception { 103 | if (igniteInstance == null) { 104 | return HealthCheck.Result.unhealthy("Ignite instance is null"); 105 | } 106 | 107 | return HealthCheck.Result.healthy("OK"); 108 | } 109 | 110 | private void putData(String[] keys, CacheEntry cacheEntry, Optional policy) { 111 | StatsDClient.get().count(IGNITE_CACHE_PUT_CALL_COUNT, 1L); 112 | long start = System.currentTimeMillis(); 113 | Long underlyingId = igniteInstance.getAtomicSequence().incrementAndGet(); 114 | 115 | // Write backwards, adding the entry first, before adding the keys. 116 | igniteInstance.getBuckDataCache(policy).put(underlyingId, cacheEntry.getBuckData()); 117 | igniteInstance.getReverseCacheKeys(policy).put(underlyingId, keys); 118 | 119 | for (String key : keys) { 120 | igniteInstance.getCacheKeys(policy).put(key, underlyingId); 121 | } 122 | StatsDClient.get().recordExecutionTimeToNow(IGNITE_CACHE_PUT_CALL_TIME, start); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/impl/ignite/IgniteInstance.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.impl.ignite; 2 | 3 | import static com.uber.buckcache.datastore.impl.ignite.IgniteConstants.KEYS_CACHE_NAME; 4 | import static com.uber.buckcache.datastore.impl.ignite.IgniteConstants.KEYS_REVERSE_CACHE_NAME; 5 | import static com.uber.buckcache.datastore.impl.ignite.IgniteConstants.METADATA_CACHE_NAME; 6 | import static com.uber.buckcache.datastore.impl.ignite.IgniteConstants.UNDERLYING_KEY_SEQUENCE_NAME; 7 | import static com.uber.buckcache.utils.MetricsRegistry.CLUSTER_SIZE; 8 | import static com.uber.buckcache.utils.MetricsRegistry.CPU_COUNT; 9 | import static com.uber.buckcache.utils.MetricsRegistry.CPU_TIME; 10 | import static com.uber.buckcache.utils.MetricsRegistry.HEAP_COUNT; 11 | import static com.uber.buckcache.utils.MetricsRegistry.OFF_HEAP_COUNT; 12 | import static com.uber.buckcache.utils.MetricsRegistry.OFF_HEAP_TIME; 13 | 14 | import java.util.Optional; 15 | import java.util.Timer; 16 | import java.util.TimerTask; 17 | 18 | import javax.cache.expiry.ExpiryPolicy; 19 | import org.apache.ignite.Ignite; 20 | import org.apache.ignite.IgniteAtomicSequence; 21 | import org.apache.ignite.IgniteCache; 22 | import org.apache.ignite.Ignition; 23 | import org.apache.ignite.cluster.ClusterMetrics; 24 | import org.apache.ignite.cluster.ClusterNode; 25 | import org.apache.ignite.configuration.IgniteConfiguration; 26 | import org.apache.ignite.events.EventType; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import com.uber.buckcache.CacheInstanceMode; 31 | import com.uber.buckcache.IgniteConfig; 32 | import com.uber.buckcache.utils.StatsDClient; 33 | 34 | public class IgniteInstance { 35 | private static final Logger logger = LoggerFactory.getLogger(IgniteInstance.class); 36 | private static final int TEN_SECONDS = 10 * 1000; 37 | private final IgniteConfig config; 38 | private final IgniteConfiguration igniteConfiguration; 39 | private final Ignite ignite; 40 | 41 | private final IgniteAtomicSequence atomicSequence; 42 | private final IgniteCache cacheKeys; 43 | private final IgniteCache reverseCacheKeys; 44 | private final IgniteCache buckDataCache; 45 | private final Timer timer = new Timer("ignite_metrics_reporter"); 46 | 47 | public IgniteInstance(CacheInstanceMode mode, IgniteConfig config) { 48 | this.config = config; 49 | this.igniteConfiguration = new IgniteConfigurationBuilder() 50 | .addMulticastBasedDiscrovery(this.config.getMulticastIP(), this.config.getMulticastPort(), 51 | this.config.getHostIPs(), this.config.getDnsLookupAddress()) 52 | .addCacheConfiguration(this.config.getCacheMode(), this.config.getCacheBackupCount(), 53 | this.config.getExpirationTimeUnit(), this.config.getExpirationTimeValue(), 54 | this.config.getOffHeapStorageSize(), config.getCacheMemoryMode(), KEYS_CACHE_NAME, KEYS_REVERSE_CACHE_NAME, METADATA_CACHE_NAME) 55 | .addAtomicSequenceConfig(config.getAtomicSequenceReserveSize()).build(); 56 | 57 | logger.info("isClientMode : {}", mode == CacheInstanceMode.CLIENT); 58 | Ignition.setClientMode(mode == CacheInstanceMode.CLIENT); 59 | ignite = Ignition.start(igniteConfiguration); 60 | 61 | cacheKeys = ignite.cluster().ignite().getOrCreateCache(KEYS_CACHE_NAME); 62 | reverseCacheKeys = ignite.cluster().ignite().getOrCreateCache(KEYS_REVERSE_CACHE_NAME); 63 | buckDataCache = ignite.cluster().ignite().getOrCreateCache(METADATA_CACHE_NAME); 64 | atomicSequence = ignite.cluster().ignite().atomicSequence(UNDERLYING_KEY_SEQUENCE_NAME, 0, true); 65 | 66 | ignite.events().localListen(new LocalCacheEventListener(buckDataCache, reverseCacheKeys, cacheKeys), 67 | EventType.EVT_CACHE_OBJECT_EXPIRED, EventType.EVT_CACHE_OBJECT_REMOVED); 68 | } 69 | 70 | public void start() { 71 | timer.scheduleAtFixedRate(new TimerTask() { 72 | 73 | @Override 74 | public void run() { 75 | reportMetrics(); 76 | } 77 | }, TEN_SECONDS, TEN_SECONDS); 78 | } 79 | 80 | public void stop() { 81 | timer.cancel(); 82 | Ignition.stop(ignite.name(), false); 83 | } 84 | 85 | public IgniteCache getCacheKeys() { 86 | return cacheKeys; 87 | } 88 | 89 | public IgniteCache getReverseCacheKeys() { 90 | return reverseCacheKeys; 91 | } 92 | 93 | public IgniteCache getBuckDataCache() { 94 | return buckDataCache; 95 | } 96 | 97 | public IgniteCache getCacheKeys(Optional policy) { 98 | if (policy.isPresent()) { 99 | return cacheKeys.withExpiryPolicy(policy.get()); 100 | } else { 101 | return cacheKeys; 102 | } 103 | } 104 | 105 | public IgniteCache getReverseCacheKeys(Optional policy) { 106 | if (policy.isPresent()) { 107 | return reverseCacheKeys.withExpiryPolicy(policy.get()); 108 | } else { 109 | return reverseCacheKeys; 110 | } 111 | } 112 | 113 | public IgniteCache getBuckDataCache(Optional policy) { 114 | if (policy.isPresent()) { 115 | return buckDataCache.withExpiryPolicy(policy.get()); 116 | } else { 117 | return buckDataCache; 118 | } 119 | } 120 | 121 | public IgniteAtomicSequence getAtomicSequence() { 122 | return atomicSequence; 123 | } 124 | 125 | public Ignite getIgnite() { 126 | return ignite.cluster().ignite(); 127 | } 128 | 129 | public void reportMetrics() { 130 | // TODO: check and make sure that these are not costly to compute 131 | ClusterNode thisNode = ignite.cluster().localNode(); 132 | ClusterMetrics metrics = thisNode.metrics(); 133 | 134 | Double cpuLoad = metrics.getCurrentCpuLoad(); 135 | long heapUsage = metrics.getHeapMemoryUsed(); 136 | long offHeapUsage = metrics.getNonHeapMemoryUsed(); 137 | 138 | StatsDClient.get().count(CPU_COUNT, cpuLoad.longValue()); 139 | StatsDClient.get().count(CPU_TIME, cpuLoad.longValue()); 140 | 141 | StatsDClient.get().count(HEAP_COUNT, heapUsage); 142 | StatsDClient.get().count(HEAP_COUNT, heapUsage); 143 | 144 | StatsDClient.get().count(OFF_HEAP_COUNT, offHeapUsage); 145 | StatsDClient.get().count(OFF_HEAP_TIME, offHeapUsage); 146 | 147 | StatsDClient.get().gauge(CLUSTER_SIZE, ignite.cluster().hostNames().size()); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/datastore/impl/ignite/LocalCacheEventListener.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.impl.ignite; 2 | 3 | import static com.uber.buckcache.datastore.impl.ignite.IgniteConstants.ARTIFACT_CACHE_NAME; 4 | import static com.uber.buckcache.datastore.impl.ignite.IgniteConstants.EVENT_TYPE_TO_NAME_MAP; 5 | 6 | import org.apache.ignite.IgniteCache; 7 | import org.apache.ignite.events.CacheEvent; 8 | import org.apache.ignite.lang.IgnitePredicate; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | public class LocalCacheEventListener implements IgnitePredicate { 13 | private static Logger logger = LoggerFactory.getLogger(LocalCacheEventListener.class); 14 | 15 | private final IgniteCache metadataCache; 16 | private final IgniteCache reverseCacheKeys; 17 | private final IgniteCache cacheKeys; 18 | 19 | public LocalCacheEventListener(IgniteCache metadataCache, IgniteCache reverseCacheKeys, 20 | IgniteCache cacheKeys) { 21 | this.metadataCache = metadataCache; 22 | this.reverseCacheKeys = reverseCacheKeys; 23 | this.cacheKeys = cacheKeys; 24 | } 25 | 26 | @Override 27 | public boolean apply(CacheEvent evt) { 28 | if (evt.cacheName().equals(ARTIFACT_CACHE_NAME)) { 29 | 30 | final String eventName = 31 | EVENT_TYPE_TO_NAME_MAP.containsKey(evt.type()) ? EVENT_TYPE_TO_NAME_MAP.get(evt.type()) : "UNKNOWN"; 32 | Long underlyingKey = evt.key(); 33 | String[] keys = reverseCacheKeys.getAndRemove(underlyingKey); 34 | 35 | logger.info(String.format("Artifact cache event {} with key {}.", eventName, underlyingKey), keys, evt); 36 | 37 | for (String key : keys) { 38 | cacheKeys.remove(key); 39 | } 40 | 41 | metadataCache.remove(underlyingKey); 42 | } 43 | 44 | return true; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/health/DataStoreHealthChecker.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.health; 2 | 3 | import com.codahale.metrics.health.HealthCheck; 4 | import com.uber.buckcache.datastore.DataStoreProvider; 5 | 6 | public class DataStoreHealthChecker extends HealthCheck { 7 | private final DataStoreProvider dataStoreProvider; 8 | 9 | public DataStoreHealthChecker(DataStoreProvider dataStoreProvider) { 10 | this.dataStoreProvider = dataStoreProvider; 11 | } 12 | 13 | @Override 14 | public HealthCheck.Result check() throws Exception { 15 | return dataStoreProvider.check(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/resources/RootResource.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.resources; 2 | 3 | import java.net.URI; 4 | 5 | import javax.ws.rs.GET; 6 | import javax.ws.rs.Path; 7 | import javax.ws.rs.Produces; 8 | import javax.ws.rs.core.Context; 9 | import javax.ws.rs.core.MediaType; 10 | import javax.ws.rs.core.Response; 11 | import javax.ws.rs.core.UriBuilder; 12 | import javax.ws.rs.core.UriInfo; 13 | 14 | /** 15 | * Redirect to cache summary. 16 | */ 17 | @Path("/") 18 | public class RootResource { 19 | 20 | @Context 21 | UriInfo uriInfo; 22 | 23 | @GET 24 | @Path("") 25 | @Produces(MediaType.TEXT_PLAIN) 26 | public Response index() { 27 | UriBuilder ub = uriInfo.getAbsolutePathBuilder(); 28 | URI redirectURI = ub 29 | .path("/artifacts/summary") 30 | .build(); 31 | 32 | return Response.seeOther(redirectURI).build(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/resources/buckcache/BuckCacheResource.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.resources.buckcache; 2 | 3 | import static com.uber.buckcache.utils.MetricsRegistry.GET_CALL_COUNT; 4 | import static com.uber.buckcache.utils.MetricsRegistry.GET_CALL_TIME; 5 | import static com.uber.buckcache.utils.MetricsRegistry.INCOMING_BYTES_PER_REQUEST; 6 | import static com.uber.buckcache.utils.MetricsRegistry.INCOMING_BYTES_TOTAL_COUNT; 7 | import static com.uber.buckcache.utils.MetricsRegistry.OUTGOING_BYTES_PER_REQUEST; 8 | import static com.uber.buckcache.utils.MetricsRegistry.OUTGOING_BYTES_TOTAL_COUNT; 9 | import static com.uber.buckcache.utils.MetricsRegistry.PUT_CALL_COUNT; 10 | import static com.uber.buckcache.utils.MetricsRegistry.PUT_CALL_TIME; 11 | import static com.uber.buckcache.utils.MetricsRegistry.SUMMARY_CALL_COUNT; 12 | import static com.uber.buckcache.utils.MetricsRegistry.CACHE_HIT_COUNT; 13 | import static com.uber.buckcache.utils.MetricsRegistry.CACHE_MISS_COUNT; 14 | 15 | 16 | import java.io.DataInputStream; 17 | import java.io.DataOutputStream; 18 | import java.io.IOException; 19 | import java.io.InputStream; 20 | import java.io.OutputStream; 21 | import java.lang.annotation.Annotation; 22 | import java.lang.reflect.Type; 23 | 24 | import java.util.concurrent.TimeUnit; 25 | import javax.annotation.security.PermitAll; 26 | import javax.ws.rs.Consumes; 27 | import javax.ws.rs.GET; 28 | import javax.ws.rs.HeaderParam; 29 | import javax.ws.rs.PUT; 30 | import javax.ws.rs.Path; 31 | import javax.ws.rs.PathParam; 32 | import javax.ws.rs.Produces; 33 | import javax.ws.rs.WebApplicationException; 34 | import javax.ws.rs.core.HttpHeaders; 35 | import javax.ws.rs.core.MediaType; 36 | import javax.ws.rs.core.MultivaluedMap; 37 | import javax.ws.rs.core.Response; 38 | import javax.ws.rs.ext.MessageBodyReader; 39 | import javax.ws.rs.ext.MessageBodyWriter; 40 | import javax.ws.rs.ext.Provider; 41 | 42 | import io.dropwizard.auth.Auth; 43 | import org.apache.commons.lang3.StringUtils; 44 | import org.apache.commons.lang3.math.NumberUtils; 45 | import org.slf4j.Logger; 46 | import org.slf4j.LoggerFactory; 47 | 48 | import com.google.common.io.ByteStreams; 49 | import com.uber.buckcache.datastore.CacheEntry; 50 | import com.uber.buckcache.datastore.DataStoreProvider; 51 | import com.uber.buckcache.datastore.exceptions.DatastoreUnavailableException; 52 | import com.uber.buckcache.datastore.exceptions.EntryNotFoundException; 53 | import com.uber.buckcache.utils.BytesRateLimiter; 54 | import com.uber.buckcache.utils.StatsDClient; 55 | /** 56 | * Buck HTTP Cache API implementation. 57 | * See: https://buckbuild.com/concept/http_cache_api.html 58 | */ 59 | @Path("/artifacts") 60 | @Consumes(MediaType.APPLICATION_OCTET_STREAM) 61 | public class BuckCacheResource { 62 | private static final Logger logger = LoggerFactory.getLogger(BuckCacheResource.class); 63 | private static final String X_CACHE_EXPIRY_SECONDS = "X-Cache-Expiry-Seconds"; 64 | private static final String X_CACHE_TAGS = "X-Cache-Tags"; 65 | 66 | private final DataStoreProvider storeProvider; 67 | private final BytesRateLimiter rateLimiter; 68 | 69 | public BuckCacheResource(DataStoreProvider storeProvider, BytesRateLimiter rateLimiter) { 70 | this.rateLimiter = rateLimiter; 71 | this.storeProvider = storeProvider; 72 | } 73 | 74 | private static String getCacheKeyWithCustomizedParam(String key, String cacheTags) { 75 | if (StringUtils.isEmpty(cacheTags)) { 76 | return key; 77 | } 78 | return String.format("%s.%s", key, cacheTags); 79 | } 80 | 81 | @GET 82 | @Path("summary") 83 | @Produces(MediaType.TEXT_PLAIN) 84 | public String getSummary() throws Exception { 85 | StatsDClient.get().count(SUMMARY_CALL_COUNT, 1L); 86 | return String.format("Storing %d artifacts with %d keys.", storeProvider.getNumberOfValues(), 87 | storeProvider.getNumberOfKeys()); 88 | } 89 | 90 | @GET 91 | @Path("key/{key}") 92 | @Produces(MediaType.APPLICATION_OCTET_STREAM) 93 | public Response getCacheArtifact(@PathParam("key") String key, 94 | @HeaderParam(X_CACHE_TAGS) String cacheTags) throws Exception { 95 | StatsDClient.get().count(getCacheKeyWithCustomizedParam(GET_CALL_COUNT, cacheTags), 1L); 96 | CacheEntry cacheEntry; 97 | long start = System.currentTimeMillis(); 98 | 99 | try { 100 | cacheEntry = storeProvider.getData(key); 101 | // TODO: need to kill the requests rather than queue here 102 | rateLimiter.checkout(cacheEntry.getBytes()); 103 | 104 | StatsDClient.get().count(OUTGOING_BYTES_TOTAL_COUNT, cacheEntry.getBytes()); 105 | StatsDClient.get().recordExecutionTime(OUTGOING_BYTES_PER_REQUEST, cacheEntry.getBytes()); 106 | } catch (DatastoreUnavailableException ex) { 107 | StatsDClient.get().recordExecutionTimeToNow(GET_CALL_TIME, start); 108 | return Response.status(Response.Status.SERVICE_UNAVAILABLE).build(); 109 | } catch (EntryNotFoundException ex) { 110 | logger.debug("Cache MISS", key); 111 | StatsDClient.get().count(getCacheKeyWithCustomizedParam(CACHE_MISS_COUNT, cacheTags), 1L); 112 | StatsDClient.get().recordExecutionTimeToNow(GET_CALL_TIME, start); 113 | return Response.status(Response.Status.NOT_FOUND).build(); 114 | } 115 | 116 | logger.debug("Cache hit", key); 117 | StatsDClient.get().count(getCacheKeyWithCustomizedParam(CACHE_HIT_COUNT, cacheTags), 1L); 118 | StatsDClient.get().recordExecutionTimeToNow(GET_CALL_TIME, start); 119 | return Response.ok(cacheEntry).build(); 120 | } 121 | 122 | @PUT 123 | @PermitAll 124 | @Path("key") 125 | @Consumes(MediaType.APPLICATION_OCTET_STREAM) 126 | public Response addArtifactToCache(PutCacheEntry putCacheEntry, 127 | @HeaderParam(X_CACHE_EXPIRY_SECONDS) String cacheExpirySeconds) throws Exception { 128 | StatsDClient.get().count(PUT_CALL_COUNT, 1L); 129 | 130 | long start = System.currentTimeMillis(); 131 | if (putCacheEntry.verify()) { 132 | try { 133 | // TODO: need to kill the requests rather than queue here 134 | rateLimiter.checkout(putCacheEntry.getBytes()); 135 | 136 | if (!StringUtils.isEmpty(cacheExpirySeconds)) { 137 | try { 138 | Long expiryTime = NumberUtils.createLong(cacheExpirySeconds); 139 | storeProvider.putData(putCacheEntry.getKeys(), 140 | putCacheEntry.getCacheEntry(), 141 | TimeUnit.SECONDS, 142 | expiryTime); 143 | } catch (NumberFormatException e) { 144 | storeProvider.putData(putCacheEntry.getKeys(), putCacheEntry.getCacheEntry()); 145 | } 146 | } else { 147 | storeProvider.putData(putCacheEntry.getKeys(), putCacheEntry.getCacheEntry()); 148 | } 149 | 150 | StatsDClient.get().recordExecutionTimeToNow(PUT_CALL_TIME, start); 151 | 152 | StatsDClient.get().count(INCOMING_BYTES_TOTAL_COUNT, putCacheEntry.getBytes()); 153 | StatsDClient.get().recordExecutionTime(INCOMING_BYTES_PER_REQUEST, putCacheEntry.getBytes()); 154 | 155 | return Response.status(Response.Status.ACCEPTED).build(); 156 | } catch (DatastoreUnavailableException ex) { 157 | StatsDClient.get().recordExecutionTimeToNow(PUT_CALL_TIME, start); 158 | return Response.status(Response.Status.SERVICE_UNAVAILABLE).build(); 159 | } 160 | } else { 161 | StatsDClient.get().recordExecutionTimeToNow(PUT_CALL_TIME, start); 162 | return Response.status(Response.Status.NOT_ACCEPTABLE).build(); 163 | } 164 | } 165 | 166 | @Provider 167 | @Produces(MediaType.APPLICATION_OCTET_STREAM) 168 | public static class CacheResultBodyWriter implements MessageBodyWriter { 169 | 170 | @Override 171 | public boolean isWriteable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { 172 | return type == CacheEntry.class; 173 | } 174 | 175 | @Override 176 | public long getSize(CacheEntry cacheEntry, Class type, Type genericType, Annotation[] annotations, 177 | MediaType mediaType) { 178 | return cacheEntry.getBytes(); 179 | } 180 | 181 | @Override 182 | public void writeTo(CacheEntry cacheEntry, Class type, Type genericType, Annotation[] annotations, 183 | MediaType mediaType, MultivaluedMap httpHeaders, OutputStream entityStream) 184 | throws IOException, WebApplicationException { 185 | DataOutputStream dataOutputStream = new DataOutputStream(entityStream); 186 | dataOutputStream.write(cacheEntry.getBuckData()); 187 | } 188 | } 189 | 190 | @Provider 191 | @Consumes(MediaType.APPLICATION_OCTET_STREAM) 192 | public static class CacheResultBodyReader implements MessageBodyReader { 193 | 194 | @Override 195 | public boolean isReadable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) { 196 | return type == PutCacheEntry.class; 197 | } 198 | 199 | @Override 200 | public PutCacheEntry readFrom(Class type, Type genericType, Annotation[] annotations, 201 | MediaType mediaType, MultivaluedMap httpHeaders, InputStream entityStream) 202 | throws IOException, WebApplicationException { 203 | int sizeInBytes = 0; 204 | DataInputStream dataInputStream = new DataInputStream(entityStream); 205 | 206 | int nKeys = dataInputStream.readInt(); 207 | sizeInBytes += Integer.BYTES; 208 | 209 | String[] keys = new String[nKeys]; 210 | for (int i = 0; i < nKeys; i++) { 211 | keys[i] = dataInputStream.readUTF(); 212 | sizeInBytes += keys[i].length(); 213 | } 214 | 215 | byte[] buckData = ByteStreams.toByteArray(dataInputStream); 216 | sizeInBytes += buckData.length; 217 | 218 | return new PutCacheEntry(keys, buckData, sizeInBytes); 219 | } 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/resources/buckcache/HealthResource.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.resources.buckcache; 2 | 3 | import javax.ws.rs.GET; 4 | import javax.ws.rs.Path; 5 | import javax.ws.rs.Produces; 6 | import javax.ws.rs.core.MediaType; 7 | 8 | @Path("/health") 9 | @Produces(MediaType.TEXT_PLAIN) 10 | public class HealthResource { 11 | 12 | @GET 13 | public String getHealth() { 14 | return "OK"; 15 | } 16 | } -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/resources/buckcache/PutCacheEntry.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.resources.buckcache; 2 | 3 | import org.apache.commons.lang3.builder.ToStringBuilder; 4 | 5 | import com.uber.buckcache.datastore.CacheEntry; 6 | 7 | public class PutCacheEntry { 8 | private final String[] keys; 9 | private final CacheEntry cacheEntry; 10 | private final int sizeInBytes; 11 | 12 | public PutCacheEntry(String[] key, byte[] buckData, int sizeInBytes) { 13 | this.keys = key; 14 | this.sizeInBytes = sizeInBytes; 15 | this.cacheEntry = new CacheEntry(buckData); 16 | } 17 | 18 | public boolean verify() { 19 | return (keys != null && cacheEntry.getBuckData() != null && keys.length > 0 && cacheEntry.getBuckData().length > 0); 20 | } 21 | 22 | public String[] getKeys() { 23 | return keys; 24 | } 25 | 26 | public int getBytes() { 27 | return sizeInBytes; 28 | } 29 | 30 | public CacheEntry getCacheEntry() { 31 | return cacheEntry; 32 | } 33 | 34 | public String toString() { 35 | return ToStringBuilder.reflectionToString(this); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/utils/BytesRateLimiter.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.utils; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import com.google.common.util.concurrent.RateLimiter; 8 | 9 | public class BytesRateLimiter { 10 | private static final Logger logger = LoggerFactory.getLogger(BytesRateLimiter.class); 11 | 12 | public enum BIT_UNIT { 13 | g(1024L * 1024L * 1024L), 14 | m(1024L * 1024L), 15 | k(1024L), 16 | b(1L); 17 | 18 | private final long sizeInBytes; 19 | 20 | BIT_UNIT(long sizeInBytes) { 21 | this.sizeInBytes = sizeInBytes; 22 | } 23 | 24 | public long getNumberOfBytes() { 25 | return sizeInBytes; 26 | } 27 | } 28 | 29 | private final Double bytesLimit; 30 | private final RateLimiter rateLimiter; 31 | 32 | public BytesRateLimiter(String throttleLimit) { 33 | String multiplier = throttleLimit.substring(0, throttleLimit.length() - 1); 34 | String unit = throttleLimit.substring(throttleLimit.length() - 1, throttleLimit.length()); 35 | bytesLimit = Double.parseDouble(multiplier) * BIT_UNIT.valueOf(StringUtils.lowerCase(unit)).getNumberOfBytes(); 36 | logger.info("multiplier : {}, unit : {}, bytesLimit : {}", multiplier, unit, bytesLimit); 37 | rateLimiter = RateLimiter.create(bytesLimit); 38 | } 39 | 40 | /** 41 | * This makes sure that both upload and 42 | * download rates are capped at an instance level. 43 | * calls are blocked if we try to acquire more bytes than capped size. 44 | * @param bytes 45 | */ 46 | public void checkout(int bytes) { 47 | rateLimiter.acquire(bytes); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/utils/MetricsRegistry.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.utils; 2 | 3 | public class MetricsRegistry { 4 | public static final double SAMPLE_RATE = 1; 5 | 6 | public static final String SUMMARY_CALL_COUNT = "summary_api_call_count"; 7 | 8 | public static final String CLUSTER_SIZE = "buck_cache_cluster_size"; 9 | 10 | public static final String GET_CALL_COUNT = "get_api_call_count"; 11 | public static final String GET_CALL_TIME = "get_api_call_time_taken"; 12 | public static final String INCOMING_BYTES_TOTAL_COUNT = "incoming_request_size_in_bytes"; 13 | public static final String INCOMING_BYTES_PER_REQUEST = "incoming_bytes_per_request"; 14 | public static final String IGNITE_CACHE_GET_CALL_COUNT = "ignite_cache_get_count"; 15 | public static final String IGNITE_CACHE_GET_CALL_TIME = "ignite_cache_get_time"; 16 | public static final String CACHE_HIT_COUNT = "cache_hit_count"; 17 | public static final String CACHE_MISS_COUNT = "cache_miss_count"; 18 | 19 | public static final String PUT_CALL_COUNT = "put_api_call_count"; 20 | public static final String PUT_CALL_TIME = "put_api_call_time_taken"; 21 | public static final String OUTGOING_BYTES_TOTAL_COUNT = "outgoing_request_size_in_bytes"; 22 | public static final String OUTGOING_BYTES_PER_REQUEST = "outgoing_bytes_per_request"; 23 | public static final String IGNITE_CACHE_PUT_CALL_COUNT = "ignite_cache_put_count"; 24 | public static final String IGNITE_CACHE_PUT_CALL_TIME = "ignite_cache_put_time"; 25 | 26 | 27 | public static final String CPU_COUNT = "buck_cache_server_cpu_usage"; 28 | public static final String CPU_TIME = "buck_cache_server_cpu_usage_timer"; 29 | public static final String HEAP_COUNT = "buck_cache_server_heap_usage"; 30 | public static final String HEAP_TIME = "buck_cache_server_heap_usage_timer"; 31 | public static final String OFF_HEAP_COUNT = "buck_cache_server_off_heap_usage"; 32 | public static final String OFF_HEAP_TIME = "buck_cache_server_off_heap_usage_timer"; 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/utils/StatsDClient.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.utils; 2 | 3 | import com.timgroup.statsd.NoOpStatsDClient; 4 | import com.timgroup.statsd.NonBlockingStatsDClient; 5 | import com.uber.buckcache.StatsdConfig; 6 | 7 | public class StatsDClient { 8 | private static StatsDClient statsDClient; 9 | 10 | private final StatsdConfig statsDConfig; 11 | private final com.timgroup.statsd.StatsDClient underlyingClient; 12 | 13 | private StatsDClient(StatsdConfig statsDConfig) { 14 | this.statsDConfig = statsDConfig; 15 | 16 | if (this.statsDConfig.isEnabled()) { 17 | underlyingClient = new NonBlockingStatsDClient(this.statsDConfig.getPrefix(), this.statsDConfig.getHost(), 18 | this.statsDConfig.getPort()); 19 | } else { 20 | underlyingClient = new NoOpStatsDClient(); 21 | } 22 | } 23 | 24 | public synchronized static void init(StatsdConfig statsDConfig) { 25 | if (statsDClient == null) { 26 | statsDClient = new StatsDClient(statsDConfig); 27 | } 28 | } 29 | 30 | public static StatsDClient get() { 31 | return statsDClient; 32 | } 33 | 34 | public void count(String metricName, long countValue) { 35 | underlyingClient.count(metricName, countValue, statsDConfig.getSampleRate()); 36 | } 37 | 38 | public void gauge(String metricName, long countValue) { 39 | underlyingClient.gauge(metricName, countValue); 40 | } 41 | 42 | public void recordExecutionTime(String metricName, long timeDifferenceInMillis) { 43 | underlyingClient.recordExecutionTime(metricName, timeDifferenceInMillis, statsDConfig.getSampleRate()); 44 | } 45 | 46 | public void recordExecutionTimeToNow(String metricName, long startTime) { 47 | underlyingClient.recordExecutionTime(metricName, System.currentTimeMillis() - startTime, 48 | statsDConfig.getSampleRate()); 49 | } 50 | 51 | public com.timgroup.statsd.StatsDClient getUnderlying() { 52 | return underlyingClient; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cache/src/main/java/com/uber/buckcache/utils/StatsDClientFactory.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.utils; 2 | 3 | public class StatsDClientFactory { 4 | } 5 | -------------------------------------------------------------------------------- /cache/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | /$$$$$$$ /$$ /$$ /$$$$$$ /$$ /$$ /$$$$$$ /$$$$$$ /$$$$$$ /$$ /$$ /$$$$$$$$ 2 | | $$__ $$| $$ | $$ /$$__ $$| $$ /$$/ /$$__ $$ /$$__ $$ /$$__ $$| $$ | $$| $$_____/ 3 | | $$ \ $$| $$ | $$| $$ \__/| $$ /$$/ | $$ \__/| $$ \ $$| $$ \__/| $$ | $$| $$ 4 | | $$$$$$$ | $$ | $$| $$ | $$$$$/ | $$ | $$$$$$$$| $$ | $$$$$$$$| $$$$$ 5 | | $$__ $$| $$ | $$| $$ | $$ $$ | $$ | $$__ $$| $$ | $$__ $$| $$__/ 6 | | $$ \ $$| $$ | $$| $$ $$| $$\ $$ | $$ $$| $$ | $$| $$ $$| $$ | $$| $$ 7 | | $$$$$$$/| $$$$$$/| $$$$$$/| $$ \ $$ | $$$$$$/| $$ | $$| $$$$$$/| $$ | $$| $$$$$$$$ 8 | |_______/ \______/ \______/ |__/ \__/ \______/ |__/ |__/ \______/ |__/ |__/|________/ 9 | -------------------------------------------------------------------------------- /cache/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout, file 3 | 4 | # Direct log messages to a log file 5 | #log4j.appender.file=org.apache.log4j.RollingFileAppender 6 | #log4j.appender.file.File=/var/log/buck-cache-client/logs/buck-main.log 7 | #log4j.appender.file.MaxBackupIndex=10 8 | #log4j.appender.file.layout=org.apache.log4j.PatternLayout 9 | #log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 10 | 11 | # Direct log messages to stdout 12 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 13 | log4j.appender.stdout.Target=System.out 14 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 15 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /cache/src/test/java/com/uber/buckcache/auth/HttpHeaderAuthFilterTest.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.auth; 2 | 3 | import org.junit.BeforeClass; 4 | import org.junit.Test; 5 | 6 | import javax.ws.rs.NotAuthorizedException; 7 | import javax.ws.rs.container.ContainerRequestContext; 8 | import javax.ws.rs.core.HttpHeaders; 9 | import javax.ws.rs.core.MultivaluedHashMap; 10 | import javax.ws.rs.core.MultivaluedMap; 11 | 12 | import java.io.IOException; 13 | import java.util.Arrays; 14 | 15 | import static org.mockito.Mockito.mock; 16 | import static org.mockito.Mockito.when; 17 | 18 | public class HttpHeaderAuthFilterTest { 19 | private static ContainerRequestContext mockRequestContextWithHeader; 20 | private static ContainerRequestContext mockRequestContextWithoutHeader; 21 | 22 | @BeforeClass 23 | public static void setup() throws Exception { 24 | mockRequestContextWithHeader = mock(ContainerRequestContext.class); 25 | MultivaluedMap mockHeaders = new MultivaluedHashMap(); 26 | mockHeaders.add(HttpHeaders.AUTHORIZATION, "testToken"); 27 | when(mockRequestContextWithHeader.getHeaders()).thenReturn(mockHeaders); 28 | 29 | mockRequestContextWithoutHeader = mock(ContainerRequestContext.class); 30 | when(mockRequestContextWithoutHeader.getHeaders()).thenReturn(new MultivaluedHashMap<>()); 31 | } 32 | 33 | @Test 34 | public void testSuccessWithEmptyAuthenticatedToken() throws IOException { 35 | (new HttpHeaderAuthFilter()).filter(mockRequestContextWithHeader); 36 | } 37 | 38 | @Test 39 | public void testSuccessWithNullAuthenticatedToken() throws IOException { 40 | (new HttpHeaderAuthFilter(null)).filter(mockRequestContextWithHeader); 41 | } 42 | 43 | @Test 44 | public void testSuccessWithAuthenticatedToken() throws IOException { 45 | (new HttpHeaderAuthFilter(Arrays.asList("testToken"))).filter(mockRequestContextWithHeader); 46 | } 47 | 48 | @Test(expected=NotAuthorizedException.class) 49 | public void testFailWithIncorrectAuthenticatedToken() throws IOException { 50 | (new HttpHeaderAuthFilter(Arrays.asList("testTokenFail"))).filter(mockRequestContextWithHeader); 51 | } 52 | 53 | @Test(expected=NotAuthorizedException.class) 54 | public void testFailWithoutAuthenticatedToken() throws IOException { 55 | (new HttpHeaderAuthFilter(Arrays.asList("testToken"))).filter(mockRequestContextWithoutHeader); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /cache/src/test/java/com/uber/buckcache/datastore/impl/ignite/IgniteConfigurationBuilderTest.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.datastore.impl.ignite; 2 | 3 | import com.spotify.dns.DnsSrvResolver; 4 | import com.spotify.dns.LookupResult; 5 | import java.net.InetSocketAddress; 6 | import java.util.ArrayList; 7 | import java.util.Arrays; 8 | import java.util.Collection; 9 | import java.util.List; 10 | 11 | import org.apache.ignite.configuration.IgniteConfiguration; 12 | import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; 13 | import org.junit.Assert; 14 | import org.junit.Test; 15 | 16 | import static org.mockito.Mockito.mock; 17 | import static org.mockito.Mockito.when; 18 | 19 | public class IgniteConfigurationBuilderTest { 20 | 21 | private static IgniteConfigurationBuilder builder; 22 | 23 | @Test 24 | public void testAddMulticastBasedDiscroveryWithoutDNSLookup() throws Exception { 25 | IgniteConfiguration configuration = builder.addMulticastBasedDiscrovery( 26 | "", 27 | 0, 28 | Arrays.asList("127.0.0.2"), 29 | null).build(); 30 | TcpDiscoverySpi spi = (TcpDiscoverySpi) configuration.getDiscoverySpi(); 31 | Collection addrs = spi.getIpFinder().getRegisteredAddresses(); 32 | Assert.assertEquals(1, addrs.size()); 33 | Assert.assertEquals("/127.0.0.2:0", addrs.toArray()[0].toString()); 34 | } 35 | 36 | @Test 37 | public void testAddMulticastBasedDiscroveryWithDNSLookup() throws Exception { 38 | List mockAddress = new ArrayList(); 39 | mockAddress.add(LookupResult.create("127.0.0.3", 1001, 0, 0, 15)); 40 | mockAddress.add(LookupResult.create("127.0.0.4", 1001, 0, 0, 15)); 41 | DnsSrvResolver resolver = mock(DnsSrvResolver.class); 42 | when(resolver.resolve("testDNSAddress")).thenReturn(mockAddress); 43 | builder = new IgniteConfigurationBuilder(resolver); 44 | 45 | IgniteConfiguration configuration = builder.addMulticastBasedDiscrovery( 46 | "", 47 | 0, 48 | new ArrayList(Arrays.asList("127.0.0.2")), 49 | "testDNSAddress").build(); 50 | 51 | TcpDiscoverySpi spi = (TcpDiscoverySpi) configuration.getDiscoverySpi(); 52 | Collection addrs = spi.getIpFinder().getRegisteredAddresses(); 53 | Assert.assertEquals(3, addrs.size()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /cache/src/test/java/com/uber/buckcache/integration/SingleServerIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.integration; 2 | 3 | import com.fasterxml.uuid.Generators; 4 | import com.fasterxml.uuid.impl.TimeBasedGenerator; 5 | import com.uber.buckcache.BuckCacheApplication; 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.util.Arrays; 9 | import java.util.Random; 10 | import org.apache.http.HttpEntity; 11 | import org.apache.http.client.ClientProtocolException; 12 | import org.apache.http.client.methods.CloseableHttpResponse; 13 | import org.apache.http.client.methods.HttpGet; 14 | import org.apache.http.client.methods.HttpPut; 15 | import org.apache.http.entity.FileEntity; 16 | import org.apache.http.entity.StringEntity; 17 | import org.apache.http.impl.client.CloseableHttpClient; 18 | import org.apache.http.impl.client.HttpClients; 19 | import org.apache.http.message.BasicHeader; 20 | import org.junit.Assert; 21 | import org.junit.BeforeClass; 22 | import org.junit.Test; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import javax.ws.rs.core.HttpHeaders; 27 | 28 | public class SingleServerIntegrationTest { 29 | private static final Logger logger = LoggerFactory.getLogger(SingleServerIntegrationTest.class); 30 | 31 | private static CloseableHttpClient httpclient; 32 | private String authenticatedToken = "testToken"; 33 | 34 | @BeforeClass 35 | public static void setup() throws Exception { 36 | httpclient = HttpClients.createDefault(); 37 | File configFile = 38 | new File(SingleServerIntegrationTest.class.getClassLoader().getResource("config/test-server-1.yml").getFile()); 39 | BuckCacheApplication.main(new String[] { 40 | "server", configFile.getAbsolutePath() 41 | }); 42 | } 43 | 44 | @Test 45 | public void testCacheupload() throws ClientProtocolException, IOException { 46 | File dataDir = new File(this.getClass().getClassLoader().getResource("cache_data").getFile()); 47 | for (File dataFile : dataDir.listFiles()) { 48 | logger.info("running testupload on file : {}", dataFile); 49 | HttpPut httpput = new HttpPut("http://localhost:8090/artifacts/key"); 50 | HttpEntity e = new FileEntity(dataFile); 51 | httpput.setEntity(e); 52 | httpput.setHeader(HttpHeaders.AUTHORIZATION, authenticatedToken); 53 | CloseableHttpResponse putResponse = httpclient.execute(httpput); 54 | try { 55 | Assert.assertEquals(202, putResponse.getStatusLine().getStatusCode()); 56 | } finally { 57 | putResponse.close(); 58 | } 59 | } 60 | } 61 | 62 | @Test 63 | public void testCacheUploadFailAuthentication() throws IOException { 64 | File dataFile = new File(this.getClass().getClassLoader().getResource("cache_data/1473551747919").getFile()); 65 | 66 | logger.info("running testupload on file : {}", dataFile); 67 | HttpPut httpput = new HttpPut("http://localhost:8090/artifacts/key"); 68 | HttpEntity e = new FileEntity(dataFile); 69 | httpput.setEntity(e); 70 | CloseableHttpResponse putResponse = httpclient.execute(httpput); 71 | try { 72 | Assert.assertEquals(401, putResponse.getStatusLine().getStatusCode()); 73 | } finally { 74 | putResponse.close(); 75 | } 76 | } 77 | 78 | @Test 79 | public void testCacheDownload() throws IOException { 80 | File dataDir = new File(SingleServerIntegrationTest.class.getClassLoader().getResource("cache_data").getFile()); 81 | for (File dataFile : dataDir.listFiles()) { 82 | logger.info("running testDownload on file : {}", dataFile); 83 | // lets upload again just to make sure 84 | HttpPut httpput = new HttpPut("http://localhost:8090/artifacts/key"); 85 | HttpEntity e = new FileEntity(dataFile); 86 | httpput.setEntity(e); 87 | httpput.setHeader(HttpHeaders.AUTHORIZATION, authenticatedToken); 88 | CloseableHttpResponse putResponse = httpclient.execute(httpput); 89 | try { 90 | Assert.assertEquals(202, putResponse.getStatusLine().getStatusCode()); 91 | } finally { 92 | putResponse.close(); 93 | } 94 | 95 | String[] keys = TestUtils.getCacheKeysForDataFile(dataFile); 96 | // now lets make sure that each key is found 97 | logger.info("get query with keys : {}", Arrays.toString(keys)); 98 | for (String key : keys) { 99 | HttpGet httpget = new HttpGet(String.format("http://localhost:8090/artifacts/key/%s", key)); 100 | CloseableHttpResponse getResponse = httpclient.execute(httpget); 101 | try { 102 | Assert.assertEquals(200, getResponse.getStatusLine().getStatusCode()); 103 | } finally { 104 | getResponse.close(); 105 | } 106 | } 107 | } 108 | } 109 | 110 | @Test 111 | public void testDataNotFound() throws ClientProtocolException, IOException { 112 | TimeBasedGenerator uuidGenerator = Generators.timeBasedGenerator(); 113 | for (int i = 0; i < 10; i++) { 114 | HttpGet httpget = 115 | new HttpGet(String.format("http://localhost:8090/artifacts/key/%s", uuidGenerator.generate().toString())); 116 | CloseableHttpResponse getResponse = httpclient.execute(httpget); 117 | try { 118 | Assert.assertEquals(404, getResponse.getStatusLine().getStatusCode()); 119 | } finally { 120 | getResponse.close(); 121 | } 122 | } 123 | } 124 | 125 | @Test 126 | public void testBadDataUpload() throws ClientProtocolException, IOException { 127 | TimeBasedGenerator uuidGenerator = Generators.timeBasedGenerator(); 128 | for (int i = 0; i < 10; i++) { 129 | HttpPut httpput = new HttpPut("http://localhost:8090/artifacts/key"); 130 | HttpEntity e = new StringEntity(String.format("1%sBadData", uuidGenerator.generate().toString())); 131 | httpput.setEntity(e); 132 | CloseableHttpResponse putResponse = httpclient.execute(httpput); 133 | try { 134 | Assert.assertEquals(true, putResponse.getStatusLine().getStatusCode() > 400); 135 | } finally { 136 | putResponse.close(); 137 | } 138 | } 139 | } 140 | 141 | @Test 142 | public void testCacheDataValidity() throws IOException { 143 | File dataDir = new File(SingleServerIntegrationTest.class.getClassLoader().getResource("cache_data").getFile()); 144 | for (File dataFile : dataDir.listFiles()) { 145 | logger.info("running testDownload on file : {}", dataFile); 146 | // lets upload again just to make sure 147 | HttpPut httpput = new HttpPut("http://localhost:8090/artifacts/key"); 148 | HttpEntity e = new FileEntity(dataFile); 149 | httpput.setEntity(e); 150 | httpput.setHeader(HttpHeaders.AUTHORIZATION, authenticatedToken); 151 | CloseableHttpResponse putResponse = httpclient.execute(httpput); 152 | try { 153 | Assert.assertEquals(202, putResponse.getStatusLine().getStatusCode()); 154 | } finally { 155 | putResponse.close(); 156 | } 157 | 158 | String[] keys = TestUtils.getCacheKeysForDataFile(dataFile); 159 | String key = keys[new Random().nextInt(keys.length)]; 160 | 161 | // now lets make sure that each key is found 162 | logger.info("get query with keys : {}", Arrays.toString(keys)); 163 | HttpGet httpget = new HttpGet(String.format("http://localhost:8090/artifacts/key/%s", key)); 164 | CloseableHttpResponse getResponse = httpclient.execute(httpget); 165 | try { 166 | Assert.assertEquals(200, getResponse.getStatusLine().getStatusCode()); 167 | String incomingDataFileMD5 = TestUtils.getMD5AfterStrippingKeys(dataFile); 168 | String outgoingDataFileMD5 = TestUtils.getMDFForEntireStream(getResponse.getEntity().getContent()); 169 | Assert.assertEquals(incomingDataFileMD5, outgoingDataFileMD5); 170 | } finally { 171 | getResponse.close(); 172 | } 173 | } 174 | } 175 | 176 | @Test 177 | public void testCacheuploadWithCustomExpiry() throws Exception { 178 | File dataDir = new File(this.getClass().getClassLoader().getResource("cache_data").getFile()); 179 | File dataFile = dataDir.listFiles()[0]; 180 | logger.info("running testupload on file : {}", dataFile); 181 | HttpPut httpput = new HttpPut("http://localhost:8090/artifacts/key"); 182 | HttpEntity e = new FileEntity(dataFile); 183 | httpput.setEntity(e); 184 | httpput.setHeader(HttpHeaders.AUTHORIZATION, authenticatedToken); 185 | httpput.addHeader(new BasicHeader("X-Cache-Expiry-Seconds", "0")); // Expire keys immediately 186 | CloseableHttpResponse putResponse = httpclient.execute(httpput); 187 | try { 188 | Assert.assertEquals(202, putResponse.getStatusLine().getStatusCode()); 189 | } finally { 190 | putResponse.close(); 191 | } 192 | 193 | String[] keys = TestUtils.getCacheKeysForDataFile(dataFile); 194 | String key = keys[new Random().nextInt(keys.length)]; 195 | 196 | // make sure key does not exist after expiration time 197 | logger.info("get query with keys : {}", Arrays.toString(keys)); 198 | HttpGet httpget = new HttpGet(String.format("http://localhost:8090/artifacts/key/%s", key)); 199 | CloseableHttpResponse getResponse = httpclient.execute(httpget); 200 | try { 201 | Assert.assertEquals(404, getResponse.getStatusLine().getStatusCode()); 202 | } finally { 203 | getResponse.close(); 204 | } 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /cache/src/test/java/com/uber/buckcache/integration/TestUtils.java: -------------------------------------------------------------------------------- 1 | package com.uber.buckcache.integration; 2 | 3 | import java.io.DataInputStream; 4 | import java.io.File; 5 | import java.io.FileInputStream; 6 | import java.io.FileNotFoundException; 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.util.Arrays; 10 | 11 | import org.apache.commons.codec.digest.DigestUtils; 12 | import org.apache.http.HttpEntity; 13 | import org.apache.http.client.ClientProtocolException; 14 | import org.apache.http.client.methods.CloseableHttpResponse; 15 | import org.apache.http.client.methods.HttpPut; 16 | import org.apache.http.entity.FileEntity; 17 | import org.apache.http.impl.client.CloseableHttpClient; 18 | import org.apache.http.impl.client.HttpClients; 19 | import org.junit.Assert; 20 | 21 | public class TestUtils { 22 | 23 | public static String[] getCacheKeysForDataFile(DataInputStream ds) throws IOException { 24 | int numKeys = ds.readInt(); 25 | String[] keys = new String[numKeys]; 26 | for (int i = 0; i < keys.length; i++) { 27 | keys[i] = ds.readUTF(); 28 | } 29 | ds.close(); 30 | return keys; 31 | } 32 | 33 | public static String[] getCacheKeysForDataFile(File dataFile) throws IOException { 34 | return getCacheKeysForDataFile(new DataInputStream(new FileInputStream(dataFile))); 35 | } 36 | 37 | public static String getMDFForEntireStream(InputStream dis) throws IOException { 38 | String md5 = DigestUtils.md5Hex(dis); 39 | dis.close(); 40 | return md5; 41 | } 42 | 43 | public static String getMD5AfterStrippingKeys(File dataFile) throws IOException { 44 | DataInputStream dis = new DataInputStream(new FileInputStream(dataFile)); 45 | 46 | // lets skip the keys 47 | int numKeys = dis.readInt(); 48 | String[] keys = new String[numKeys]; 49 | for (int i = 0; i < keys.length; i++) { 50 | keys[i] = dis.readUTF(); 51 | } 52 | 53 | String md5 = DigestUtils.md5Hex(dis); 54 | dis.close(); 55 | return md5; 56 | } 57 | 58 | public static void main(String[] args) throws ClientProtocolException, IOException { 59 | CloseableHttpClient httpclient = HttpClients.createDefault(); 60 | File dataDir = new File(TestUtils.class.getClassLoader().getResource("cache_data").getFile()); 61 | for (File dataFile : dataDir.listFiles()) { 62 | HttpPut httpput = new HttpPut("http://localhost:6457/artifacts/key"); 63 | HttpEntity e = new FileEntity(dataFile); 64 | httpput.setEntity(e); 65 | CloseableHttpResponse putResponse = httpclient.execute(httpput); 66 | try { 67 | Assert.assertEquals(202, putResponse.getStatusLine().getStatusCode()); 68 | } finally { 69 | putResponse.close(); 70 | } 71 | 72 | System.out.println(Arrays.toString(getCacheKeysForDataFile(new DataInputStream(new FileInputStream(dataFile))))); 73 | break; 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747919: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747919 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747939: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747939 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747950: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747950 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747956: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747956 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747961: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747961 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747965: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747965 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747969: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747969 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747973: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747973 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747980: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747980 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747983: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747983 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747987: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747987 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551747991: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551747991 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748673: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748673 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748681: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748681 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748688: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748688 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748707: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748707 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748733: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748733 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748747: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748747 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748787: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748787 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748849: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748849 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748852: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748852 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748856: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748856 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551748863: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551748863 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749391: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749391 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749402: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749402 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749410: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749410 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749414: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749414 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749423: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749423 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749428: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749428 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749433: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749433 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749436: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749436 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749440: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749440 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749443: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749443 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749448: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749448 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749463: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749463 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749467: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749467 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749470: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749470 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749473: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749473 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749479: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749479 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749482: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749482 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749485: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749485 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749490: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749490 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749529: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749529 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749533: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749533 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749537: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749537 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749540: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749540 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749544: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749544 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749550: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749550 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749553: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749553 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749557: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749557 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749560: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749560 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749566: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749566 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749571: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749571 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749575: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749575 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749578: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749578 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749582: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749582 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749585: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749585 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749589: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749589 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749595: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749595 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749598: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749598 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749602: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749602 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749605: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749605 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749610: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749610 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749655: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749655 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749953: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749953 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749957: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749957 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749965: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749965 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749972: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749972 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749977: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749977 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749982: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749982 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749991: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749991 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749994: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749994 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551749997: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551749997 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750001 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750004: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750004 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750008: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750008 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750012: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750012 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750016: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750016 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750019: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750019 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750025: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750025 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750033: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750033 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750037: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750037 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750048: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750048 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750051: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750051 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750056: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750056 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750061: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750061 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750066: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750066 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750082: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750082 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750086: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750086 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750097: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750097 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750116: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750116 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750130: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750130 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750145: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750145 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750153: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750153 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750222: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750222 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750431: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750431 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750451: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750451 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750483: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750483 -------------------------------------------------------------------------------- /cache/src/test/resources/cache_data/1473551750498: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/cache/src/test/resources/cache_data/1473551750498 -------------------------------------------------------------------------------- /cache/src/test/resources/config/test-server-1.yml: -------------------------------------------------------------------------------- 1 | server: 2 | applicationConnectors: 3 | - type: http 4 | port: 8090 5 | adminConnectors: 6 | - type: http 7 | port: 8091 8 | 9 | mode: DUAL 10 | 11 | # per instance, per second payload throttle limit 12 | # available units, b,k,m & g 13 | throttleLimit: 500m 14 | 15 | authenticationConfig: 16 | tokens: 17 | - testToken 18 | 19 | storeProviderKlass: com.uber.buckcache.datastore.impl.ignite.IgniteDataStoreProvider 20 | 21 | storeProviderConfig: 22 | config: 23 | multicastIP: 228.10.10.157 24 | multicastPort: 6734 25 | hostIPs: 26 | - 127.0.0.1 27 | cacheMode: PARTITIONED 28 | cacheBackupCount: 1 29 | expirationTimeUnit: DAYS 30 | expirationTimeValue: 7 31 | atomicSequenceReserveSize: 10000 32 | offHeapStorageSize: 40g # 40GB 33 | 34 | statsd: 35 | enabled: false 36 | -------------------------------------------------------------------------------- /cache/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /classes/production/cache/banner.txt: -------------------------------------------------------------------------------- 1 | /$$$$$$$ /$$ /$$ /$$$$$$ /$$ /$$ /$$$$$$ /$$$$$$ /$$$$$$ /$$ /$$ /$$$$$$$$ 2 | | $$__ $$| $$ | $$ /$$__ $$| $$ /$$/ /$$__ $$ /$$__ $$ /$$__ $$| $$ | $$| $$_____/ 3 | | $$ \ $$| $$ | $$| $$ \__/| $$ /$$/ | $$ \__/| $$ \ $$| $$ \__/| $$ | $$| $$ 4 | | $$$$$$$ | $$ | $$| $$ | $$$$$/ | $$ | $$$$$$$$| $$ | $$$$$$$$| $$$$$ 5 | | $$__ $$| $$ | $$| $$ | $$ $$ | $$ | $$__ $$| $$ | $$__ $$| $$__/ 6 | | $$ \ $$| $$ | $$| $$ $$| $$\ $$ | $$ $$| $$ | $$| $$ $$| $$ | $$| $$ 7 | | $$$$$$$/| $$$$$$/| $$$$$$/| $$ \ $$ | $$$$$$/| $$ | $$| $$$$$$/| $$ | $$| $$$$$$$$ 8 | |_______/ \______/ \______/ |__/ \__/ \______/ |__/ |__/ \______/ |__/ |__/|________/ 9 | -------------------------------------------------------------------------------- /dependencies.gradle: -------------------------------------------------------------------------------- 1 | def versions = [ 2 | igniteVersion: "1.9.0", 3 | dropwizardVersion: "1.0.0", 4 | junitVersion: "4.12", 5 | httpClientVersion: "4.5.2", 6 | httpCoreVersion: "4.4.4", 7 | jacksonVersion: "2.8.2", 8 | uuidGenVersion: "3.1.3", 9 | statsdClientVersion: "3.0.1", 10 | dnsVersion: "3.1.4", 11 | ] 12 | 13 | def build = [ 14 | oneJar : "com.github.rholder:gradle-one-jar:1.0.4", 15 | cobertura: "net.saliman:gradle-cobertura-plugin:2.3.2" 16 | ] 17 | 18 | def external = [ 19 | dropwizard : "io.dropwizard:dropwizard-core:${versions.dropwizardVersion}", 20 | dropwizardAuth : "io.dropwizard:dropwizard-auth:${versions.dropwizardVersion}", 21 | ignite : "org.apache.ignite:ignite-core:${versions.igniteVersion}", 22 | junit : "junit:junit:${versions.junitVersion}", 23 | httpClient : "org.apache.httpcomponents:httpclient:${versions.httpClientVersion}", 24 | httpCore : "org.apache.httpcomponents:httpcore:${versions.httpCoreVersion}", 25 | jackson : "com.fasterxml.jackson.core:jackson-databind:${versions.jacksonVersion}", 26 | uuidGen : "com.fasterxml.uuid:java-uuid-generator:${versions.uuidGenVersion}", 27 | statsdClient : "com.timgroup:java-statsd-client:${versions.statsdClientVersion}", 28 | dns : "com.spotify:dns:${versions.dnsVersion}", 29 | ] 30 | 31 | ext.deps = [ 32 | "build" : build, 33 | "external": external, 34 | "versions": versions 35 | ] 36 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/buck-http-cache/70d7940b54e0d169d18f47ee89fcc6f5fb6d522a/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Aug 22 17:04:25 CEST 2016 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn ( ) { 37 | echo "$*" 38 | } 39 | 40 | die ( ) { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | -------------------------------------------------------------------------------- /run_buck_cache_client.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ./gradlew distJar 4 | 5 | $JAVA_HOME/bin/java -Xmx16g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:MaxDirectMemorySize=6g -Dlog.home=/var/log/buck-cache-client/logs -jar cache/build/libs/cache-1.0.0-standalone.jar server cache/src/dist/config/$1.yml 6 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include ':cache' 2 | --------------------------------------------------------------------------------