├── GROUPS ├── util-zk-common ├── GROUPS ├── BUILD └── src │ ├── test │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── zk │ │ └── CommonConnectorTest.scala │ └── main │ └── scala │ ├── BUILD │ └── com │ └── twitter │ ├── conversions │ └── common │ │ ├── zookeeper.scala │ │ └── quantity.scala │ └── zk │ └── CommonConnector.scala ├── OWNERS ├── project ├── build.properties └── plugins.sbt ├── util-zk ├── OWNERS ├── GROUPS ├── BUILD └── src │ ├── main │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── zk │ │ ├── AuthInfo.scala │ │ ├── LiftableFuture.scala │ │ ├── ZOp.scala │ │ ├── AsyncCallbackPromise.scala │ │ └── RetryPolicy.scala │ └── test │ └── scala │ ├── BUILD │ └── com │ └── twitter │ └── zk │ ├── ZNodeTest.scala │ └── ConnectorTest.scala ├── util-eval ├── src │ ├── test │ │ ├── resources │ │ │ ├── OnePlusOne.scala │ │ │ ├── Derived.scala │ │ │ ├── file-with-dash.scala │ │ │ ├── DerivedWithInclude.scala │ │ │ ├── RubyInclude.scala │ │ │ ├── Base.scala │ │ │ ├── hello.rb │ │ │ ├── BUILD │ │ │ ├── HelloJoe.scala │ │ │ ├── Deprecated.scala │ │ │ └── IncludeInclude.scala │ │ └── scala │ │ │ └── BUILD │ └── main │ │ └── scala │ │ └── BUILD └── BUILD ├── util-app ├── OWNERS ├── src │ ├── main │ │ ├── java │ │ │ ├── BUILD │ │ │ └── com │ │ │ │ └── twitter │ │ │ │ └── app │ │ │ │ └── GlobalFlagVisible.java │ │ └── scala │ │ │ └── BUILD │ └── test │ │ ├── scala │ │ ├── BUILD │ │ └── com │ │ │ └── twitter │ │ │ └── app │ │ │ └── AppTest.scala │ │ └── java │ │ └── BUILD └── BUILD ├── util-codec ├── OWNERS ├── src │ ├── test │ │ └── scala │ │ │ └── BUILD │ └── main │ │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── util │ │ └── StringEncoder.scala └── BUILD ├── util-benchmark ├── OWNERS ├── BUILD └── src │ └── main │ └── scala │ ├── BUILD │ └── com │ └── twitter │ ├── util │ ├── events │ │ └── SinkBenchmark.scala │ └── benchmark │ │ ├── Stopwatch.scala │ │ └── CpuProfile.scala │ ├── concurrent │ └── OfferBenchmark.scala │ └── io │ └── benchmark │ └── Buf.scala ├── util-events ├── OWNERS ├── src │ ├── test │ │ └── scala │ │ │ ├── BUILD │ │ │ └── com │ │ │ └── twitter │ │ │ └── util │ │ │ └── events │ │ │ ├── SinkTest.scala │ │ │ └── SizedSinkTest.scala │ └── main │ │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── util │ │ └── events │ │ ├── package.scala │ │ ├── Event.scala │ │ └── Sink.scala └── BUILD ├── util-logging ├── OWNERS ├── src │ ├── test │ │ ├── java │ │ │ ├── BUILD │ │ │ └── com │ │ │ │ └── twitter │ │ │ │ └── logging │ │ │ │ ├── LoggerCompilationTest.java │ │ │ │ └── LoggerFactoryTest.java │ │ └── scala │ │ │ ├── BUILD │ │ │ └── com │ │ │ └── twitter │ │ │ └── logging │ │ │ ├── PolicyTest.scala │ │ │ ├── AppTest.scala │ │ │ └── LogRecordTest.scala │ └── main │ │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── logging │ │ ├── package.scala │ │ ├── config │ │ └── package.scala │ │ └── LazyLogRecord.scala └── BUILD ├── util-core ├── OWNERS ├── src │ ├── main │ │ ├── scala │ │ │ ├── com │ │ │ │ └── twitter │ │ │ │ │ ├── io │ │ │ │ │ ├── OWNERS │ │ │ │ │ ├── TempDirectory.scala │ │ │ │ │ ├── BufReader.scala │ │ │ │ │ ├── StreamIO.scala │ │ │ │ │ ├── Files.scala │ │ │ │ │ ├── TempFile.scala │ │ │ │ │ └── BufInputStream.scala │ │ │ │ │ ├── concurrent │ │ │ │ │ ├── Permit.scala │ │ │ │ │ ├── AsyncMutex.scala │ │ │ │ │ ├── Serialized.scala │ │ │ │ │ ├── NamedPoolThreadFactory.scala │ │ │ │ │ ├── ConcurrentMultiMap.scala │ │ │ │ │ ├── ConcurrentPool.scala │ │ │ │ │ └── ConcurrentBijection.scala │ │ │ │ │ ├── util │ │ │ │ │ ├── JavaSingleton.scala │ │ │ │ │ ├── TimeoutException.scala │ │ │ │ │ ├── Unsafe.scala │ │ │ │ │ ├── NoStacktrace.scala │ │ │ │ │ ├── CountDownLatch.scala │ │ │ │ │ ├── StateMachine.scala │ │ │ │ │ ├── package.scala │ │ │ │ │ ├── Stopwatch.scala │ │ │ │ │ ├── LongOverflowArith.scala │ │ │ │ │ ├── RandomSocket.scala │ │ │ │ │ ├── NonFatal.scala │ │ │ │ │ ├── Cancellable.scala │ │ │ │ │ ├── TempFolder.scala │ │ │ │ │ ├── Codec.scala │ │ │ │ │ ├── Bijection.scala │ │ │ │ │ └── LastWriteWinsQueue.scala │ │ │ │ │ └── conversions │ │ │ │ │ ├── thread.scala │ │ │ │ │ ├── storage.scala │ │ │ │ │ └── time.scala │ │ │ └── BUILD │ │ └── java │ │ │ ├── BUILD │ │ │ └── com │ │ │ └── twitter │ │ │ ├── util │ │ │ ├── Events.java │ │ │ ├── Witnesses.java │ │ │ ├── Vars.java │ │ │ └── Closables.java │ │ │ └── concurrent │ │ │ ├── Spools.java │ │ │ └── Offers.java │ └── test │ │ ├── scala │ │ ├── BUILD │ │ └── com │ │ │ └── twitter │ │ │ ├── util │ │ │ ├── HandledPromise.scala │ │ │ ├── CancellableTest.scala │ │ │ ├── LastWriteWinsQueueTest.scala │ │ │ ├── FutureOfferTest.scala │ │ │ ├── BijectionTest.scala │ │ │ ├── ClosableTest.scala │ │ │ ├── DiffTest.scala │ │ │ ├── AwaitableTest.scala │ │ │ ├── StateMachineTest.scala │ │ │ ├── Base64LongTest.scala │ │ │ ├── CredentialsTest.scala │ │ │ └── StringConversionsTest.scala │ │ │ ├── conversions │ │ │ └── time.scala │ │ │ ├── concurrent │ │ │ ├── AsyncMutexTest.scala │ │ │ ├── ConcurrentMultiMapTest.scala │ │ │ ├── ConcurrentPoolTest.scala │ │ │ ├── SerializedTest.scala │ │ │ ├── ConcurrentBijectionTest.scala │ │ │ ├── SchedulerTest.scala │ │ │ └── TxTest.scala │ │ │ └── io │ │ │ ├── StreamIOTest.scala │ │ │ ├── BufReaderTest.scala │ │ │ └── FilesTest.scala │ │ ├── java │ │ ├── BUILD │ │ └── com │ │ │ └── twitter │ │ │ ├── util │ │ │ ├── FunctionCompilationTest.java │ │ │ ├── ActivityCompilationTest.java │ │ │ ├── ClosableCompilationTest.java │ │ │ └── AwaitableCompilationTest.java │ │ │ └── concurrent │ │ │ └── SpoolCompilationTest.java │ │ └── BUILD └── BUILD ├── CONFIG.ini ├── util-stats ├── src │ ├── main │ │ └── scala │ │ │ ├── com │ │ │ └── twitter │ │ │ │ └── finagle │ │ │ │ └── stats │ │ │ │ ├── OWNERS │ │ │ │ ├── package.scala │ │ │ │ ├── StatsRegistry.scala │ │ │ │ ├── RollupStatsReceiver.scala │ │ │ │ └── InMemoryStatsReceiver.scala │ │ │ └── BUILD │ └── test │ │ ├── java │ │ ├── BUILD │ │ └── com │ │ │ └── twitter │ │ │ └── finagle │ │ │ └── stats │ │ │ └── StatsReceiverCompilationTest.java │ │ └── scala │ │ └── BUILD └── BUILD ├── util-hashing ├── src │ ├── test │ │ ├── resources │ │ │ └── BUILD │ │ └── scala │ │ │ ├── BUILD │ │ │ └── com │ │ │ └── twitter │ │ │ └── hashing │ │ │ ├── HashableTest.scala │ │ │ ├── DistributionTester.scala │ │ │ ├── DiagnosticsTest.scala │ │ │ └── KeyHasherTest.scala │ └── main │ │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── hashing │ │ ├── Distributor.scala │ │ └── KeyHasher.scala ├── BUILD └── README.markdown ├── util-thrift ├── src │ ├── test │ │ ├── java │ │ │ └── BUILD │ │ └── scala │ │ │ └── BUILD │ └── main │ │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── util │ │ ├── ThriftCodec.scala │ │ └── ThriftSerializer.scala └── BUILD ├── util-class-preloader ├── BUILD └── src │ └── main │ └── java │ ├── BUILD │ └── com │ └── twitter │ └── class-preloader │ └── manifest ├── util-cache ├── src │ ├── test │ │ ├── java │ │ │ ├── BUILD │ │ │ └── com │ │ │ │ └── twitter │ │ │ │ └── cache │ │ │ │ └── JavaRefreshTest.java │ │ └── scala │ │ │ ├── BUILD │ │ │ └── com │ │ │ └── twitter │ │ │ └── cache │ │ │ ├── ConcurrentMapCacheTest.scala │ │ │ ├── guava │ │ │ └── GuavaCacheTest.scala │ │ │ ├── KeyEncodingCacheTest.scala │ │ │ └── EvictingCacheTest.scala │ └── main │ │ └── scala │ │ ├── BUILD │ │ └── com │ │ └── twitter │ │ └── cache │ │ ├── AsyncMemoize.scala │ │ ├── ConcurrentMapCache.scala │ │ ├── KeyEncodingCache.scala │ │ └── EvictingCache.scala └── BUILD ├── util-jvm ├── BUILD └── src │ ├── main │ └── scala │ │ ├── com │ │ └── twitter │ │ │ └── jvm │ │ │ ├── Cpu.scala │ │ │ ├── NilJvm.scala │ │ │ ├── Opts.scala │ │ │ ├── GcPredictor.scala │ │ │ └── ContentionSnapshot.scala │ │ └── BUILD │ └── test │ └── scala │ ├── BUILD │ └── com │ └── twitter │ └── jvm │ ├── NumProcsTest.scala │ ├── OptsTest.scala │ ├── MockScheduledExecutorService.scala │ ├── CpuProfileTest.scala │ └── ContentionTest.scala ├── util-reflect ├── src │ ├── test │ │ └── scala │ │ │ └── BUILD │ └── main │ │ └── scala │ │ └── BUILD └── BUILD ├── .gitignore ├── util-collection ├── BUILD └── src │ ├── test │ └── scala │ │ └── BUILD │ └── main │ └── scala │ ├── BUILD │ └── com │ └── twitter │ └── util │ ├── SetMaker.scala │ └── LruMap.scala ├── .travis.yml ├── util-test └── src │ └── main │ └── scala │ └── BUILD ├── updatedocs.bash ├── util └── util-thrift │ └── src │ └── test │ └── scala │ └── com │ └── twitter │ └── util │ └── ThriftCodecTest.scala ├── sbt └── CONTRIBUTING.md /GROUPS: -------------------------------------------------------------------------------- 1 | csl 2 | -------------------------------------------------------------------------------- /util-zk-common/GROUPS: -------------------------------------------------------------------------------- 1 | coord-team 2 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | marius 2 | mnakamura 3 | stevegury 4 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.7 2 | -------------------------------------------------------------------------------- /util-zk/OWNERS: -------------------------------------------------------------------------------- 1 | marius 2 | stuhood 3 | ver 4 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/OnePlusOne.scala: -------------------------------------------------------------------------------- 1 | 1 + 1 2 | -------------------------------------------------------------------------------- /util-zk/GROUPS: -------------------------------------------------------------------------------- 1 | coord-team 2 | csl 3 | observability 4 | -------------------------------------------------------------------------------- /util-app/OWNERS: -------------------------------------------------------------------------------- 1 | evm 2 | marius 3 | mnakamura 4 | stevegury 5 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/Derived.scala: -------------------------------------------------------------------------------- 1 | new Base { } 2 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/file-with-dash.scala: -------------------------------------------------------------------------------- 1 | "hello" 2 | -------------------------------------------------------------------------------- /util-codec/OWNERS: -------------------------------------------------------------------------------- 1 | koliver 2 | marius 3 | mnakamura 4 | stevegury 5 | -------------------------------------------------------------------------------- /util-benchmark/OWNERS: -------------------------------------------------------------------------------- 1 | koliver 2 | marius 3 | mnakamura 4 | stevegury 5 | -------------------------------------------------------------------------------- /util-events/OWNERS: -------------------------------------------------------------------------------- 1 | koliver 2 | marius 3 | mnakamura 4 | stevegury 5 | -------------------------------------------------------------------------------- /util-logging/OWNERS: -------------------------------------------------------------------------------- 1 | koliver 2 | marius 3 | mnakamura 4 | stevegury 5 | -------------------------------------------------------------------------------- /util-core/OWNERS: -------------------------------------------------------------------------------- 1 | arya 2 | evm 3 | koliver 4 | marius 5 | mnakamura 6 | stevegury 7 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/DerivedWithInclude.scala: -------------------------------------------------------------------------------- 1 | new Base { 2 | #include HelloJoe.scala 3 | } 4 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/io/OWNERS: -------------------------------------------------------------------------------- 1 | koliver 2 | marius 3 | mnakamura 4 | stevegury 5 | ver 6 | -------------------------------------------------------------------------------- /CONFIG.ini: -------------------------------------------------------------------------------- 1 | ; See http://go/CONFIG.ini 2 | 3 | ; owned by Core System Libraries 4 | [jira] 5 | project: CSL 6 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/RubyInclude.scala: -------------------------------------------------------------------------------- 1 | object CodeThatIncludesSomeRuby { 2 | #include hello.rb 3 | } 4 | -------------------------------------------------------------------------------- /util-stats/src/main/scala/com/twitter/finagle/stats/OWNERS: -------------------------------------------------------------------------------- 1 | koliver 2 | marius 3 | mnakamura 4 | stevegury 5 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/Base.scala: -------------------------------------------------------------------------------- 1 | trait Base extends (() => String) { 2 | def apply() = "hello" 3 | } 4 | 5 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/hello.rb: -------------------------------------------------------------------------------- 1 | def ruby_script 2 | "the newest runtime" * 16.to_i(16).to_s.to_i 3 | end 4 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/BUILD: -------------------------------------------------------------------------------- 1 | resources(name='resources', 2 | sources=rglobs('*') - rglobs('BUILD*', '*.pyc') 3 | ) 4 | -------------------------------------------------------------------------------- /util-hashing/src/test/resources/BUILD: -------------------------------------------------------------------------------- 1 | resources(name='resources', 2 | sources=rglobs('*') - rglobs('BUILD*', '*.pyc') 3 | ) 4 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/HelloJoe.scala: -------------------------------------------------------------------------------- 1 | /* real-time declarative programming now */ 2 | override def toString = "hello, joe" 3 | 4 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/concurrent/Permit.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | trait Permit { 4 | def release() 5 | } 6 | -------------------------------------------------------------------------------- /util-benchmark/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-benchmark', 4 | dependencies=[ 5 | 'util/util-benchmark/src/main/scala' 6 | ] 7 | ) 8 | -------------------------------------------------------------------------------- /util-thrift/src/test/java/BUILD: -------------------------------------------------------------------------------- 1 | java_library(name='java', 2 | dependencies=[ 3 | '3rdparty:libthrift-0.5.0', 4 | ], 5 | sources=rglobs('*.java') 6 | ) 7 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/Deprecated.scala: -------------------------------------------------------------------------------- 1 | new (() => String) { 2 | @deprecated("don't use hello") 3 | def hello() = "hello" 4 | 5 | def apply() = hello() 6 | } 7 | -------------------------------------------------------------------------------- /util-class-preloader/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-class-preloader', 4 | dependencies=[ 5 | 'util/util-class-preloader/src/main/java' 6 | ] 7 | ) 8 | -------------------------------------------------------------------------------- /util-class-preloader/src/main/java/BUILD: -------------------------------------------------------------------------------- 1 | java_agent(name='java', 2 | sources=rglobs('*.java'), 3 | premain='com.twitter.classpreloader.ClassPreloader', 4 | can_retransform=True 5 | ) 6 | -------------------------------------------------------------------------------- /util-class-preloader/src/main/java/com/twitter/class-preloader/manifest: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Premain-Class: com.twitter.classpreloader.ClassPreloader 3 | Can-Retransform-Classes: true 4 | -------------------------------------------------------------------------------- /util-logging/src/test/java/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='java', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | 'util/util-logging/src/main/scala', 5 | ], 6 | sources=rglobs('*.java') 7 | ) -------------------------------------------------------------------------------- /util-stats/src/test/java/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='java', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | 'util/util-stats/src/main/scala', 5 | ], 6 | sources=rglobs('*.java') 7 | ) 8 | -------------------------------------------------------------------------------- /util-core/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | 'util/util-core/src/test:base', 4 | 'util/util-core/src/test/java', 5 | ], 6 | sources=rglobs('*.scala') 7 | ) 8 | -------------------------------------------------------------------------------- /util-eval/src/test/resources/IncludeInclude.scala: -------------------------------------------------------------------------------- 1 | val b1 = 2 | #include DerivedWithInclude.scala 3 | val b2 = 4 | #include DerivedWithInclude.scala 5 | new Base { 6 | override def toString = b1 + "; " + b2 7 | } 8 | -------------------------------------------------------------------------------- /util-app/src/main/java/BUILD: -------------------------------------------------------------------------------- 1 | java_library(name='java', 2 | provides = artifact( 3 | org = 'com.twitter', 4 | name = 'util-app-java', 5 | repo = artifactory, 6 | ), 7 | sources=rglobs('*.java') 8 | ) 9 | -------------------------------------------------------------------------------- /util-app/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:scalatest', 5 | 'util/util-app/src/main/scala' 6 | ], 7 | sources=rglobs('*.scala') 8 | ) 9 | -------------------------------------------------------------------------------- /util-core/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-core', 5 | repo = artifactory, 6 | ), 7 | sources=rglobs('*.scala') 8 | ) 9 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/JavaSingleton.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | /** 4 | * A mixin to allow scala objects to be used from java. 5 | */ 6 | trait JavaSingleton { 7 | def get = this 8 | } 9 | -------------------------------------------------------------------------------- /util-events/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:scalatest', 5 | 'util/util-events/src/main/scala' 6 | ], 7 | sources=rglobs('*.scala') 8 | ) 9 | -------------------------------------------------------------------------------- /util-hashing/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-hashing', 5 | repo = artifactory, 6 | ), 7 | sources=rglobs('*.scala'), 8 | ) 9 | -------------------------------------------------------------------------------- /util-cache/src/test/java/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='java', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scala-library', 6 | 'util/util-cache', 7 | ], 8 | sources=rglobs('*.java') 9 | ) 10 | -------------------------------------------------------------------------------- /util-cache/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalatest', 6 | 'util/util-cache', 7 | ], 8 | sources=rglobs('*.scala') 9 | ) 10 | -------------------------------------------------------------------------------- /util-jvm/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-jvm', 4 | dependencies=[ 5 | 'util/util-jvm/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-jvm/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-jvm/src/main/scala/com/twitter/jvm/Cpu.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import com.twitter.app.GlobalFlag 4 | 5 | object numProcs extends GlobalFlag[Double]( 6 | Runtime.getRuntime.availableProcessors().toDouble, 7 | "number of logical cores" 8 | ) 9 | -------------------------------------------------------------------------------- /util-zk/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-zk', 4 | dependencies=[ 5 | 'util/util-zk/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-zk/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-app/src/test/java/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='java', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:scala-library', 5 | 'util/util-app/src/main/java', 6 | 'util/util-app/src/main/scala' 7 | ], 8 | sources=rglobs('*.java') 9 | ) 10 | -------------------------------------------------------------------------------- /util-codec/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalatest', 6 | 'util/util-codec/src/main/scala', 7 | ], 8 | sources=rglobs('*.scala') 9 | ) 10 | -------------------------------------------------------------------------------- /util-eval/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-eval', 4 | dependencies=[ 5 | 'util/util-eval/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-eval/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-stats/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalatest', 6 | 'util/util-stats/src/main/scala', 7 | ], 8 | sources=rglobs('*.scala') 9 | ) 10 | -------------------------------------------------------------------------------- /util-codec/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-codec', 4 | dependencies=[ 5 | 'util/util-codec/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-codec/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-events/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-events', 4 | dependencies=[ 5 | 'util/util-events/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-events/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-reflect/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:scalatest', 5 | 'util/util-core/src/main/scala', 6 | 'util/util-reflect/src/main/scala' 7 | ], 8 | sources=rglobs('*.scala') 9 | ) 10 | -------------------------------------------------------------------------------- /util-thrift/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-thrift', 4 | dependencies=[ 5 | 'util/util-thrift/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-thrift/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-thrift/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:scalatest', 5 | 'util/util-thrift/src/main/scala', 6 | 'util/util-thrift/src/test/java', 7 | ], 8 | sources=rglobs('*.scala') 9 | ) 10 | -------------------------------------------------------------------------------- /util-hashing/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-hashing', 4 | dependencies=[ 5 | 'util/util-hashing/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-hashing/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-reflect/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-reflect', 4 | dependencies=[ 5 | 'util/util-reflect/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-reflect/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | dist/ 3 | project/boot/ 4 | project/plugins/project/ 5 | project/plugins/src_managed/ 6 | *.log 7 | *.tmproj 8 | lib_managed/ 9 | *.swp 10 | *.iml 11 | .idea/ 12 | .idea/* 13 | .DS_Store 14 | .ensime 15 | .ivyjars 16 | sbt 17 | out/ 18 | sbt-launch.jar 19 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/concurrent/AsyncMutex.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | class AsyncMutex private (maxWaiters: Option[Int]) extends AsyncSemaphore(1, maxWaiters) { 4 | def this() = this(None) 5 | def this(maxWaiters: Int) = this(Some(maxWaiters)) 6 | } 7 | -------------------------------------------------------------------------------- /util-core/src/test/java/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='java', 2 | dependencies=[ 3 | '3rdparty:scala-library', 4 | 'util/util-core/src/test:base', 5 | 'util/util-core/src/main/scala', 6 | 'util/util-core/src/main/java' 7 | ], 8 | sources=rglobs('*.java') 9 | ) 10 | -------------------------------------------------------------------------------- /util-zk-common/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-zk-common', 4 | dependencies=[ 5 | 'util/util-zk-common/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-zk-common/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-collection/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-collection', 4 | dependencies=[ 5 | 'util/util-collection/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-collection/src/test/scala' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-app/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-app', 4 | dependencies=[ 5 | 'util/util-app/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-app/src/test/scala', 12 | 'util/util-app/src/test/java', 13 | ] 14 | ) 15 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/HandledPromise.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | class HandledPromise[A] extends Promise[A] { 4 | @volatile var _handled: Option[Throwable] = None 5 | def handled: Option[Throwable] = _handled 6 | setInterruptHandler { case e => _handled = Some(e) } 7 | } 8 | -------------------------------------------------------------------------------- /util-stats/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-stats', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | 'util/util-core/src/main/scala', 9 | ], 10 | sources=rglobs('*.scala') 11 | ) 12 | -------------------------------------------------------------------------------- /util-cache/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-cache', 4 | dependencies=[ 5 | 'util/util-cache/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-cache/src/test/java', 12 | 'util/util-cache/src/test/scala' 13 | ] 14 | ) 15 | -------------------------------------------------------------------------------- /util-events/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-events', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | 'util/util-app/src/main/scala', 9 | ], 10 | sources=rglobs('*.scala'), 11 | ) 12 | -------------------------------------------------------------------------------- /util-logging/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalatest', 6 | 'util/util-core/src/main/scala', 7 | 'util/util-logging/src/main/scala', 8 | ], 9 | sources=rglobs('*.scala') 10 | ) 11 | -------------------------------------------------------------------------------- /util-stats/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-stats', 4 | dependencies=[ 5 | 'util/util-stats/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-stats/src/test/scala', 12 | 'util/util-stats/src/test/java' 13 | ] 14 | ) 15 | -------------------------------------------------------------------------------- /util-collection/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalatest', 6 | 'util/util-collection/src/main/scala', 7 | 'util/util-core/src/main/scala', 8 | ], 9 | sources=rglobs('*.scala') 10 | ) 11 | -------------------------------------------------------------------------------- /util-logging/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-logging', 4 | dependencies=[ 5 | 'util/util-logging/src/main/scala' 6 | ] 7 | ) 8 | 9 | jar_library(name='tests', 10 | dependencies=[ 11 | 'util/util-logging/src/test/scala', 12 | 'util/util-logging/src/test/java' 13 | ] 14 | ) 15 | -------------------------------------------------------------------------------- /util-core/src/main/java/BUILD: -------------------------------------------------------------------------------- 1 | java_library(name='java', 2 | provides = artifact( 3 | org = 'com.twitter', 4 | name = 'util-core-java', 5 | repo = artifactory, 6 | ), 7 | sources=rglobs('*.java'), 8 | dependencies=[ 9 | '3rdparty:scala-library', 10 | 'util/util-core/src/main/scala' 11 | ] 12 | ) 13 | -------------------------------------------------------------------------------- /util-eval/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-eval', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty:scala-compiler', 9 | 'util/util-core/src/main/scala' 10 | ], 11 | sources=rglobs('*.scala'), 12 | ) 13 | -------------------------------------------------------------------------------- /util-app/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-app', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | 'util/util-app/src/main/java', 9 | 'util/util-core/src/main/scala', 10 | ], 11 | sources=rglobs('*.scala'), 12 | ) 13 | -------------------------------------------------------------------------------- /util-codec/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-codec', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty/jvm/commons-codec', 9 | 'util/util-core/src/main/scala' 10 | ], 11 | sources=rglobs('*.scala'), 12 | ) 13 | -------------------------------------------------------------------------------- /util-jvm/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-jvm', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | 'util/util-app/src/main/scala', 9 | 'util/util-core/src/main/scala' 10 | ], 11 | sources=rglobs('*.scala'), 12 | ) 13 | -------------------------------------------------------------------------------- /util-core/BUILD: -------------------------------------------------------------------------------- 1 | maven_layout() 2 | 3 | jar_library(name='util-core', 4 | dependencies=[ 5 | 'util/util-core/src/main/java', 6 | 'util/util-core/src/main/scala' 7 | ] 8 | ) 9 | 10 | jar_library(name='tests', 11 | dependencies=[ 12 | 'util/util-core/src/test/java', 13 | 'util/util-core/src/test/scala' 14 | ] 15 | ) 16 | -------------------------------------------------------------------------------- /util-core/src/test/BUILD: -------------------------------------------------------------------------------- 1 | jar_library(name='base', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalacheck', 6 | '3rdparty:scalatest', 7 | '3rdparty/jvm/com/twitter/common:objectsize', 8 | 'util/util-core/src/main/scala', 9 | 'util/util-core/src/main/java' 10 | ], 11 | ) 12 | -------------------------------------------------------------------------------- /util-cache/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | sources=rglobs('*.scala'), 3 | dependencies=[ 4 | '3rdparty/jvm/com/google/guava', 5 | 'util/util-core' 6 | ], 7 | provides=scala_artifact(org='com.twitter', 8 | name='util-cache', 9 | repo = artifactory,), 10 | ) 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.10.4 5 | - 2.11.2 6 | 7 | jdk: 8 | - oraclejdk7 9 | # Note: not currently testing on JDK 8 internally 10 | - oraclejdk8 11 | - openjdk7 12 | 13 | before_script: 14 | # default $SBT_OPTS is irrelevant to sbt lancher 15 | - unset SBT_OPTS 16 | 17 | script: 18 | - ./sbt ++$TRAVIS_SCALA_VERSION test 19 | -------------------------------------------------------------------------------- /util-eval/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalatest', 6 | 'util/util-core/src/main/scala', 7 | 'util/util-eval/src/main/scala', 8 | ], 9 | sources=rglobs('*.scala'), 10 | resources=[ 11 | 'util/util-eval/src/test/resources' 12 | ] 13 | ) 14 | -------------------------------------------------------------------------------- /util-zk/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-zk', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty:zookeeper', 9 | 'util/util-core/src/main/scala', 10 | 'util/util-logging/src/main/scala' 11 | ], 12 | sources=rglobs('*.scala'), 13 | ) 14 | -------------------------------------------------------------------------------- /util-test/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-test', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty:scalatest', 9 | 'util/util-logging/src/main/scala', 10 | 'util/util-core/src/main/scala', 11 | ], 12 | sources=rglobs('*.scala'), 13 | ) 14 | -------------------------------------------------------------------------------- /util-zk/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:cglib', 4 | '3rdparty:jmock', 5 | '3rdparty:junit', 6 | '3rdparty:mockito-all', 7 | '3rdparty:scalatest', 8 | '3rdparty:zookeeper', 9 | 'util/util-core/src/main/scala', 10 | 'util/util-zk/src/main/scala', 11 | ], 12 | sources=rglobs('*.scala') 13 | ) 14 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/TimeoutException.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import java.util.concurrent.{TimeoutException => JUCTimeoutException} 4 | 5 | // Now that this is inherits from the usual TimeoutException, we can move to 6 | // j.u.c.TimeoutException during our next API break. 7 | class TimeoutException(message: String) extends JUCTimeoutException(message) 8 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/Unsafe.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | object Unsafe { 4 | private lazy val instance: sun.misc.Unsafe = { 5 | val fld = classOf[sun.misc.Unsafe].getDeclaredField("theUnsafe") 6 | fld.setAccessible(true) 7 | fld.get(null).asInstanceOf[sun.misc.Unsafe] 8 | } 9 | 10 | def apply(): sun.misc.Unsafe = instance 11 | } 12 | -------------------------------------------------------------------------------- /util-logging/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-logging', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | 'util/util-app/src/main/scala', 9 | 'util/util-core/src/main/scala', 10 | 'util/util-stats/src/main/scala', 11 | ], 12 | sources=rglobs('*.scala'), 13 | ) 14 | -------------------------------------------------------------------------------- /util-collection/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-collection', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty/jvm/com/google/guava', 9 | '3rdparty/jvm/commons-collections', 10 | 'util/util-core/src/main/scala', 11 | ], 12 | sources=rglobs('*.scala'), 13 | ) 14 | -------------------------------------------------------------------------------- /util-core/src/main/java/com/twitter/util/Events.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | /** 4 | * A Java adaptation of {@link com.twitter.util.Event} companion object. 5 | */ 6 | public final class Events { 7 | private Events() { } 8 | 9 | /** 10 | * @see com.twitter.util.Event$#apply() 11 | */ 12 | public static WitnessedEvent newEvent() { 13 | return new WitnessedEvent(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /util-thrift/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-thrift', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind', 9 | '3rdparty:libthrift-0.5.0', 10 | '3rdparty:slf4j-api', 11 | 'util/util-codec' 12 | ], 13 | sources=rglobs('*.scala'), 14 | ) 15 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/NoStacktrace.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | trait NoStacktrace extends Exception { 4 | override def fillInStackTrace = this 5 | // specs expects non-empty stacktrace array 6 | this.setStackTrace(NoStacktrace.NoStacktraceArray) 7 | } 8 | 9 | object NoStacktrace { 10 | val NoStacktraceArray = Array(new StackTraceElement("com.twitter.util", "NoStacktrace", null, -1)) 11 | } 12 | -------------------------------------------------------------------------------- /util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | case class AuthInfo(mode: String, data: Array[Byte]) 4 | 5 | object AuthInfo { 6 | def digest(user: String, secret: String) = { 7 | val authString = "%s:%s".format(user, secret).getBytes("UTF-8")//DigestAuthenticationProvider.generateDigest("%s:%s".format(user, secret)).getBytes("UTF-8") 8 | AuthInfo("digest", authString) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /util-reflect/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-reflect', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty/jvm/asm', 9 | '3rdparty/jvm/asm:asm-commons', 10 | '3rdparty/jvm/asm:asm-util', 11 | '3rdparty:cglib', 12 | 'util/util-core/src/main/scala' 13 | ], 14 | sources=rglobs('*.scala'), 15 | ) 16 | -------------------------------------------------------------------------------- /util-benchmark/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | benchmark(name='scala', 2 | dependencies=[ 3 | '3rdparty:caliper', 4 | '3rdparty:scala-library', 5 | 'util/util-core/src/main/scala', 6 | 'util/util-events/src/main/scala', 7 | 'util/util-jvm/src/main/scala' 8 | ], 9 | sources=rglobs('*.scala') 10 | ) 11 | 12 | jvm_binary(name='caliper', 13 | main='com.google.caliper.Runner', 14 | dependencies=[ 15 | ':scala' 16 | ] 17 | ) 18 | -------------------------------------------------------------------------------- /util-hashing/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty/jvm/commons-codec', 4 | '3rdparty:junit', 5 | '3rdparty:mockito-all', 6 | '3rdparty:scalacheck', 7 | '3rdparty:scalatest', 8 | 'util/util-core/src/main/scala', 9 | 'util/util-hashing/src/main/scala', 10 | ], 11 | sources=rglobs('*.scala'), 12 | resources=[ 13 | 'util/util-hashing/src/test/resources' 14 | ] 15 | ) 16 | -------------------------------------------------------------------------------- /util-app/src/main/java/com/twitter/app/GlobalFlagVisible.java: -------------------------------------------------------------------------------- 1 | package com.twitter.app; 2 | 3 | import java.lang.annotation.Inherited; 4 | import java.lang.annotation.Retention; 5 | import java.lang.annotation.RetentionPolicy; 6 | 7 | /** 8 | * An annotation to support the visibilty of global flags. Used 9 | * internally in the flags implementation. 10 | */ 11 | public @Retention(RetentionPolicy.RUNTIME) @Inherited @interface GlobalFlagVisible {} 12 | -------------------------------------------------------------------------------- /util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.hashing 2 | 3 | trait Distributor[A] { 4 | def entryForHash(hash: Long): (Long, A) 5 | def nodeForHash(hash: Long): A 6 | def nodeCount: Int 7 | def nodes: Seq[A] 8 | } 9 | 10 | class SingletonDistributor[A](node: A) extends Distributor[A] { 11 | def entryForHash(hash: Long) = (hash, node) 12 | def nodeForHash(hash: Long) = node 13 | def nodeCount = 1 14 | def nodes = Seq(node) 15 | } 16 | -------------------------------------------------------------------------------- /util-jvm/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty:junit', 4 | '3rdparty:mockito-all', 5 | '3rdparty:scalatest', 6 | 'util/util-core/src/main/scala', 7 | 'util/util-jvm/src/main/scala', 8 | 'util/util-logging/src/main/scala', 9 | 'util/util-test/src/main/scala', 10 | ], 11 | sources=rglobs('*.scala') 12 | ) 13 | 14 | jvm_binary(name='bin', 15 | main='com.twitter.jvm.EstimatorApp', 16 | dependencies=[ 17 | ':scala' 18 | ] 19 | ) 20 | -------------------------------------------------------------------------------- /util-stats/src/main/scala/com/twitter/finagle/stats/package.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.finagle 2 | 3 | import com.twitter.util.{Future, Stopwatch} 4 | import java.util.concurrent.TimeUnit 5 | 6 | /** 7 | * These stats are scoped to `com.twitter.finagle.stats` for historical reasons. 8 | * 9 | * They used to be in the `finagle-core` package, although we moved them 10 | * because we found they didn't depend on anything finagle-specific. To ease 11 | * the transition, we kept the namespace. 12 | */ 13 | package object stats {} 14 | -------------------------------------------------------------------------------- /util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import com.twitter.conversions.storage._ 4 | import com.twitter.util.Time 5 | 6 | object NilJvm extends Jvm { 7 | val opts: Opts = new Opts { 8 | def compileThresh = None 9 | } 10 | def forceGc() = System.gc() 11 | val snapCounters: Map[String, String] = Map() 12 | val snap: Snapshot = Snapshot( 13 | Time.epoch, 14 | Heap(0, 0, Seq()), 15 | Seq()) 16 | val edenPool = new Pool { def state() = PoolState(0, 0.bytes, 0.bytes) } 17 | } 18 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/CancellableTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | 4 | import org.junit.runner.RunWith 5 | import org.scalatest.WordSpec 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class CancellableTest extends WordSpec { 10 | "CancellableSink" should { 11 | "cancel once" in { 12 | var count = 0 13 | val s = new CancellableSink { count += 1 } 14 | s.cancel() 15 | assert(count === 1) 16 | s.cancel() 17 | assert(count === 1) 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /util-zk-common/src/test/scala/BUILD: -------------------------------------------------------------------------------- 1 | junit_tests(name='scala', 2 | dependencies=[ 3 | '3rdparty/jvm/com/google/guava', 4 | '3rdparty/jvm/com/twitter/common/zookeeper:client', 5 | '3rdparty/jvm/com/twitter/common/zookeeper:server-set', 6 | '3rdparty/jvm/com/twitter/common:net-util', 7 | '3rdparty/jvm/com/twitter/common:service-thrift', 8 | '3rdparty:cglib', 9 | '3rdparty:jmock', 10 | '3rdparty:junit', 11 | '3rdparty:scalatest', 12 | 'util/util-core/src/main/scala', 13 | 'util/util-zk-common/src/main/scala', 14 | ], 15 | sources=rglobs('*.scala') 16 | ) 17 | -------------------------------------------------------------------------------- /util-cache/src/test/scala/com/twitter/cache/ConcurrentMapCacheTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache 2 | 3 | import com.twitter.util.Future 4 | import java.util.concurrent.ConcurrentHashMap 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class ConcurrentMapCacheTest extends AbstractFutureCacheTest { 10 | def name: String = "ConcurrentMapCache" 11 | 12 | def mkCtx(): Ctx = new Ctx { 13 | val map = new ConcurrentHashMap[String, Future[String]]() 14 | val cache = new ConcurrentMapCache[String, String](map) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /updatedocs.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | dir=/tmp/util.$$ 6 | trap "rm -fr $dir" 0 1 2 7 | 8 | unidoc=target/scala-2.10/unidoc/ 9 | rm -fr "$unidoc" 10 | 11 | echo 'making unidoc...' 1>&2 12 | ./sbt unidoc >/dev/null 2>&1 13 | 14 | echo 'cloning...' 1>&2 15 | git clone -b gh-pages git@github.com:twitter/util.git $dir >/dev/null 2>&1 16 | 17 | savedir=$(pwd) 18 | cd $dir 19 | rsync -a --delete "$savedir/$unidoc" "docs" 20 | git add . 21 | echo 'pushing...!' 1>&2 22 | git diff-index --quiet HEAD || (git commit -am"site push by $(whoami)"; git push origin gh-pages:gh-pages;) 23 | echo 'finished!' 1>&2 24 | -------------------------------------------------------------------------------- /util-benchmark/src/main/scala/com/twitter/util/events/SinkBenchmark.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util.events 2 | 3 | import com.google.caliper.SimpleBenchmark 4 | 5 | class SinkBenchmark extends SimpleBenchmark { 6 | 7 | private[this] val sizedSink = SizedSink(10000) 8 | private[this] val eventType = new Event.Type { } 9 | 10 | private[this] def event(sink: Sink, reps: Int): Unit = { 11 | var i = 0 12 | while (i < reps) { 13 | sink.event(eventType, doubleVal = 2.5d) 14 | i += 1 15 | } 16 | } 17 | 18 | def timeEventSizedSink(reps: Int): Unit = { 19 | event(sizedSink, reps) 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/CountDownLatch.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | class CountDownLatch(val initialCount: Int) { 6 | val underlying = new java.util.concurrent.CountDownLatch(initialCount) 7 | def count = underlying.getCount 8 | def isZero = count == 0 9 | def countDown() = underlying.countDown() 10 | def await() = underlying.await() 11 | def await(timeout: Duration) = underlying.await(timeout.inMillis, TimeUnit.MILLISECONDS) 12 | def within(timeout: Duration) = await(timeout) || { 13 | throw new TimeoutException(timeout.toString) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /util-cache/src/test/scala/com/twitter/cache/guava/GuavaCacheTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache.guava 2 | 3 | import com.google.common.cache.CacheBuilder 4 | import com.twitter.cache.AbstractFutureCacheTest 5 | import com.twitter.util.Future 6 | import org.junit.runner.RunWith 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class GuavaCacheTest extends AbstractFutureCacheTest { 11 | def name: String = "GuavaCache" 12 | 13 | def mkCtx(): Ctx = new Ctx { 14 | val guava = CacheBuilder.newBuilder().build[String, Future[String]]() 15 | val cache = new GuavaCache[String, String](guava) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /util-stats/src/main/scala/com/twitter/finagle/stats/StatsRegistry.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.finagle.stats 2 | 3 | /** 4 | * Interface used via the LoadService mechanism to obtain an 5 | * efficient mechanism to sample stats. 6 | */ 7 | private[twitter] trait StatsRegistry { 8 | def apply(): Map[String, StatEntry] 9 | } 10 | 11 | private[twitter] trait StatEntry { 12 | /** 13 | * The delta since the entry was last sampled. 14 | * Note, this field is identical to `value` for 15 | * instantaneous entries (ex. gauges). 16 | */ 17 | val delta: Double 18 | 19 | /** The instantaneous value of the entry. */ 20 | val value: Double 21 | } 22 | -------------------------------------------------------------------------------- /util-benchmark/src/main/scala/com/twitter/util/benchmark/Stopwatch.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util.benchmark 2 | 3 | import com.google.caliper.SimpleBenchmark 4 | 5 | import com.twitter.util.Stopwatch 6 | 7 | class StopwatchBenchmark extends SimpleBenchmark { 8 | // Multiplier required to do enough work 9 | // to be able to measure anything at all. 10 | val N = 100 11 | 12 | def timeMakeCallback(reps: Int) { 13 | var i = 0 14 | while (i < reps) { 15 | Stopwatch.start() 16 | i += 1 17 | } 18 | } 19 | 20 | def timeTime(reps: Int) { 21 | var i = 0 22 | while (i < reps) { 23 | Stopwatch.start()() 24 | i += 1 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/io/TempDirectory.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import java.io.File 4 | 5 | object TempDirectory { 6 | /** 7 | * Create a new temporary directory, which will be deleted upon the exit of the VM. 8 | * 9 | * @return File representing the directory 10 | */ 11 | def create(deleteAtExit: Boolean = true): File = { 12 | var file = File.createTempFile("temp", "dir") 13 | file.delete() 14 | file.mkdir() 15 | 16 | if (deleteAtExit) 17 | Runtime.getRuntime().addShutdownHook(new Thread { 18 | override def run() { 19 | Files.delete(file) 20 | } 21 | }) 22 | 23 | file 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/StateMachine.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | object StateMachine { 4 | class InvalidStateTransition(fromState: String, command: String) extends 5 | Exception("Transitioning from " + fromState + " via command " + command) 6 | } 7 | 8 | trait StateMachine { 9 | import StateMachine._ 10 | 11 | protected abstract class State 12 | protected var state: State = _ 13 | 14 | protected def transition[A](command: String)(f: PartialFunction[State, A]) = synchronized { 15 | if (f.isDefinedAt(state)) { 16 | f(state) 17 | } else { 18 | throw new InvalidStateTransition(state.toString, command) 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /util-logging/src/test/java/com/twitter/logging/LoggerCompilationTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.logging; 2 | 3 | /** 4 | * Basic java wrapper compilation and execution test. 5 | */ 6 | public class LoggerCompilationTest { 7 | Logger LOG = Logger.get(); 8 | Logger LOG1 = Logger.get("fonzbonz"); 9 | Logger LOG2 = Logger.get(Logger.class); 10 | 11 | public void testLogging() { 12 | LOG2.log(Logger.ERROR(), "how about some logging"); 13 | LOG.debug("howdy"); 14 | LOG.debug("how bout with a string %s", "string goes here"); 15 | LOG.error("this is an error"); 16 | LOG1.trace("with an integer %d", 22); 17 | Logger.reset(); 18 | LOG2.clearHandlers(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/conversions/time.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.conversions 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | import com.twitter.util.Duration 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class TimeTest extends FunSuite { 11 | import time._ 12 | 13 | test("converts Duration.Zero") { 14 | assert(0.seconds eq Duration.Zero) 15 | assert(0.milliseconds eq Duration.Zero) 16 | assert(0.seconds eq 0.seconds) 17 | } 18 | 19 | test("converts nonzero durations") { 20 | assert(1.seconds === Duration.fromSeconds(1)) 21 | assert(123.milliseconds === Duration.fromMilliseconds(123)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /util-events/src/main/scala/com/twitter/util/events/package.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import com.twitter.app.GlobalFlag 4 | 5 | package object events { 6 | 7 | // Note: these flags should generally be specified via System properties 8 | // to ensure that their values are available very early in the application's 9 | // lifecycle. 10 | 11 | private[events] object sinkEnabled extends GlobalFlag[Boolean]( 12 | false, 13 | "Whether or not event capture is enabled. Prefer setting via System properties.") 14 | 15 | private[events] object approxNumEvents extends GlobalFlag[Int]( 16 | 0, 17 | "Approximate number of events to keep in memory. Prefer setting via System properties.") 18 | 19 | } 20 | -------------------------------------------------------------------------------- /util-cache/src/main/scala/com/twitter/cache/AsyncMemoize.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache 2 | 3 | import com.twitter.util.Future 4 | 5 | object AsyncMemoize { 6 | /** 7 | * Produces a function which caches the result of an asynchronous 8 | * computation in the supplied Cache. 9 | * 10 | * Does not guarantee that `fn` will be computed exactly once for each key. 11 | */ 12 | def apply[A, B](fn: A => Future[B], cache: FutureCache[A, B]): A => Future[B] = 13 | new MemoizedFunction(fn, cache) 14 | } 15 | 16 | private[cache] class MemoizedFunction[A, B]( 17 | fn: A => Future[B], 18 | cache: FutureCache[A, B]) extends (A => Future[B]) { 19 | 20 | def apply(a: A): Future[B] = cache.getOrElseUpdate(a) { 21 | fn(a) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache 2 | 3 | import java.util.concurrent.ConcurrentHashMap 4 | import com.twitter.util.Future 5 | import org.junit.runner.RunWith 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class KeyEncodingCacheTest extends AbstractFutureCacheTest { 10 | def name: String = "KeyEncodingCache" 11 | 12 | def mkCtx(): Ctx = new Ctx { 13 | val underlyingMap: ConcurrentHashMap[Int, Future[String]] = new ConcurrentHashMap() 14 | val underlyingCache: FutureCache[Int, String] = new ConcurrentMapCache(underlyingMap) 15 | val cache: FutureCache[String, String] = 16 | new KeyEncodingCache({ num: String => num.hashCode }, underlyingCache) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /util-zk-common/src/main/scala/BUILD: -------------------------------------------------------------------------------- 1 | scala_library(name='scala', 2 | provides = scala_artifact( 3 | org = 'com.twitter', 4 | name = 'util-zk-common', 5 | repo = artifactory, 6 | ), 7 | dependencies=[ 8 | '3rdparty/jvm/com/google/guava', 9 | '3rdparty/jvm/com/twitter/common/zookeeper:client', 10 | '3rdparty/jvm/com/twitter/common/zookeeper:server-set', 11 | '3rdparty/jvm/com/twitter/common:quantity', 12 | '3rdparty/jvm/com/twitter/common:service-thrift', 13 | '3rdparty:zookeeper', 14 | 'util/util-collection/src/main/scala', 15 | 'util/util-core/src/main/scala', 16 | 'util/util-hashing/src/main/scala', 17 | 'util/util-logging/src/main/scala', 18 | 'util/util-zk/src/main/scala', 19 | ], 20 | sources=rglobs('*.scala'), 21 | ) 22 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | sbtResolver <<= (sbtResolver) { r => 2 | Option(System.getenv("SBT_PROXY_REPO")) map { x => 3 | Resolver.url("proxy repo for sbt", url(x))(Resolver.ivyStylePatterns) 4 | } getOrElse r 5 | } 6 | 7 | resolvers <<= (resolvers) { r => 8 | (Option(System.getenv("SBT_PROXY_REPO")) map { url => 9 | Seq("proxy-repo" at url) 10 | } getOrElse { 11 | r ++ Seq( 12 | "twitter.com" at "http://maven.twttr.com/", 13 | "scala-tools" at "http://scala-tools.org/repo-releases/", 14 | "maven" at "http://repo1.maven.org/maven2/", 15 | "freemarker" at "http://freemarker.sourceforge.net/maven2/" 16 | ) 17 | }) ++ Seq("local" at ("file:" + System.getProperty("user.home") + "/.m2/repo/")) 18 | } 19 | 20 | externalResolvers <<= (resolvers) map identity 21 | -------------------------------------------------------------------------------- /util-jvm/src/test/scala/com/twitter/jvm/NumProcsTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSpec 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class NumProcsTest extends FunSpec { 9 | describe("numProcs") { 10 | it("should be the number of available processors according to the runtime by default") { 11 | assert(System.getProperty("com.twitter.jvm.numProcs") === null) 12 | assert(numProcs() === Runtime.getRuntime().availableProcessors()) 13 | } 14 | 15 | it("should be settable as a flag") { 16 | val old = numProcs() 17 | numProcs.parse("10.0") 18 | 19 | assert(numProcs() === 10.0) 20 | 21 | numProcs.parse() 22 | assert(numProcs() === old) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /util-logging/src/main/scala/com/twitter/logging/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter 18 | 19 | package object logging { 20 | type HandlerFactory = (() => Handler) 21 | } 22 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.WordSpec 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class LastWriteWinsQueueTest extends WordSpec { 9 | "LastWriteWinsQueue" should { 10 | val queue = new LastWriteWinsQueue[String] 11 | 12 | "add & remove items" in { 13 | assert(queue.size === 0) 14 | queue.add("1") 15 | assert(queue.size === 1) 16 | assert(queue.remove() === "1") 17 | assert(queue.size === 0) 18 | } 19 | 20 | "last write wins" in { 21 | queue.add("1") 22 | queue.add("2") 23 | assert(queue.size === 1) 24 | assert(queue.poll() === "2") 25 | assert(queue.size === 0) 26 | assert(queue.poll() === null) 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /util-events/src/test/scala/com/twitter/util/events/SinkTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util.events 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class SinkTest extends FunSuite { 9 | 10 | test("Null") { 11 | val sink = Sink.Null 12 | assert(sink.events.size === 0) 13 | 14 | sink.event(new Event.Type { }, objectVal = "hi") 15 | assert(sink.events.size === 0) 16 | } 17 | 18 | test("newDefault") { 19 | sinkEnabled.let(false) { 20 | assert(Sink.newDefault === Sink.Null) 21 | } 22 | 23 | sinkEnabled.let(true) { 24 | approxNumEvents.let(0) { 25 | assert(Sink.newDefault === Sink.Null) 26 | } 27 | approxNumEvents.let(1) { 28 | assert(Sink.newDefault !== Sink.Null) 29 | } 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /util-cache/src/test/java/com/twitter/cache/JavaRefreshTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.cache; 2 | 3 | import com.twitter.util.Await; 4 | import com.twitter.util.Duration; 5 | import scala.Function0; 6 | import com.twitter.util.Future; 7 | import junit.framework.TestCase; 8 | import static org.mockito.Mockito.*; 9 | 10 | import static org.mockito.Mockito.mock; 11 | 12 | public class JavaRefreshTest extends TestCase { 13 | public void testRefreshWorksFromJava() throws Exception { 14 | Function0> provider = mock(Function0.class); 15 | when(provider.apply()).thenReturn(Future.value(1)); 16 | Function0> memoized = Refresh.every(Duration.fromSeconds(10), provider); 17 | 18 | assertEquals(1, Await.result(memoized.apply()).intValue()); 19 | assertEquals(1, Await.result(memoized.apply()).intValue()); 20 | verify(provider, times(1)).apply(); 21 | } 22 | } -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FlatSpec 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | import com.twitter.util.Await 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class AsyncMutexTest extends FlatSpec { 11 | "AsyncMutex" should "admit only one operation at a time" in { 12 | val m = new AsyncMutex 13 | 14 | val a0 = m.acquire() 15 | val a1 = m.acquire() 16 | 17 | assert(a0.isDefined === true) 18 | assert(a1.isDefined === false) 19 | 20 | Await.result(a0).release() // satisfy operation 0 21 | assert(a1.isDefined === true) // 1 now available 22 | 23 | val a2 = m.acquire() 24 | assert(a2.isDefined === false) 25 | Await.result(a1).release() // satisfy operation 1 26 | assert(a2.isDefined === true) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /util/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class ThriftCodecTest extends FunSuite { 9 | 10 | private def roundTrip(codec: ThriftCodec[TestThriftStructure, _]): Unit = { 11 | val struct = new TestThriftStructure("aString", 5) 12 | val encoded: Array[Byte] = codec.encode(struct) 13 | val decoded: TestThriftStructure = codec.decode(encoded) 14 | 15 | assert(decoded === struct) 16 | } 17 | 18 | test("BinaryThriftCodec") { 19 | val codec = new BinaryThriftCodec[TestThriftStructure]() 20 | roundTrip(codec) 21 | } 22 | 23 | test("CompactThriftCodec") { 24 | val codec = new CompactThriftCodec[TestThriftStructure]() 25 | roundTrip(codec) 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/io/BufReader.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import com.twitter.util.{Future, Return, Try, Throw} 4 | 5 | /** 6 | * Construct a Reader from a Buf. 7 | */ 8 | private[io] class BufReader(buf: Buf) extends Reader { 9 | @volatile private[this] var state: Try[Buf] = Return(buf) 10 | 11 | def read(n: Int) = synchronized { 12 | state match { 13 | case Return(buf) => 14 | if (buf.isEmpty) Future.None else { 15 | val f = Future.value(Some(buf.slice(0, n))) 16 | state = Return(buf.slice(n, Int.MaxValue)) 17 | f 18 | } 19 | case Throw(exc) => Future.exception(exc) 20 | } 21 | } 22 | 23 | def discard() = synchronized { 24 | state = Throw(new Reader.ReaderDiscarded) 25 | } 26 | } 27 | 28 | object BufReader { 29 | def apply(buf: Buf): Reader = 30 | if (buf.isEmpty) Reader.Null else new BufReader(buf) 31 | } 32 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | 4 | import org.junit.runner.RunWith 5 | import org.scalatest.WordSpec 6 | import org.scalatest.junit.JUnitRunner 7 | import org.scalatest.mock.MockitoSugar 8 | 9 | import com.twitter.util.{Promise, Return} 10 | 11 | @RunWith(classOf[JUnitRunner]) 12 | class FutureOfferTest extends WordSpec with MockitoSugar { 13 | "Future.toOffer" should { 14 | "activate when future is satisfied (poll)" in { 15 | val p = new Promise[Int] 16 | val o = p.toOffer 17 | assert(o.prepare().poll === None) 18 | p() = Return(123) 19 | assert(o.prepare().poll match { 20 | case Some(Return(tx)) => 21 | tx.ack().poll match { 22 | case Some(Return(Tx.Commit(Return(123)))) => true 23 | case _ => false 24 | } 25 | case _ => false 26 | }) 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.hashing 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.junit.JUnitRunner 5 | import org.scalatest.FunSuite 6 | import org.scalatest.prop.GeneratorDrivenPropertyChecks 7 | 8 | @RunWith(classOf[JUnitRunner]) 9 | class HashableTest extends FunSuite with GeneratorDrivenPropertyChecks { 10 | 11 | private[this] val algorithms = Seq( 12 | Hashable.CRC32_ITU, Hashable.FNV1_32, Hashable.FNV1_64, Hashable.FNV1A_32, 13 | Hashable.FNV1A_64, Hashable.HSIEH, Hashable.JENKINS, Hashable.MD5_LEInt 14 | ) 15 | 16 | def testConsistency[T](algo: Hashable[Array[Byte], T]) { 17 | forAll { input: Array[Byte] => 18 | assert(algo(input) === algo(input)) 19 | } 20 | } 21 | 22 | algorithms foreach { algo => 23 | test(s"$algo hashing algorithm should be consistent") { 24 | testConsistency(algo) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /util-jvm/src/main/scala/com/twitter/jvm/Opts.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import java.lang.management.ManagementFactory 4 | import javax.management.openmbean.CompositeDataSupport 5 | import javax.management.{ObjectName, RuntimeMBeanException} 6 | 7 | /** 8 | * Retrieve the named JVM option. 9 | */ 10 | object Opt { 11 | private[this] val DiagnosticName = 12 | ObjectName.getInstance("com.sun.management:type=HotSpotDiagnostic") 13 | 14 | def apply(name: String): Option[String] = try Some { 15 | val o = ManagementFactory.getPlatformMBeanServer().invoke( 16 | DiagnosticName, "getVMOption", 17 | Array(name), Array("java.lang.String")) 18 | o.asInstanceOf[CompositeDataSupport].get("value").asInstanceOf[String] 19 | } catch { 20 | case _: IllegalArgumentException => 21 | None 22 | case rbe: RuntimeMBeanException 23 | if rbe.getCause.isInstanceOf[IllegalArgumentException] => 24 | None 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/io/StreamIOTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import scala.util.Random 4 | 5 | import org.junit.runner.RunWith 6 | import org.scalatest.WordSpec 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class StreamIOTest extends WordSpec { 11 | "StreamIO.copy" should { 12 | "copy the entire stream" in { 13 | val buf = new Array[Byte](2048) 14 | (new Random).nextBytes(buf) 15 | val bis = new java.io.ByteArrayInputStream(buf) 16 | val bos = new java.io.ByteArrayOutputStream() 17 | StreamIO.copy(bis, bos) 18 | assert(bos.toByteArray.toSeq === buf.toSeq) 19 | } 20 | 21 | "produce empty streams from empty streams" in { 22 | val bis = new java.io.ByteArrayInputStream(new Array[Byte](0)) 23 | val bos = new java.io.ByteArrayOutputStream() 24 | StreamIO.copy(bis, bos) 25 | assert(bos.size === 0) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /util-hashing/README.markdown: -------------------------------------------------------------------------------- 1 | # util-hashing 2 | 3 | Util-hashing is a collection of hash functions and hashing distributors (eg. ketama). 4 | 5 | This library is released under the Apache Software License, version 2, which 6 | should be included with the source in a file named `LICENSE`. 7 | 8 | 9 | ## Building 10 | 11 | Use sbt (simple-build-tool) to build: 12 | 13 | $ sbt clean update package-dist 14 | 15 | The finished jar will be in `dist/`. 16 | 17 | 18 | ## Using 19 | 20 | To use hash functions: 21 | 22 | KeyHasher.FNV1_32.hashKey("string".getBytes) 23 | 24 | Available hash functions are: 25 | 26 | FNV1_32 27 | FNV1A_32 28 | FNV1_64 29 | FNV1A_64 30 | KETAMA 31 | CRC32_ITU 32 | HSIEH 33 | 34 | To use ketama distributor: 35 | 36 | val nodes = List(KetamaNode("host:port", weight, client)) 37 | val distributor = new KetamaDistributor(nodes, KeyHasher.FNV1_32) 38 | distributor.nodeForKey("abc") // => client -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/io/BufReaderTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import com.twitter.util.Await 4 | import org.junit.runner.RunWith 5 | import org.scalacheck.Prop 6 | import org.scalatest.FunSuite 7 | import org.scalatest.junit.JUnitRunner 8 | import org.scalatest.prop.Checkers 9 | 10 | @RunWith(classOf[JUnitRunner]) 11 | class BufReaderTest extends FunSuite with Checkers { 12 | import Prop.{forAll, throws} 13 | 14 | test("BufReader") { 15 | check(forAll { (bytes: String) => 16 | val buf = Buf.Utf8(bytes) 17 | val r = BufReader(buf) 18 | Await.result(Reader.readAll(r)) == buf 19 | }) 20 | } 21 | 22 | test("BufReader - discard") { 23 | check(forAll { (bytes: String, n: Int) => 24 | val r = BufReader(Buf.Utf8(bytes)) 25 | r.discard() 26 | n < 0 || 27 | bytes.length == 0 || 28 | throws(classOf[Reader.ReaderDiscarded])({ Await.result(r.read(n)) }) 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/BijectionTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.WordSpec 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class BijectionTest extends WordSpec { 9 | case class Foo(i: Int) 10 | 11 | val fooject = new Bijection[Foo, Int] { 12 | def apply(f: Foo) = f.i 13 | def invert(i: Int) = if (i % 2 == 0) Foo(i) else fail("not really a bijection, natch") 14 | } 15 | 16 | def isAFoo(i: Int) = i match { 17 | case fooject(f) => "a foo! "+ f.toString 18 | case _ => "not a foo" 19 | } 20 | 21 | "Bijection" should { 22 | "return the original when inverting the inverse" in { 23 | assert(fooject.inverse.inverse == fooject) 24 | } 25 | 26 | "can be used for pattern-match" in { 27 | assert(isAFoo(2) == "a foo! Foo(2)") 28 | assert(isAFoo(1) == "not a foo") 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /util-logging/src/main/scala/com/twitter/logging/config/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.logging 18 | 19 | package object config { 20 | type Level = com.twitter.logging.Level 21 | val Level = com.twitter.logging.Level 22 | type Policy = com.twitter.logging.Policy 23 | val Policy = com.twitter.logging.Policy 24 | } 25 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter 18 | 19 | package object util { 20 | // backward compat for people who imported "com.twitter.util.TimeConversions._" 21 | val TimeConversions = com.twitter.conversions.time 22 | val StorageUnitConversions = com.twitter.conversions.storage 23 | } 24 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/concurrent/ConcurrentMultiMapTest.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Twitter, Inc. 2 | 3 | package com.twitter.concurrent 4 | 5 | import org.junit.runner.RunWith 6 | import org.scalatest.WordSpec 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class ConcurrentMultiMapTest extends WordSpec { 11 | "behave like a multimap" in { 12 | val map = new ConcurrentMultiMap[Int, Int] 13 | map += 1 -> 2 14 | map += 1 -> 3 15 | map += 1 -> 4 16 | 17 | assert(map.get(1) === List(2, 3, 4)) 18 | assert(map.get(0) === List()) 19 | assert(map.get(2) === List()) 20 | 21 | map += 0 -> 20 22 | map += 3 -> 30 23 | map += 10 -> 40 24 | 25 | assert(map.get(1) === List(2, 3, 4)) 26 | assert(map.get(0) === List(20)) 27 | assert(map.get(2) === List()) 28 | assert(map.get(3) === List(30)) 29 | assert(map.get(10) === List(40)) 30 | assert(map.get(110) === List()) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/io/FilesTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | 4 | import java.io.File 5 | 6 | import org.junit.runner.RunWith 7 | import org.scalatest.WordSpec 8 | import org.scalatest.junit.JUnitRunner 9 | 10 | import com.twitter.util.TempFolder 11 | 12 | @RunWith(classOf[JUnitRunner]) 13 | class FilesTest extends WordSpec with TempFolder { 14 | "Files" should { 15 | 16 | "delete" in withTempFolder { 17 | val tempFolder = new File(canonicalFolderName) 18 | 19 | val file = new File(tempFolder, "file") 20 | assert(file.createNewFile() === true) 21 | 22 | val folder = new File(tempFolder, "folder") 23 | assert(folder.mkdir() === true) 24 | 25 | val subfile = new File(folder, "file-in-folder") 26 | assert(subfile.createNewFile() === true) 27 | 28 | assert(Files.delete(tempFolder) === true) 29 | Seq(file, subfile, folder, tempFolder).foreach { x => assert(!x.exists) } 30 | } 31 | 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /util-zk-common/src/main/scala/com/twitter/conversions/common/zookeeper.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.conversions.common 2 | 3 | import com.twitter.common.zookeeper.ZooKeeperClient 4 | import com.twitter.conversions.common.quantity.COMMON_FOREVER 5 | import com.twitter.util.{Duration, FuturePool} 6 | import com.twitter.zk.{CommonConnector, ZkClient} 7 | 8 | /** Adapters for common's ZooKeeperClient (and, later, serversets, etc) */ 9 | object zookeeper { 10 | class CommonZkClientAdapter(zkc: ZooKeeperClient) { 11 | def toConnector(timeout: Duration = COMMON_FOREVER) 12 | (implicit pool: FuturePool): CommonConnector = { 13 | new CommonConnector(zkc, timeout) 14 | } 15 | 16 | def toZkClient(timeout: Duration = COMMON_FOREVER)(implicit pool: FuturePool): ZkClient = { 17 | ZkClient(toConnector(timeout)) 18 | } 19 | } 20 | 21 | /** Implicit conversion of ZooKeeperClient to CommonZkClient */ 22 | implicit def commonZkClient(zkc: ZooKeeperClient) = new CommonZkClientAdapter(zkc) 23 | } 24 | -------------------------------------------------------------------------------- /util-hashing/src/test/scala/com/twitter/hashing/DistributionTester.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.hashing 2 | 3 | 4 | import scala.collection.mutable 5 | 6 | class DistributionTester[A](distributor: Distributor[A]) { 7 | 8 | /** 9 | * Returns a normalized standard deviation indicating how well the keys 10 | * are distributed between the nodes. The closer to 0 the better. 11 | */ 12 | def distributionDeviation(keys: Seq[Long]): Double = { 13 | val keysPerNode = mutable.Map[A, Int]() 14 | keys map { distributor.nodeForHash(_) } foreach { key => 15 | if (!keysPerNode.contains(key)) keysPerNode(key) = 0 16 | keysPerNode(key) += 1 17 | } 18 | var frequencies = keysPerNode.values.toList 19 | frequencies ++= 0 until (distributor.nodeCount - frequencies.size) map { _ => 0 } 20 | val average = frequencies.sum.toDouble / frequencies.size 21 | val diffs = frequencies.map { v => math.pow((v - average), 2) } 22 | val sd = math.sqrt(diffs.sum / (frequencies.size - 1)) 23 | sd / average 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /util-logging/src/test/java/com/twitter/logging/LoggerFactoryTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.logging; 2 | 3 | import junit.framework.TestCase; 4 | import scala.Function0; 5 | 6 | // Just make sure this compiles 7 | public class LoggerFactoryTest extends TestCase { 8 | 9 | public void testBuilder() { 10 | LoggerFactoryBuilder builder = LoggerFactory.newBuilder(); 11 | } 12 | 13 | public void testFactory() { 14 | LoggerFactoryBuilder builder = LoggerFactory.newBuilder(); 15 | 16 | LoggerFactoryBuilder intermediate = builder 17 | .node("OK") 18 | .level(Logger.INFO()) 19 | .parentLevel() 20 | .unhandled() 21 | .useParents() 22 | .ignoreParents(); 23 | 24 | LoggerFactory factory = intermediate.build(); 25 | } 26 | 27 | public void testAddHandler() { 28 | LoggerFactoryBuilder builder = LoggerFactory.newBuilder(); 29 | 30 | Function0 handler = StringHandler.apply(); 31 | 32 | LoggerFactoryBuilder intermediate = builder 33 | .addHandler(handler); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /util-core/src/main/java/com/twitter/concurrent/Spools.java: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent; 2 | 3 | import scala.collection.JavaConversions; 4 | import scala.collection.Seq; 5 | 6 | import java.util.ArrayList; 7 | import java.util.Collection; 8 | import java.util.Collections; 9 | 10 | /** 11 | * A Java adaptation of {@link com.twitter.concurrent.Spool} companion object. 12 | */ 13 | public final class Spools { 14 | private Spools() { } 15 | 16 | /** 17 | * @see Spool$#empty() 18 | */ 19 | public static final Spool EMPTY = Spool$.MODULE$.empty(); 20 | 21 | /** 22 | * Creates a new `Spool` of given `elems`. 23 | */ 24 | public static Spool newSpool(Collection elems) { 25 | Seq seq = JavaConversions.asScalaBuffer(new ArrayList(elems)).toSeq(); 26 | return new Spool.ToSpool(seq).toSpool(); 27 | } 28 | 29 | /** 30 | * Creates an empty `Spool`. 31 | */ 32 | public static Spool newEmptySpool() { 33 | Collection empty = Collections.emptyList(); 34 | return Spools.newSpool(empty); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/io/StreamIO.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import java.io.{ByteArrayOutputStream, InputStream, OutputStream} 4 | 5 | import scala.annotation.tailrec 6 | 7 | object StreamIO { 8 | /** 9 | * Copy an InputStream to an OutputStream in chunks of the given 10 | * buffer size (default = 1KB). 11 | */ 12 | @tailrec 13 | final def copy( 14 | inputStream: InputStream, 15 | outputStream: OutputStream, 16 | bufferSize: Int = 1024 17 | ) { 18 | val buf = new Array[Byte](bufferSize) 19 | inputStream.read(buf, 0, buf.length) match { 20 | case -1 => () 21 | case n => 22 | outputStream.write(buf, 0, n) 23 | copy(inputStream, outputStream, bufferSize) 24 | } 25 | } 26 | 27 | /** 28 | * Buffer (fully) the given input stream by creating & copying it to 29 | * a ByteArrayOutputStream. 30 | */ 31 | def buffer(inputStream: InputStream): ByteArrayOutputStream = { 32 | val bos = new java.io.ByteArrayOutputStream 33 | copy(inputStream, bos) 34 | bos 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /util-zk/src/main/scala/com/twitter/zk/LiftableFuture.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | import org.apache.zookeeper.KeeperException 4 | 5 | import com.twitter.util.{Future, Return, Throw} 6 | 7 | protected[zk] object LiftableFuture { 8 | implicit def liftableFuture[T](f: Future[T]) = new LiftableFuture(f) 9 | } 10 | 11 | /** 12 | * Allows Future[T] to be mapped to Future[Try[T]]. This is particularly useful in lifting 13 | * KeepereException.NoNodeExceptions for ZOp.watch(). 14 | */ 15 | protected[zk] class LiftableFuture[T](f: Future[T]) { 16 | /** Lift a value to a Return. */ 17 | def liftSuccess = f map { Return(_) } 18 | 19 | /** Lift all errors to a Throw */ 20 | def liftFailure = liftSuccess handle { case e => Throw(e) } 21 | 22 | /** Lift all KeeperExceptions to a Throw */ 23 | def liftKeeperException = liftSuccess handle { case e: KeeperException => Throw(e) } 24 | 25 | /** Lift failures when a watch would have been successfully installed */ 26 | def liftNoNode = liftSuccess handle { case e: KeeperException.NoNodeException => Throw(e) } 27 | } 28 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/conversions/thread.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter 18 | package conversions 19 | 20 | import java.util.concurrent.Callable 21 | 22 | /** 23 | * Implicits for turning a block of code into a Runnable or Callable. 24 | */ 25 | object thread { 26 | implicit def makeRunnable(f: => Unit): Runnable = new Runnable() { def run() = f } 27 | 28 | implicit def makeCallable[T](f: => T): Callable[T] = new Callable[T]() { def call() = f } 29 | } 30 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/ClosableTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.concurrent.{Eventually, IntegrationPatience} 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | import com.twitter.util.TimeConversions.intToTimeableNumber 9 | 10 | @RunWith(classOf[JUnitRunner]) 11 | class ClosableTest extends FunSuite with Eventually with IntegrationPatience { 12 | test("Closable.close(Duration)") { 13 | Time.withCurrentTimeFrozen { _ => 14 | var time: Option[Time] = None 15 | val c = Closable.make { t => 16 | time = Some(t) 17 | Future.Done 18 | } 19 | val dur = 1.minute 20 | c.close(dur) 21 | assert(time === Some(Time.now + dur)) 22 | } 23 | } 24 | 25 | test("Closable.closeOnCollect") { 26 | @volatile var closed = false 27 | Closable.closeOnCollect( 28 | Closable.make { t => 29 | closed = true 30 | Future.Done 31 | }, 32 | new Object{} 33 | ) 34 | System.gc() 35 | eventually { assert(closed) } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/Stopwatch.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | /** 4 | * A stopwatch may be used to measure elapsed time. 5 | */ 6 | trait Stopwatch { 7 | type Elapsed = () => Duration 8 | 9 | /** 10 | * Start the stopwatch. The returned timer may be read any time, 11 | * returning the duration of time elapsed since start. 12 | */ 13 | def start(): Elapsed 14 | } 15 | 16 | /** 17 | * The system [[com.twitter.util.Stopwatch]] measures elapsed time 18 | * using `System.nanoTime`. 19 | */ 20 | object Stopwatch extends Stopwatch { 21 | 22 | def start(): Elapsed = { 23 | val timeFn = Time.localGetTime().getOrElse(() => Time.fromNanoseconds(System.nanoTime())) 24 | val off = timeFn() 25 | () => timeFn() - off 26 | } 27 | 28 | def const(dur: Duration): Stopwatch = new Stopwatch { 29 | private[this] val fn = () => dur 30 | def start() = fn 31 | } 32 | } 33 | 34 | /** 35 | * A trivial implementation of [[com.twitter.util.Stopwatch]] for use as a null 36 | * object. 37 | */ 38 | object NilStopwatch extends Stopwatch { 39 | def start(): Elapsed = () => Duration.Bottom 40 | } 41 | -------------------------------------------------------------------------------- /util-events/src/main/scala/com/twitter/util/events/Event.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util.events 2 | 3 | import com.twitter.util.Time 4 | 5 | object Event { 6 | 7 | val NoObject: AnyRef = new Object() 8 | val NoLong: Long = Long.MinValue 9 | val NoDouble: Double = Double.NegativeInfinity 10 | 11 | /** 12 | * Represents a type of event that can be recorded. 13 | * 14 | * A recommended usage pattern is to create a singleton 15 | * in the companion object and use that for all events 16 | * of that type. 17 | */ 18 | abstract class Type 19 | 20 | } 21 | 22 | /** 23 | * A somewhat flexible schema representing various event types. 24 | * 25 | * @param when when the event happened. 26 | * @param longVal should be `Event.NoLong` if there is no supplied value. 27 | * @param objectVal should be `Event.NoObject` if there is no supplied value. 28 | * @param doubleVal should be `Event.NoDouble` if there is no supplied value. 29 | */ 30 | case class Event( 31 | etype: Event.Type, 32 | when: Time, 33 | longVal: Long = Event.NoLong, 34 | objectVal: Object = Event.NoObject, 35 | doubleVal: Double = Event.NoDouble) 36 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/DiffTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | import org.scalacheck.Arbitrary.arbitrary 7 | import org.scalatest.prop.GeneratorDrivenPropertyChecks 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class DiffTest extends FunSuite with GeneratorDrivenPropertyChecks { 11 | val f: Int => String = _.toString 12 | 13 | test("Diffable.ofSet") { 14 | forAll(arbitrary[Set[Int]], arbitrary[Set[Int]]) { (a, b) => 15 | Diffable.diff(a, b).patch(a) === b 16 | } 17 | 18 | forAll(arbitrary[Set[Int]], arbitrary[Set[Int]]) { (a, b) => 19 | val diff = Diffable.diff(a, b) 20 | diff.map(f).patch(a.map(f)) === b.map(f) 21 | } 22 | } 23 | 24 | test("Diffable.ofSeq") { 25 | forAll(arbitrary[Seq[Int]], arbitrary[Seq[Int]]) { (a, b) => 26 | Diffable.diff(a, b).patch(a) === b 27 | } 28 | 29 | forAll(arbitrary[Seq[Int]], arbitrary[Seq[Int]]) { (a, b) => 30 | val diff = Diffable.diff(a, b) 31 | diff.map(f).patch(a.map(f)) === b.map(f) 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /util-zk/src/main/scala/com/twitter/zk/ZOp.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | import com.twitter.concurrent.{Broker, Offer} 4 | import com.twitter.util.{Future, Try} 5 | 6 | /** A ZNode-read operation. */ 7 | trait ZOp[T <: ZNode.Exists] { 8 | /** Returns a Future that is satisfied when the operation is complete. */ 9 | def apply(): Future[T] 10 | 11 | /** 12 | * Get a ZNode and watch for updates to it. 13 | * If the node does not exist the returned Future is satisfied with a ZNode.Watch containing 14 | * a Throw(KeeperException.NoNodeException). 15 | */ 16 | def watch(): Future[ZNode.Watch[T]] 17 | 18 | /** Repeatedly performs a Watch operation and publishes results on an Offer. */ 19 | def monitor(): Offer[Try[T]] = { 20 | val broker = new Broker[Try[T]] 21 | // Set the watch, send the result to the broker, and repeat this when an event occurs 22 | def setWatch() { 23 | watch() onSuccess { case ZNode.Watch(result, update) => 24 | broker ! result onSuccess { _ => 25 | update onSuccess { _ => setWatch() } 26 | } 27 | } 28 | } 29 | setWatch() 30 | broker.recv 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/AwaitableTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class CloseAwaitablyTest extends FunSuite { 9 | def make() = new Closable with CloseAwaitably { 10 | val p = new Promise[Unit] 11 | var n = 0 12 | def close(deadline: Time) = closeAwaitably { 13 | n += 1 14 | p 15 | } 16 | } 17 | 18 | test("close") { 19 | val c = make() 20 | assert(c.n == 0) 21 | val f = c.close(Time.now) 22 | assert(f != c.p) 23 | assert(c.n == 1) 24 | assert(c.close(Time.now) == f) 25 | assert(c.n == 1) 26 | assert(f.poll == None) 27 | c.p.setDone() 28 | assert(f.poll == Some(Return.Unit)) 29 | } 30 | 31 | test("Await.ready") { 32 | val c = make() 33 | val t = new Thread { 34 | start() 35 | override def run() { 36 | Await.ready(c) 37 | } 38 | } 39 | 40 | c.close(Time.now) 41 | assert(t.isAlive) 42 | c.p.setDone() 43 | t.join(10000) 44 | assert(!t.isAlive) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/concurrent/ConcurrentPoolTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.WordSpec 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class ConcurrentPoolTest extends WordSpec { 9 | "reserve items" in { 10 | val pool = new ConcurrentPool[Int, Int] 11 | 12 | pool.put(1, 2) 13 | assert(pool.get(1) === Some(2)) 14 | assert(pool.get(1) === None) 15 | } 16 | 17 | "yield items in FIFO order" in { 18 | val pool = new ConcurrentPool[Int, Int] 19 | 20 | for (i <- 0 until 10) 21 | pool.put(1, i) 22 | 23 | for (i <- 0 until 10) 24 | assert(pool.get(1) === Some(i)) 25 | 26 | assert(pool.get(1) === None) 27 | } 28 | 29 | "kill empty lists" in { 30 | val pool = new ConcurrentPool[Int, Int] 31 | 32 | pool.put(1, 1) 33 | assert(pool.get(1) === Some(1)) 34 | 35 | assert(pool.map.containsKey(1) === false) 36 | assert(pool.deathQueue.size === 1) 37 | 38 | pool.put(2, 1) 39 | assert(pool.deathQueue.size === 0) 40 | } 41 | 42 | // Can't really test the race condition case :-/ 43 | } 44 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/StateMachineTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.WordSpec 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class StateMachineTest extends WordSpec { 9 | "StateMachine" should { 10 | class StateMachineHelper { 11 | val stateMachine = new StateMachine { 12 | case class State1() extends State 13 | case class State2() extends State 14 | state = State1() 15 | 16 | def command1() { 17 | transition("command1") { 18 | case State1() => state = State2() 19 | } 20 | } 21 | } 22 | } 23 | 24 | "allows transitions that are permitted" in { 25 | val h = new StateMachineHelper 26 | import h._ 27 | 28 | stateMachine.command1() 29 | } 30 | 31 | "throws exceptions when a transition is not permitted" in { 32 | val h = new StateMachineHelper 33 | import h._ 34 | 35 | stateMachine.command1() 36 | intercept[StateMachine.InvalidStateTransition] { 37 | stateMachine.command1() 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /util-collection/src/main/scala/com/twitter/util/SetMaker.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import scala.collection.JavaConverters._ 4 | import scala.collection.mutable.{Map, Set} 5 | 6 | import com.google.common.collect.{MapMaker => GoogleMapMaker} 7 | 8 | object SetMaker { 9 | def apply[A](f: Config[A] => Any): Set[A] = { 10 | val config = new Config[A] 11 | f(config) 12 | config() 13 | } 14 | 15 | class Config[A] { 16 | private val mapMaker = new GoogleMapMaker 17 | 18 | def weakValues = { mapMaker.weakKeys; mapMaker.weakValues; this } 19 | def concurrencyLevel(level: Int) = { mapMaker.concurrencyLevel(level); this } 20 | def initialCapacity(capacity: Int) = { mapMaker.initialCapacity(capacity); this } 21 | 22 | def apply() = new MapToSetAdapter[A]( 23 | mapMaker.makeMap().asScala) 24 | } 25 | } 26 | 27 | class MapToSetAdapter[A](map: Map[A, A]) extends Set[A] { 28 | def +=(elem: A) = { 29 | map(elem) = elem 30 | this 31 | } 32 | def -=(elem: A) = { 33 | map -= elem 34 | this 35 | } 36 | override def size = map.size 37 | def iterator = map.keysIterator 38 | def contains(elem: A) = map.contains(elem) 39 | } 40 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/LongOverflowArith.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | class LongOverflowException(msg: String) extends Exception(msg) 4 | 5 | object LongOverflowArith { 6 | def add(a: Long, b: Long) = { 7 | val c = a + b 8 | if (((a ^ c) & (b ^ c)) < 0) 9 | throw new LongOverflowException(a + " + " + b) 10 | else 11 | c 12 | } 13 | 14 | def sub(a: Long, b: Long) = { 15 | val c = a - b 16 | if (((a ^ c) & (-b ^ c)) < 0) 17 | throw new LongOverflowException(a + " - " + b) 18 | else 19 | c 20 | } 21 | 22 | def mul(a: Long, b: Long): Long = { 23 | if (a > b) { 24 | // normalize so that a <= b to keep conditionals to a minimum 25 | mul(b, a) 26 | } else if (a < 0L) { 27 | if (b < 0L) { 28 | if (a < Long.MaxValue / b) throw new LongOverflowException(a + " * " + b) 29 | } else if (b > 0L) { 30 | if (Long.MinValue / b > a) throw new LongOverflowException(a + " * " + b) 31 | } 32 | } else if (a > 0L) { 33 | // and b > 0L 34 | if (a > Long.MaxValue / b) throw new LongOverflowException(a + " * " + b) 35 | } 36 | 37 | a * b 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/Base64LongTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import scala.util.Random 4 | 5 | import org.junit.runner.RunWith 6 | import org.scalatest.WordSpec 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | import com.twitter.util.Base64Long.toBase64 10 | 11 | @RunWith(classOf[JUnitRunner]) 12 | class Base64LongTest extends WordSpec { 13 | "toBase64" should { 14 | "properly convert zero" in { 15 | assert(toBase64(0) == "A") 16 | } 17 | 18 | "properly convert a large number" in { 19 | assert(toBase64(202128261025763330L) == "LOGpUdghAC") 20 | } 21 | 22 | "Use the expected number of digits" in { 23 | val expectedLength: Long => Int = { 24 | case 0 => 1 // Special case in the implementation 25 | case n if n < 0 => 11 // High bit set, treated as unsigned 26 | case n => (math.log(n + 1)/math.log(64)).ceil.toInt 27 | } 28 | val checkExpectedLength = (n: Long) => assert(toBase64(n).length == expectedLength(n)) 29 | Seq(0L, 1L, 63L, 64L, 4095L, 4096L, -1L) foreach checkExpectedLength 30 | (1 to 200) foreach { _ => checkExpectedLength(Random.nextLong) } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/concurrent/Serialized.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | import java.util.concurrent.ConcurrentLinkedQueue 4 | import java.util.concurrent.atomic.AtomicInteger 5 | 6 | import com.twitter.util.{Future, Promise, Try} 7 | 8 | /** 9 | * Efficient ordered ''serialization'' of operations. 10 | * 11 | * '''Note:''' This should not be used in place of Scala's 12 | * `synchronized`, but rather only when serialization semantics are 13 | * required. 14 | */ 15 | trait Serialized { 16 | protected case class Job[T](promise: Promise[T], doItToIt: () => T) { 17 | def apply() { 18 | promise.update { Try { doItToIt() } } 19 | } 20 | } 21 | 22 | private[this] val nwaiters = new AtomicInteger(0) 23 | protected val serializedQueue: java.util.Queue[Job[_]] = new ConcurrentLinkedQueue[Job[_]] 24 | 25 | protected def serialized[A](f: => A): Future[A] = { 26 | val result = new Promise[A] 27 | 28 | serializedQueue add { Job(result, () => f) } 29 | 30 | if (nwaiters.getAndIncrement() == 0) { 31 | do { 32 | Try { serializedQueue.remove()() } 33 | } while (nwaiters.decrementAndGet() > 0) 34 | } 35 | 36 | result 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.logging 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | import com.twitter.util.StorageUnit 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class PolicyTest extends FunSuite { 11 | import Policy._ 12 | 13 | test("Policy.parse: never") { 14 | assert(parse("never") === Never) 15 | } 16 | 17 | test("Policy.parse: hourly") { 18 | assert(parse("hourly") === Hourly) 19 | } 20 | 21 | test("Policy.parse: daily") { 22 | assert(parse("daily") === Daily) 23 | } 24 | 25 | test("Policy.parse: sighup") { 26 | assert(parse("sighup") === SigHup) 27 | } 28 | 29 | test("Policy.parse: weekly") { 30 | assert(parse("weekly(3)") == Weekly(3)) 31 | } 32 | 33 | test("Policy.parse: maxsize") { 34 | val size = "3.megabytes" 35 | assert(parse(size) == MaxSize(StorageUnit.parse(size))) 36 | } 37 | 38 | test("Policy.parse: should be case-insensitive") { 39 | assert(parse("DAily") === Daily) 40 | assert(parse("weEkLy(3)") == Weekly(3)) 41 | assert(parse("3.meGabYteS") == MaxSize(StorageUnit.parse("3.megabytes"))) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | import com.google.caliper.SimpleBenchmark 4 | import com.twitter.concurrent.Tx.{Commit, Result} 5 | import com.twitter.util.{Future, Promise} 6 | import scala.util.Random 7 | 8 | class OfferBenchmark extends SimpleBenchmark { 9 | 10 | private class SimpleOffer[T](fut: Future[Tx[T]]) extends Offer[T] { 11 | override def prepare() = fut 12 | } 13 | private class SimpleTx[T](t: T) extends Tx[T] { 14 | private val futAck = Future.value(Commit(t)) 15 | override def nack(): Unit = () 16 | override def ack(): Future[Result[T]] = futAck 17 | } 18 | 19 | def timeChoose(reps: Int) { 20 | val tx = Tx.const(5) 21 | val data: Array[Seq[Offer[Int]]] = Array.fill(reps) { 22 | val pendingTxs = Seq.fill(3) { new Promise[Tx[Int]] } 23 | val offers = pendingTxs map { p => new SimpleOffer[Int](p) } 24 | pendingTxs(1).setValue(tx) // mark one as completed 25 | offers 26 | } 27 | 28 | val rnd = new Random(4125128989L) 29 | var i = 0 30 | while (i < reps) { 31 | val offers = data(i) 32 | Offer.choose(rnd, offers) 33 | i += 1 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /util-benchmark/src/main/scala/com/twitter/io/benchmark/Buf.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io.benchmark 2 | 3 | import com.google.caliper.SimpleBenchmark 4 | import com.twitter.io.{Buf, ConcatBuf} 5 | 6 | class BufBenchmark extends SimpleBenchmark { 7 | val N = 1000 8 | val delta = 10 9 | 10 | def mkConcattedBuf(size: Int, parts: Int): Buf = 11 | (for (i <- 0 until parts) yield Buf.ByteArray(Array.fill[Byte](size / parts)(i.toByte))) reduce (_ concat _) 12 | 13 | val simple = mkConcattedBuf(N, 4) 14 | 15 | def timeSimpleSlice(nreps: Int) = { 16 | var i = 0 17 | while (i < nreps) { 18 | sliceBench(simple) 19 | i += 1 20 | } 21 | } 22 | 23 | val complex = mkConcattedBuf(N, 100) 24 | 25 | /** 26 | * We use while loops here so that we don't add closure allocations in the loop. 27 | */ 28 | @inline 29 | def sliceBench(buf: Buf) { 30 | var j = 0 31 | while (j < N) { 32 | var k = j 33 | while (k < N + delta) { 34 | buf.slice(j, k) 35 | k += 1 36 | } 37 | j += 1 38 | } 39 | } 40 | 41 | def timeComplexSlice(nreps: Int) = { 42 | var i = 0 43 | while (i < nreps) { 44 | sliceBench(complex) 45 | i += 1 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /util-thrift/src/main/scala/com/twitter/util/ThriftCodec.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import org.apache.thrift.TBase 4 | import org.apache.thrift.protocol.{TBinaryProtocol, TCompactProtocol, TProtocolFactory} 5 | 6 | object ThriftCodec { 7 | def apply[T <: TBase[_, _]: Manifest, P <: TProtocolFactory: Manifest]: ThriftCodec[T, P] = 8 | new ThriftCodec[T, P] 9 | } 10 | 11 | class ThriftCodec[T <: TBase[_, _]: Manifest, P <: TProtocolFactory: Manifest] 12 | extends Codec[T, Array[Byte]] 13 | with ThriftSerializer { 14 | 15 | protected lazy val prototype: T = 16 | manifest[T].runtimeClass.asInstanceOf[Class[T]].newInstance 17 | 18 | override lazy val protocolFactory: TProtocolFactory = 19 | manifest[P].runtimeClass.asInstanceOf[Class[P]].newInstance 20 | 21 | override def encode(item: T): Array[Byte] = toBytes(item) 22 | 23 | override def decode(bytes: Array[Byte]): T = { 24 | val obj = prototype.deepCopy 25 | fromBytes(obj, bytes) 26 | obj.asInstanceOf[T] 27 | } 28 | } 29 | 30 | class BinaryThriftCodec[T <: TBase[_, _]: Manifest] 31 | extends ThriftCodec[T, TBinaryProtocol.Factory] 32 | 33 | class CompactThriftCodec[T <: TBase[_, _]: Manifest] 34 | extends ThriftCodec[T, TCompactProtocol.Factory] 35 | -------------------------------------------------------------------------------- /util-core/src/main/java/com/twitter/util/Witnesses.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | import scala.runtime.BoxedUnit; 4 | 5 | import java.util.concurrent.atomic.AtomicReference; 6 | 7 | /** 8 | * A Java adaptation of {@link com.twitter.util.Witness} companion object. 9 | */ 10 | public final class Witnesses { 11 | private Witnesses() { } 12 | 13 | /** 14 | * @see com.twitter.util.Witness$#apply(java.util.concurrent.atomic.AtomicReference) ; 15 | */ 16 | public static Witness newWitness(AtomicReference reference) { 17 | return Witness$.MODULE$.apply(reference); 18 | } 19 | 20 | /** 21 | * @see com.twitter.util.Witness$#apply(Promise) 22 | */ 23 | public static Witness newWitness(Promise promise) { 24 | return Witness$.MODULE$.apply(promise); 25 | } 26 | 27 | /** 28 | * @see com.twitter.util.Witness$#apply(scala.Function1) 29 | */ 30 | public static Witness newWitness(Function function) { 31 | return Witness$.MODULE$.apply(function); 32 | } 33 | 34 | /** 35 | * @see com.twitter.util.Witness$#apply(Updatable) 36 | */ 37 | public static Witness newWitness(Updatable updatable) { 38 | return Witness$.MODULE$.apply(updatable); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /util-jvm/src/test/scala/com/twitter/jvm/OptsTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import java.lang.management.ManagementFactory 4 | import javax.management.ObjectName 5 | import org.junit.runner.RunWith 6 | import org.scalatest.FunSuite 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class OptsTest extends FunSuite { 11 | test("Opts") { 12 | if (System.getProperty("java.vm.name").contains("HotSpot")) { 13 | val DiagnosticName = 14 | ObjectName.getInstance("com.sun.management:type=HotSpotDiagnostic") 15 | 16 | val originalValue: String = Opt("MaxHeapFreeRatio").getOrElse("100") 17 | 18 | ManagementFactory.getPlatformMBeanServer().invoke( 19 | DiagnosticName, "setVMOption", 20 | Array("MaxHeapFreeRatio", "99"), 21 | Array("java.lang.String", "java.lang.String")) 22 | 23 | assert(Opt("MaxHeapFreeRatio") === Some("99")) 24 | 25 | ManagementFactory.getPlatformMBeanServer().invoke( 26 | DiagnosticName, "setVMOption", 27 | Array("MaxHeapFreeRatio", originalValue), 28 | Array("java.lang.String", "java.lang.String")) 29 | 30 | assert(Opt("MaxHeapFreeRatio") === Some(originalValue)) 31 | 32 | assert(Opt("NonexistentOption") === None) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.logging 18 | 19 | import java.util.{logging => javalog} 20 | 21 | class LazyLogRecord( 22 | level: javalog.Level, 23 | messageGenerator: => AnyRef 24 | ) extends LogRecord(level, "") { 25 | 26 | override lazy val getMessage = messageGenerator.toString 27 | } 28 | 29 | /** 30 | * A lazy LogRecord that needs formatting 31 | */ 32 | class LazyLogRecordUnformatted(level: javalog.Level, message: String, items: Any*) 33 | extends LazyLogRecord(level, { message.format(items: _*) }) { 34 | require(items.size > 0) 35 | val preformatted = message 36 | } 37 | -------------------------------------------------------------------------------- /util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | /** 4 | * @author ver@twitter.com 5 | */ 6 | 7 | 8 | import org.junit.runner.RunWith 9 | import org.scalatest.WordSpec 10 | import org.scalatest.junit.JUnitRunner 11 | import org.scalatest.mock.MockitoSugar 12 | 13 | @RunWith(classOf[JUnitRunner]) 14 | class ZNodeTest extends WordSpec with MockitoSugar { 15 | "ZNode" should { 16 | class ZNodeSpecHelper { 17 | val zk = mock[ZkClient] 18 | } 19 | def pathTest(path: String, parent: String, name: String) { 20 | val h = new ZNodeSpecHelper 21 | import h._ 22 | 23 | val znode = ZNode(zk, path) 24 | path should { 25 | "parentPath" in { assert(znode.parentPath === parent) } 26 | "name" in { assert(znode.name === name ) } 27 | } 28 | } 29 | 30 | pathTest("/", "/", "") 31 | pathTest("/some/long/path/to/a/znode", "/some/long/path/to/a", "znode") 32 | pathTest("/path", "/", "path") 33 | 34 | "hash together" in { 35 | val h = new ZNodeSpecHelper 36 | import h._ 37 | 38 | val zs = (0 to 1) map { _ => ZNode(zk, "/some/path") } 39 | val table = Map(zs(0) -> true) 40 | assert(table.keys.toList.contains(zs(0))) 41 | assert(table.keys.toList.contains(zs(1))) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /sbt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sbtver=0.13.7 4 | sbtjar=sbt-launch.jar 5 | sbtsha128=b407b2a76ad72165f806ac7e7ea09132b951ef53 6 | 7 | sbtrepo=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch 8 | 9 | if [ ! -f $sbtjar ]; then 10 | echo "downloading $sbtjar" 1>&2 11 | if ! curl --silent --fail --remote-name $sbtrepo/$sbtver/$sbtjar; then 12 | exit 1 13 | fi 14 | fi 15 | 16 | checksum=`openssl dgst -sha1 $sbtjar | awk '{ print $2 }'` 17 | if [ "$checksum" != $sbtsha128 ]; then 18 | echo "bad $sbtjar. delete $sbtjar and run $0 again." 19 | exit 1 20 | fi 21 | 22 | [ -f ~/.sbtconfig ] && . ~/.sbtconfig 23 | 24 | java -ea \ 25 | $SBT_OPTS \ 26 | $JAVA_OPTS \ 27 | -Djava.net.preferIPv4Stack=true \ 28 | -XX:+AggressiveOpts \ 29 | -XX:+UseParNewGC \ 30 | -XX:+UseConcMarkSweepGC \ 31 | -XX:+CMSParallelRemarkEnabled \ 32 | -XX:+CMSClassUnloadingEnabled \ 33 | -XX:ReservedCodeCacheSize=128m \ 34 | -XX:MaxPermSize=1024m \ 35 | -XX:SurvivorRatio=128 \ 36 | -XX:MaxTenuringThreshold=0 \ 37 | -Xss8M \ 38 | -Xms512M \ 39 | -Xmx2G \ 40 | -server \ 41 | -jar $sbtjar "$@" 42 | -------------------------------------------------------------------------------- /util-jvm/src/main/scala/com/twitter/jvm/GcPredictor.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import com.twitter.conversions.time._ 4 | import com.twitter.util.{Duration, Time, Timer} 5 | 6 | /** 7 | * A Gc predictor. This predicts a time based on measuring rates, 8 | * which are reported to the given estimator. Rates are measured 9 | * every `period`. The estimated GC time is based on interpolating 10 | * the rate estimated by `estimator`. 11 | */ 12 | class GcPredictor(pool: Pool, period: Duration, timer: Timer, estimator: Estimator[Double]) { 13 | private[this] def loop() { 14 | for (bps <- pool.estimateAllocRate(period, timer)) { 15 | synchronized { estimator.measure(bps.toDouble) } 16 | loop() 17 | } 18 | } 19 | loop() 20 | 21 | def nextGcEstimate(): Time = { 22 | val e = synchronized(estimator.estimate).toLong 23 | if (e == 0) Time.Top else { 24 | val PoolState(_, capacity, used) = pool.state() 25 | val r = (capacity - used).inBytes 26 | Time.now + ((1000*r) / e).milliseconds 27 | } 28 | } 29 | } 30 | 31 | // Note: it may be preoductive to make use of several inputs 32 | // (average, short-term average, instantaneous rate) and average them 33 | // using something like ARIMA, or using a Kalman filter for the 34 | // output of *that*. This can be done by composing Estimators. 35 | -------------------------------------------------------------------------------- /util-jvm/src/test/scala/com/twitter/jvm/MockScheduledExecutorService.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import java.util.concurrent.{AbstractExecutorService, Callable, ScheduledExecutorService, ScheduledFuture, TimeUnit} 4 | 5 | import scala.collection.mutable 6 | 7 | // A mostly empty implementation so that we can successfully 8 | // mock it. 9 | class MockScheduledExecutorService extends AbstractExecutorService with ScheduledExecutorService { 10 | val schedules = mutable.Buffer[(Runnable, Long, Long, TimeUnit)]() 11 | 12 | def schedule[V](c: Callable[V], delay: Long, unit: TimeUnit) = throw new Exception 13 | def schedule(command: Runnable, delay: Long, unit: TimeUnit) = throw new Exception 14 | def scheduleWithFixedDelay(command: Runnable, initialDelay: Long, delay: Long, unit: TimeUnit) = 15 | throw new Exception 16 | def execute(command: Runnable) = throw new Exception 17 | def awaitTermination(timeout: Long, unit: TimeUnit) = throw new Exception 18 | def isTerminated() = throw new Exception 19 | def isShutdown() = throw new Exception 20 | def shutdownNow() = throw new Exception 21 | def shutdown() = throw new Exception 22 | 23 | def scheduleAtFixedRate(r: Runnable, initialDelay: Long, period: Long, unit: TimeUnit): ScheduledFuture[_] = { 24 | schedules += ((r, initialDelay, period, unit)) 25 | null 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /util-logging/src/test/scala/com/twitter/logging/AppTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.logging 18 | 19 | import org.junit.runner.RunWith 20 | import org.scalatest.FunSuite 21 | import org.scalatest.junit.JUnitRunner 22 | 23 | import com.twitter.app.App 24 | 25 | @RunWith(classOf[JUnitRunner]) 26 | class AppTest extends FunSuite { 27 | 28 | object TestLoggingApp extends App with Logging { 29 | override def handlers = ScribeHandler() :: super.handlers 30 | } 31 | 32 | test("TestLoggingApp should have one factory with two log handlers") { 33 | TestLoggingApp.main(Array.empty) 34 | assert(TestLoggingApp.loggerFactories.size === 1) 35 | assert(TestLoggingApp.loggerFactories.head.handlers.size === 2) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /util-core/src/test/java/com/twitter/util/FunctionCompilationTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | import org.junit.Test; 4 | 5 | /** 6 | * Tests are not currently run for java, but for our purposes, if the test compiles at all, it's 7 | * a success. 8 | */ 9 | public class FunctionCompilationTest { 10 | 11 | /** Confirm that we can extend ExceptionalFunction with applyE(). */ 12 | @Test 13 | public void testDefineWithException() { 14 | ExceptionalFunction fun = new ExceptionalFunction() { 15 | @Override 16 | public String applyE(Integer in) throws Exception { 17 | throw new Exception("Expected"); 18 | } 19 | }; 20 | try { 21 | fun.apply(1); 22 | assert false : "Should have thrown"; 23 | } catch (Exception e) { 24 | // pass: expected 25 | } 26 | } 27 | 28 | /** Confirm that we can extend ExceptionalFunction0 with applyE(). */ 29 | @Test 30 | public void testExceptionalFunction0() { 31 | ExceptionalFunction0 fun = new ExceptionalFunction0() { 32 | @Override 33 | public Integer applyE() throws Exception { 34 | throw new Exception("Expected"); 35 | } 36 | }; 37 | try { 38 | fun.apply(); 39 | assert false : "Should have thrown"; 40 | } catch (Exception e) { 41 | // pass: expected 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/RandomSocket.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import java.io.IOException 4 | import java.net.{InetSocketAddress, Socket} 5 | 6 | /** 7 | * A generator of random local [[java.net.InetSocketAddress]] objects with 8 | * ephemeral ports. 9 | */ 10 | object RandomSocket { 11 | private[this] def localSocketOnPort(port: Int) = 12 | new InetSocketAddress(port) 13 | private[this] val ephemeralSocketAddress = localSocketOnPort(0) 14 | 15 | @deprecated("RandomSocket cannot ensure that the address is not in use.", "2014-11-13") 16 | def apply() = nextAddress() 17 | 18 | @deprecated("RandomSocket cannot ensure that the address is not in use.", "2014-11-13") 19 | def nextAddress(): InetSocketAddress = 20 | localSocketOnPort(nextPort()) 21 | 22 | @deprecated("RandomSocket cannot ensure that the address is not in use.", "2014-11-13") 23 | def nextPort(): Int = { 24 | val s = new Socket 25 | s.setReuseAddress(true) 26 | try { 27 | s.bind(ephemeralSocketAddress) 28 | s.getLocalPort 29 | } catch { 30 | case NonFatal(e) => 31 | if (e.getClass == classOf[IOException] || e.getClass == classOf[IllegalArgumentException]) 32 | throw new Exception("Couldn't find an open port: %s".format(e.getMessage)) 33 | else 34 | throw e 35 | } finally { 36 | s.close() 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | import java.util.concurrent.CountDownLatch 4 | 5 | import org.junit.runner.RunWith 6 | import org.scalatest.WordSpec 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | 10 | @RunWith(classOf[JUnitRunner]) 11 | class SerializedTest extends WordSpec with Serialized { 12 | "Serialized" should { 13 | "runs blocks, one at a time, in the order received" in { 14 | val t1CallsSerializedFirst = new CountDownLatch(1) 15 | val t1FinishesWork = new CountDownLatch(1) 16 | val orderOfExecution = new collection.mutable.ListBuffer[Thread] 17 | 18 | val t1 = new Thread { 19 | override def run { 20 | serialized { 21 | t1CallsSerializedFirst.countDown() 22 | t1FinishesWork.await() 23 | orderOfExecution += this 24 | () 25 | } 26 | } 27 | } 28 | 29 | val t2 = new Thread { 30 | override def run { 31 | t1CallsSerializedFirst.await() 32 | serialized { 33 | orderOfExecution += this 34 | () 35 | } 36 | t1FinishesWork.countDown() 37 | } 38 | } 39 | 40 | t1.start() 41 | t2.start() 42 | t1.join() 43 | t2.join() 44 | 45 | assert(orderOfExecution.toList === List(t1, t2)) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache 2 | 3 | import com.twitter.util.{Promise, Future} 4 | import scala.annotation.tailrec 5 | import java.util.concurrent.ConcurrentMap 6 | 7 | /** 8 | * A [[com.twitter.cache.FutureCache]] backed by a 9 | * [[java.util.concurrent.ConcurrentMap]] 10 | * 11 | * Any correct implementation should make sure that you evict failed 12 | * results, and don't interrupt the underlying request that has been 13 | * fired off. [[EvictingCache]] and [[Future#interrupting]] are 14 | * useful tools for building correct FutureCaches. A reference 15 | * implementation for caching the results of an asynchronous function 16 | * with a [[ConcurrentMap]] can be found at [[FutureCache$.fromMap]]. 17 | */ 18 | class ConcurrentMapCache[K, V](underlying: ConcurrentMap[K, Future[V]]) extends FutureCache[K, V] { 19 | def get(key: K): Option[Future[V]] = Option(underlying.get(key)) 20 | 21 | def set(key: K, value: Future[V]): Unit = underlying.put(key, value) 22 | 23 | def getOrElseUpdate(key: K)(compute: => Future[V]): Future[V] = { 24 | val p = Promise[V] 25 | underlying.putIfAbsent(key, p) match { 26 | case null => 27 | p.become(compute) 28 | p 29 | case oldv => oldv 30 | } 31 | } 32 | 33 | def evict(key: K, value: Future[V]): Boolean = underlying.remove(key, value) 34 | 35 | def size: Int = underlying.size() 36 | } 37 | -------------------------------------------------------------------------------- /util-cache/src/main/scala/com/twitter/cache/KeyEncodingCache.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache 2 | 3 | import com.twitter.util.Future 4 | 5 | /** 6 | * Encodes keys, producing a Cache that takes keys of a different type. 7 | * 8 | * Useful for compressing keys, or discarding extra information. This is 9 | * especially useful when information is useful for computing a value for the 10 | * first time, but isn't necessary for demonstrating distinctness. 11 | * 12 | * e.g. 13 | * 14 | * val fibCache: FutureCache[Int, Int] 15 | * val fn: (Int, Int, Int, Int) => Int = { case (target, prev, cur, idx) => 16 | * if (idx == target) cur else fn((target, cur, prev + cur, idx + 1)) 17 | * } 18 | * val memo: (Int, Int, Int, Int) => Int = Cache(fn, new KeyEncodingCache( 19 | * new FutureCache[Int, Int], 20 | * { case (target: Int, _, _, _) => target } 21 | * )) 22 | */ 23 | private[cache] class KeyEncodingCache[K, V, U]( 24 | encode: K => V, 25 | underlying: FutureCache[V, U] 26 | ) extends FutureCache[K, U] { 27 | override def get(key: K): Option[Future[U]] = underlying.get(encode(key)) 28 | 29 | def set(key: K, value: Future[U]): Unit = underlying.set(encode(key), value) 30 | 31 | def getOrElseUpdate(key: K)(compute: => Future[U]): Future[U] = 32 | underlying.getOrElseUpdate(encode(key))(compute) 33 | 34 | def evict(key: K, value: Future[U]): Boolean = underlying.evict(encode(key), value) 35 | 36 | def size: Int = underlying.size 37 | } 38 | -------------------------------------------------------------------------------- /util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.logging 2 | 3 | import java.util.logging.{Level => JLevel, LogRecord => JRecord} 4 | 5 | import org.junit.runner.RunWith 6 | import org.scalatest.FunSuite 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class LogRecordTest extends FunSuite { 11 | test("LogRecord should getMethod properly") { 12 | Logger.withLoggers(Nil) { 13 | new LogRecordTestHelper({ r: JRecord => r.getSourceMethodName() }) { 14 | def makingLogRecord() { 15 | logger.log(Level.INFO, "OK") 16 | assert(handler.get === "makingLogRecord") 17 | } 18 | makingLogRecord() 19 | } 20 | } 21 | } 22 | 23 | test("LogRecord should getClass properly") { 24 | Logger.withLoggers(Nil) { 25 | new Foo { 26 | assert(handler.get === "com.twitter.logging.Foo") 27 | } 28 | } 29 | } 30 | } 31 | 32 | abstract class LogRecordTestHelper(formats: JRecord => String) { 33 | val formatter = new Formatter { 34 | override def format(r: JRecord): String = formats(r) 35 | } 36 | val handler = new StringHandler(formatter) 37 | val logger = Logger.get("") 38 | logger.addHandler(handler) 39 | } 40 | 41 | class Foo extends LogRecordTestHelper({ r: JRecord => r.getSourceClassName() }) { 42 | def makingLogRecord() { 43 | logger.log(Level.INFO, "OK") 44 | } 45 | 46 | makingLogRecord() 47 | } 48 | -------------------------------------------------------------------------------- /util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import com.twitter.conversions.time._ 4 | import com.twitter.util.{Duration, Time} 5 | import java.io.ByteArrayOutputStream 6 | import org.junit.runner.RunWith 7 | import org.scalatest.FunSuite 8 | import org.scalatest.junit.JUnitRunner 9 | 10 | @RunWith(classOf[JUnitRunner]) 11 | class CpuProfileTest extends FunSuite { 12 | test("record") { 13 | 14 | // record() calls Time.now 3 times initially, and then 3 times on every loop iteration. 15 | val times: Stream[Int] = (0 #:: Stream.from(0)).map(x => List(x, x, x)).flatten 16 | val iter = times.iterator 17 | val start = Time.now 18 | def nextTime: Time = start + (iter.next.milliseconds)*10 19 | 20 | val t = new Thread("CpuProfileTest") { 21 | override def run() { 22 | Thread.sleep(10000) 23 | } 24 | } 25 | t.setDaemon(true) 26 | t.start() 27 | 28 | // Profile for 100ms at 100 Hz => 10ms period; produces 10 samples. 29 | val profile: CpuProfile = Time.withTimeFunction(nextTime) { _ => 30 | CpuProfile.record(100.milliseconds, 100, Thread.State.TIMED_WAITING) 31 | } 32 | 33 | assert(profile.count === 10) 34 | assert(profile.missed === 0) 35 | 36 | val baos = new ByteArrayOutputStream 37 | profile.writeGoogleProfile(baos) 38 | assert(baos.toString.contains("CpuProfileTest.scala")) 39 | assert(baos.toString.contains("Thread.sleep")) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/NonFatal.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import scala.util.control.ControlThrowable 4 | 5 | /** 6 | * A classifier of fatal exceptions 7 | */ 8 | object NonFatal { 9 | /** 10 | * This is identitical in behavior to the upcoming 11 | * [[scala.util.control.NonFatal]] (which appears in scala 2.10). 12 | */ 13 | def isNonFatal(t: Throwable): Boolean = t match { 14 | // StackOverflowError ok even though it is a VirtualMachineError 15 | case _: StackOverflowError => true 16 | // VirtualMachineError includes OutOfMemoryError and other fatal errors 17 | case _: VirtualMachineError | _: ThreadDeath | _: InterruptedException | 18 | _: LinkageError | _: ControlThrowable /*scala 2.10 | _: NotImplementedError*/ => false 19 | case _ => true 20 | } 21 | 22 | /** 23 | * Determines whether `t` is a fatal exception. 24 | * 25 | * @return true when `t` is '''not''' a fatal exception. 26 | */ 27 | def apply(t: Throwable): Boolean = t match { 28 | case _: NoSuchMethodException => false 29 | case t => isNonFatal(t) 30 | } 31 | 32 | /** 33 | * A deconstructor to be used in pattern matches, allowing use in exception 34 | * handlers. 35 | * 36 | * {{{ 37 | * try dangerousOperation() catch { 38 | * case NonFatal(e) => log.error("Chillax") 39 | * case e => log.error("Freak out") 40 | * } 41 | * }}} 42 | */ 43 | def unapply(t: Throwable): Option[Throwable] = if (apply(t)) Some(t) else None 44 | } 45 | -------------------------------------------------------------------------------- /util-events/src/test/scala/com/twitter/util/events/SizedSinkTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util.events 2 | 3 | import com.twitter.util.events.Event._ 4 | import com.twitter.util.Time 5 | import org.junit.runner.RunWith 6 | import org.scalatest.FunSuite 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class SizedSinkTest extends FunSuite { 11 | 12 | test("SizedSink#apply must be given a positive approxSize") { 13 | intercept[IllegalArgumentException] { 14 | SizedSink(0) 15 | } 16 | } 17 | 18 | test("constructor must be given power of 2 for capacity") { 19 | intercept[IllegalArgumentException] { 20 | new SizedSink(3, () => 1L) 21 | } 22 | } 23 | 24 | test("event and events") { 25 | val type1 = new Event.Type { } 26 | val type2 = new Event.Type { } 27 | val type3 = new Event.Type { } 28 | Time.withCurrentTimeFrozen { _ => 29 | def event(etype: Type) = 30 | Event(etype, Time.now, NoLong, NoObject, 1.0f) 31 | 32 | val sink = new SizedSink(2, () => Time.now.inMillis) 33 | assert(sink.events.size === 0) 34 | 35 | sink.event(type1, doubleVal = 1.0f) 36 | assert(sink.events.toSeq === Seq(event(type1))) 37 | 38 | sink.event(type2, doubleVal = 1.0f) 39 | assert(sink.events.toSeq === Seq(event(type1), event(type2))) 40 | 41 | // wrap around 42 | sink.event(type3, doubleVal = 1.0f) 43 | assert(sink.events.toSeq === Seq(event(type3), event(type2))) 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.finagle.stats 2 | 3 | /** 4 | * A RollupStatsReceiver reports stats on multiple Counter/Stat/Gauge based on the sequence of 5 | * names you pass. 6 | * e.g. 7 | * counter("errors", "clientErrors", "java_net_ConnectException").incr() 8 | * will actually increment those three counters: 9 | * - "/errors" 10 | * - "/errors/clientErrors" 11 | * - "/errors/clientErrors/java_net_ConnectException" 12 | */ 13 | class RollupStatsReceiver(val self: StatsReceiver) 14 | extends StatsReceiver with Proxy 15 | { 16 | val repr = self.repr 17 | 18 | private[this] def tails[A](s: Seq[A]): Seq[Seq[A]] = { 19 | s match { 20 | case s@Seq(_) => 21 | Seq(s) 22 | 23 | case Seq(hd, tl@_*) => 24 | Seq(Seq(hd)) ++ (tails(tl) map { t => Seq(hd) ++ t }) 25 | } 26 | } 27 | 28 | def counter(names: String*) = new Counter { 29 | private[this] val allCounters = BroadcastCounter( 30 | tails(names) map (self.counter(_: _*)) 31 | ) 32 | def incr(delta: Int) = allCounters.incr(delta) 33 | } 34 | 35 | def stat(names: String*) = new Stat { 36 | private[this] val allStats = BroadcastStat( 37 | tails(names) map (self.stat(_: _*)) 38 | ) 39 | def add(value: Float) = allStats.add(value) 40 | } 41 | 42 | def addGauge(names: String*)(f: => Float) = new Gauge { 43 | private[this] val underlying = tails(names) map { self.addGauge(_: _*)(f) } 44 | def remove() = underlying foreach { _.remove() } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /util-app/src/test/scala/com/twitter/app/AppTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.app 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | class TestApp(f: () => Unit) extends App { 8 | var reason: Option[String] = None 9 | protected override def exitOnError(reason: String) = { 10 | this.reason = Some(reason) 11 | } 12 | 13 | def main() = f() 14 | } 15 | 16 | @RunWith(classOf[JUnitRunner]) 17 | class AppTest extends FunSuite { 18 | test("App: propagate underlying exception from app") { 19 | val throwApp = new TestApp(() => throw new RuntimeException) 20 | 21 | intercept[RuntimeException] { 22 | throwApp.main(Array.empty) 23 | } 24 | } 25 | 26 | test("App: register on main call, last App wins") { 27 | val test1 = new TestApp(() => ()) 28 | val test2 = new TestApp(() => ()) 29 | 30 | assert(App.registered != Some(test1)) 31 | assert(App.registered != Some(test2)) 32 | 33 | test1.main(Array.empty) 34 | assert(App.registered === Some(test1)) 35 | 36 | test2.main(Array.empty) 37 | assert(App.registered === Some(test2)) 38 | } 39 | 40 | test("App: pass in bad args and expect usage") { 41 | val test1 = new TestApp(() => ()) 42 | 43 | test1.main(Array("-environment=staging", "-environment=staging")) 44 | val theReason: String = test1.reason.getOrElse { 45 | fail("There should have been a usage printed and was not") 46 | } 47 | 48 | assert(theReason.contains("""Error parsing flag "environment"""")) 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /util-zk-common/src/main/scala/com/twitter/conversions/common/quantity.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.conversions.common 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.twitter.common.quantity.{Amount, Time => CommonTime} 6 | import com.twitter.conversions.time._ 7 | import com.twitter.util.Duration 8 | 9 | object quantity { 10 | val COMMON_FOREVER: Duration = 0.millis 11 | 12 | class CommonDurationAdapter(d: Duration) { 13 | def toIntAmount = Amount.of(d.inMillis.toInt, CommonTime.MILLISECONDS) 14 | def toLongAmount = Amount.of(d.inMillis.toLong, CommonTime.MILLISECONDS) 15 | } 16 | 17 | /** Implicit conversion of Duration to CommonDuration */ 18 | implicit def commonDuration(d: Duration) = new CommonDurationAdapter(d) 19 | 20 | class DurationAmountAdapter(a: Amount[java.lang.Long, CommonTime]) { 21 | def toDuration: Duration = Duration(a.getValue.longValue, translateUnit(a.getUnit)) 22 | 23 | def translateUnit(unit: CommonTime) = unit match { 24 | case CommonTime.DAYS => TimeUnit.DAYS 25 | case CommonTime.HOURS => TimeUnit.HOURS 26 | case CommonTime.MINUTES => TimeUnit.MINUTES 27 | case CommonTime.MICROSECONDS => TimeUnit.MICROSECONDS 28 | case CommonTime.MILLISECONDS => TimeUnit.MILLISECONDS 29 | case CommonTime.NANOSECONDS => TimeUnit.NANOSECONDS 30 | case CommonTime.SECONDS => TimeUnit.SECONDS 31 | } 32 | } 33 | 34 | /** Implicit conversion of Amount to DurationAmountAdapter */ 35 | implicit def commonDuration(a: Amount[java.lang.Long, CommonTime]) = new DurationAmountAdapter(a) 36 | } 37 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/Cancellable.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import java.util.concurrent.atomic.AtomicBoolean 4 | 5 | /** 6 | * Defines a trait that makes the underlying object *cancellable*. 7 | * Cancellable objects may be linked to each other (one way) in order 8 | * to propagate cancellation. 9 | * 10 | * Note that the underlying object may or may not _respond_ to the 11 | * cancellation request. That is, calling 'cancel()' does not 12 | * guarantee the cancellation of the computation; rather it is a hint 13 | * that the provider of the computation may choose to ignore. 14 | */ 15 | 16 | trait Cancellable { 17 | def isCancelled: Boolean 18 | 19 | /** 20 | * Cancel the computation. The cancellation is propagated to linked 21 | * cancellable objects. 22 | */ 23 | def cancel() 24 | 25 | /** 26 | * Link this cancellable computation to 'other'. This means 27 | * cancellation of 'this' computation will propagate to 'other'. 28 | */ 29 | def linkTo(other: Cancellable): Unit 30 | } 31 | 32 | object Cancellable { 33 | val nil: Cancellable = new Cancellable { 34 | def isCancelled = false 35 | def cancel() {} 36 | def linkTo(other: Cancellable) {} 37 | } 38 | } 39 | 40 | class CancellableSink(f: => Unit) extends Cancellable { 41 | private[this] val wasCancelled = new AtomicBoolean(false) 42 | def isCancelled = wasCancelled.get 43 | def cancel() { if (wasCancelled.compareAndSet(false, true)) f } 44 | def linkTo(other: Cancellable) { 45 | throw new Exception("linking not supported in CancellableSink") 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /util-core/src/test/java/com/twitter/util/ActivityCompilationTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | import scala.Tuple2; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Collection; 9 | 10 | public class ActivityCompilationTest { 11 | 12 | private static final Activity.State ok = new Activity.Ok("ok"); 13 | 14 | @Test 15 | public void testActivityCreation() { 16 | Activity a = Activities.newPendingActivity(); 17 | Activity b = Activities.newFailedActivity(new Exception()); 18 | Activity c = Activities.newFutureActivity(Future.value(null)); 19 | Activity d = Activities.newValueActivity(new Object()); 20 | Tuple2, Witness>> e = Activities.newActivity(); 21 | Activity f = Activities.newActivity(Vars.newConstVar(ok)); 22 | 23 | Activity all = a.join(b).join(c).join(d).join(e._1()).join(f); 24 | Assert.assertTrue(all != null); 25 | } 26 | 27 | @Test 28 | public void testSample() { 29 | Activity a = Activities.newValueActivity("42"); 30 | Assert.assertEquals("42", Activities.sample(a)); 31 | } 32 | 33 | @Test 34 | public void testS() { 35 | ArrayList> list = new ArrayList>(); 36 | list.add(Activities.newValueActivity("42")); 37 | list.add(Activities.newValueActivity("24")); 38 | 39 | Activity> activities = Activities.collect(list); 40 | Assert.assertArrayEquals(new String[]{"42", "24"}, Activities.sample(activities).toArray()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/io/Files.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import java.io.{ByteArrayOutputStream, File, FileInputStream} 4 | 5 | /** 6 | * Utilities for working with `java.io.File`s 7 | */ 8 | object Files { 9 | /** 10 | * Read a file fully into a byte array. 11 | * 12 | * @param file file to read 13 | * @param limit number of bytes to read, default 4MB 14 | * @return array of bytes 15 | */ 16 | def readBytes(file: File, limit: Int = 1024 * 1024 * 4): Array[Byte]= { 17 | require(file.length() < limit, "File '%s' is too big".format(file.getAbsolutePath())) 18 | val buf = new ByteArrayOutputStream(math.min(limit, file.length().intValue())) 19 | val in = new FileInputStream(file) 20 | try { 21 | StreamIO.copy(in, buf) 22 | } finally { 23 | in.close() 24 | } 25 | buf.toByteArray() 26 | } 27 | 28 | /** 29 | * Deletes a given file or folder. 30 | * 31 | * Note since symlink detection in java is not reliable across platforms, 32 | * symlinks are in fact traversed and in the case of directories, what they 33 | * target will be deleted as well. Perhaps when we upgrade to jdk7 we can 34 | * use File.isSymbolicLink() here and make following symlinks optional. 35 | * 36 | * Returns whether or not the entire delete was successful 37 | */ 38 | def delete(file: File): Boolean = { 39 | if (!file.exists) { 40 | true 41 | } else if (file.isFile) { 42 | file.delete() 43 | } else { 44 | file.listFiles.foreach { f => 45 | delete(f) 46 | } 47 | file.delete() 48 | } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import java.util.concurrent.locks.ReentrantLock 4 | 5 | import org.junit.runner.RunWith 6 | import org.scalatest.FunSuite 7 | import org.scalatest.concurrent.Eventually 8 | import org.scalatest.junit.JUnitRunner 9 | import org.scalatest.time.{Millis, Seconds, Span} 10 | 11 | import com.twitter.util.{Await, Promise} 12 | 13 | class Philosopher { 14 | val ready = new Promise[Unit] 15 | private val lock = new ReentrantLock() 16 | def dine(neighbor: Philosopher): Unit = { 17 | lock.lockInterruptibly() 18 | ready.setValue(Unit) 19 | Await.ready(neighbor.ready) 20 | neighbor.dine(this) 21 | lock.unlock() 22 | } 23 | } 24 | 25 | @RunWith(classOf[JUnitRunner]) 26 | class ContentionTest extends FunSuite with Eventually { 27 | 28 | implicit override val patienceConfig = 29 | PatienceConfig(timeout = scaled(Span(2, Seconds)), interval = scaled(Span(5, Millis))) 30 | 31 | test("Deadlocks") { 32 | val c = new ContentionSnapshot() 33 | 34 | val descartes = new Philosopher() 35 | val plato = new Philosopher() 36 | 37 | val d = new Thread(new Runnable() { 38 | def run() { descartes.dine(plato) } 39 | }) 40 | d.start() 41 | 42 | val p = new Thread(new Runnable() { 43 | def run() { plato.dine(descartes) } 44 | }) 45 | p.start() 46 | Await.all(descartes.ready, plato.ready) 47 | 48 | eventually { assert(c.snap().deadlocks.size === 2) } 49 | d.interrupt() 50 | p.interrupt() 51 | p.join() 52 | d.join() 53 | assert(c.snap().deadlocks.size == 0) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.hashing 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.WordSpec 5 | import org.scalatest.junit.JUnitRunner 6 | 7 | @RunWith(classOf[JUnitRunner]) 8 | class DiagnosticsTest extends WordSpec { 9 | "Diagnostics" should { 10 | "print distribution" in { 11 | val hosts = 1 until 500 map { "10.1.1." + _ + ":11211:4" } 12 | 13 | val nodes = hosts.map { s => 14 | val Array(host, port, weight) = s.split(":") 15 | val identifier = host + ":" + port 16 | KetamaNode(identifier, weight.toInt, identifier) 17 | } 18 | 19 | val hashFunctions = List( 20 | "FNV1_32" -> KeyHasher.FNV1_32, 21 | "FNV1A_32" -> KeyHasher.FNV1A_32, 22 | "FNV1_64" -> KeyHasher.FNV1_64, 23 | "FNV1A_64" -> KeyHasher.FNV1A_64, 24 | "CRC32-ITU" -> KeyHasher.CRC32_ITU, 25 | "HSIEH" -> KeyHasher.HSIEH, 26 | "JENKINS" -> KeyHasher.JENKINS 27 | ) 28 | 29 | val keys = (1 until 1000000).map(_.toString).toList 30 | // Comment this out unless you're running it results 31 | // hashFunctions foreach { case (s, h) => 32 | // val distributor = new KetamaDistributor(nodes, 160) 33 | // val tester = new DistributionTester(distributor) 34 | // val start = Time.now 35 | // val dev = tester.distributionDeviation(keys map { s => h.hashKey(s.getBytes) }) 36 | // val duration = (Time.now - start).inMilliseconds 37 | // println("%s\n distribution: %.5f\n duration: %dms\n".format(s, dev, duration)) 38 | // } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /util-zk-common/src/main/scala/com/twitter/zk/CommonConnector.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | import scala.collection.JavaConverters._ 4 | 5 | import com.twitter.common.zookeeper.ZooKeeperClient 6 | import com.twitter.conversions.common.quantity._ 7 | import com.twitter.util.{Duration, FuturePool} 8 | 9 | /** 10 | * Adapts a common ZooKeeperClient for use as a Connector. 11 | * This connector blocks, but it doesn't seem worth a FuturePool only for this connector. 12 | */ 13 | class CommonConnector( 14 | val underlying: ZooKeeperClient, 15 | timeout: Duration = COMMON_FOREVER) 16 | (implicit pool: FuturePool) 17 | extends Connector { 18 | underlying.register(sessionBroker) 19 | 20 | def apply() = pool { underlying.get(timeout.toLongAmount) } 21 | def release() = pool { underlying.close() } 22 | } 23 | 24 | object CommonConnector { 25 | def apply(underlying: ZooKeeperClient, 26 | connectTimeout: Duration = COMMON_FOREVER) 27 | (implicit pool: FuturePool): CommonConnector = { 28 | new CommonConnector(underlying, connectTimeout)(pool) 29 | } 30 | 31 | def apply(addrs: Seq[java.net.InetSocketAddress], 32 | sessionTimeout: Duration) 33 | (implicit pool: FuturePool): CommonConnector = { 34 | apply(new ZooKeeperClient(sessionTimeout.toIntAmount, addrs.asJava))(pool) 35 | } 36 | 37 | def apply(addrs: Seq[java.net.InetSocketAddress], 38 | sessionTimeout: Duration, 39 | connectTimeout: Duration) 40 | (implicit pool: FuturePool): CommonConnector = { 41 | apply(new ZooKeeperClient(sessionTimeout.toIntAmount, addrs.asJava), connectTimeout)(pool) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /util-core/src/test/java/com/twitter/util/ClosableCompilationTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | import org.junit.Test; 4 | import scala.runtime.BoxedUnit; 5 | 6 | public class ClosableCompilationTest { 7 | 8 | private static class ImplementsClosable implements Closable { 9 | @Override 10 | public Future close(Time deadline) { 11 | return Future.Done(); 12 | } 13 | 14 | @Override 15 | public Future close(Duration after) { 16 | return Future.Done(); 17 | } 18 | 19 | @Override 20 | public Future close() { 21 | return Future.Done(); 22 | } 23 | } 24 | 25 | private static class ExtendsClosable extends AbstractClosable { 26 | @Override 27 | public Future close(Time deadline) { 28 | return Future.Done(); 29 | } 30 | } 31 | 32 | @Test 33 | public void testClose() { 34 | Closable a = new ImplementsClosable(); 35 | Closable b = new ExtendsClosable(); 36 | 37 | a.close(); 38 | b.close(); 39 | } 40 | 41 | @Test 42 | public void testMake() { 43 | Closable closable = Closables.newClosable( 44 | new Function>() { 45 | @Override 46 | public Future apply(Time time) { 47 | return Future.Done(); 48 | } 49 | } 50 | ); 51 | 52 | closable.close(); 53 | } 54 | 55 | @Test 56 | public void testAllAndSequence() { 57 | Closable a = new ImplementsClosable(); 58 | Closable b = new ImplementsClosable(); 59 | Closable c = new ImplementsClosable(); 60 | 61 | Closables.all(a, b, c); 62 | Closables.sequence(a, b, c); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | import java.util.concurrent.ThreadFactory 4 | import java.util.concurrent.atomic.AtomicInteger 5 | 6 | /** 7 | * A [[java.util.concurrent.ThreadFactory]] which creates threads with a name 8 | * indicating the pool from which they originated. 9 | * 10 | * A new [[java.lang.ThreadGroup]] (named `name`) is created as a sub-group of 11 | * whichever group to which the thread that created the factory belongs. Each 12 | * thread created by this factory will be a member of this group and have a 13 | * unique name including the group name and an monotonically increasing number. 14 | * The intention of this naming is to ease thread identification in debugging 15 | * output. 16 | * 17 | * For example, a `NamedPoolThreadFactory` with `name="writer"` will create a 18 | * `ThreadGroup` named "writer" and new threads will be named "writer-1", 19 | * "writer-2", etc. 20 | * 21 | * @param name the name of the new thread group 22 | * @param makeDaemons determines whether or not this factory will creates 23 | * daemon threads. 24 | */ 25 | class NamedPoolThreadFactory(name: String, makeDaemons: Boolean) extends ThreadFactory { 26 | def this(name: String) = this(name, false) 27 | 28 | val group = new ThreadGroup(Thread.currentThread().getThreadGroup(), name) 29 | val threadNumber = new AtomicInteger(1) 30 | 31 | def newThread(r: Runnable) = { 32 | val thread = new Thread(group, r, name + "-" + threadNumber.getAndIncrement()) 33 | thread.setDaemon(makeDaemons) 34 | if (thread.getPriority != Thread.NORM_PRIORITY) { 35 | thread.setPriority(Thread.NORM_PRIORITY) 36 | } 37 | thread 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/concurrent/ConcurrentMultiMap.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Twitter, Inc. 2 | 3 | package com.twitter.concurrent 4 | 5 | import java.util.concurrent.ConcurrentSkipListMap 6 | 7 | @deprecated("use guava's Multimaps.synchronizedMultimap", "6.2.x") 8 | class ConcurrentMultiMap[K <% Ordered[K], V <% Ordered[V]] { 9 | class Container(k: K, v: Option[V]) 10 | // TODO: extending tuples is deprecated and will be removed in the next version. 11 | // Remove this inheritance in the next major version 12 | extends Tuple2[K, Option[V]](k, v) 13 | with Comparable[Container] 14 | { 15 | def key = k 16 | def value = v 17 | 18 | def isDefined = value.isDefined 19 | 20 | def compareTo(that: Container) = this.key.compare(that.key) match { 21 | case 0 if ( this.isDefined && that.isDefined) => this.value.get.compare(that.value.get) 22 | case 0 if (!this.isDefined && !that.isDefined) => 0 23 | case 0 if (!this.isDefined) => -1 24 | case 0 if (!that.isDefined) => 1 25 | 26 | case x => x 27 | } 28 | } 29 | 30 | val underlying = new ConcurrentSkipListMap[Container, Unit] 31 | 32 | def +=(kv:(K, V)) { 33 | val (k, v) = kv 34 | underlying.putIfAbsent(new Container(k, Some(v)), ()) 35 | } 36 | 37 | def get(k:K):List[V] = { 38 | def traverse(entry: Container): List[V] = { 39 | val nextEntry = underlying.higherKey(entry) 40 | if (nextEntry == null || nextEntry.key != k) { 41 | Nil 42 | } else { 43 | assert(nextEntry.value.isDefined) 44 | nextEntry.value.get :: traverse(nextEntry) 45 | } 46 | } 47 | 48 | traverse(new Container(k, None)) 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /util-core/src/main/java/com/twitter/util/Vars.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | import java.util.List; 6 | 7 | /** 8 | * A Java adaptation of {@link com.twitter.util.Var} companion object. 9 | */ 10 | public final class Vars { 11 | private Vars() { } 12 | 13 | /** 14 | * @see com.twitter.util.Var$#sample(Var) 15 | */ 16 | public static T sample(Var var) { 17 | return Var$.MODULE$.sample(var); 18 | } 19 | 20 | /** 21 | * @see com.twitter.util.Var$#apply(Object) 22 | */ 23 | public static ReadWriteVar newVar(T init) { 24 | return new ReadWriteVar(init); 25 | } 26 | 27 | /** 28 | * @see com.twitter.util.Var$#apply(Object, Event) 29 | */ 30 | public static Var newVar(T init, Event event) { 31 | return Var$.MODULE$.apply(init, event); 32 | } 33 | 34 | /** 35 | * @see com.twitter.util.Var$#value(Object) 36 | */ 37 | public static Var newConstVar(T constant) { 38 | return Var$.MODULE$.value(constant); 39 | } 40 | 41 | /** 42 | * @see com.twitter.util.Var$#collect(java.util.List) 43 | */ 44 | public static Var> collect(Collection> vars) { 45 | List> in = new ArrayList>(vars); 46 | Var> out = Var$.MODULE$.collect(in); 47 | return out.map(new Function, Collection>() { 48 | @Override 49 | public Collection apply(List list) { 50 | return list; 51 | } 52 | }); 53 | } 54 | 55 | /** 56 | * @see com.twitter.util.Var$#async(Object, scala.Function1) 57 | */ 58 | public static Var async(T empty, Function, Closable> function) { 59 | return Var$.MODULE$.async(empty, function); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/conversions/storage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter 18 | package conversions 19 | 20 | import com.twitter.util.StorageUnit 21 | 22 | object storage { 23 | class RichWholeNumber(wrapped: Long) { 24 | def byte = bytes 25 | def bytes = new StorageUnit(wrapped) 26 | def kilobyte = kilobytes 27 | def kilobytes = new StorageUnit(wrapped * 1024) 28 | def megabyte = megabytes 29 | def megabytes = new StorageUnit(wrapped * 1024 * 1024) 30 | def gigabyte = gigabytes 31 | def gigabytes = new StorageUnit(wrapped * 1024 * 1024 * 1024) 32 | def terabyte = terabytes 33 | def terabytes = new StorageUnit(wrapped * 1024 * 1024 * 1024 * 1024) 34 | def petabyte = petabytes 35 | def petabytes = new StorageUnit(wrapped * 1024 * 1024 * 1024 * 1024 * 1024) 36 | 37 | def thousand = wrapped * 1000 38 | def million = wrapped * 1000 * 1000 39 | def billion = wrapped * 1000 * 1000 * 1000 40 | } 41 | 42 | implicit def intToStorageUnitableWholeNumber(i: Int) = new RichWholeNumber(i) 43 | implicit def longToStorageUnitableWholeNumber(l: Long) = new RichWholeNumber(l) 44 | } 45 | -------------------------------------------------------------------------------- /util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.hashing 2 | 3 | import scala.collection.mutable.ListBuffer 4 | 5 | import org.apache.commons.codec.binary.Base64 6 | import org.junit.runner.RunWith 7 | import org.scalatest.WordSpec 8 | import org.scalatest.junit.JUnitRunner 9 | 10 | import com.twitter.io.TempFile 11 | 12 | @RunWith(classOf[JUnitRunner]) 13 | class KeyHasherTest extends WordSpec { 14 | def readResource(name: String) = { 15 | var lines = new ListBuffer[String]() 16 | val src = scala.io.Source.fromFile(TempFile.fromResourcePath(getClass, "/"+name)) 17 | src.getLines 18 | } 19 | 20 | val base64 = new Base64() 21 | def decode(str: String) = base64.decode(str) 22 | 23 | def testHasher(name: String, hasher: KeyHasher) = { 24 | val sources = readResource(name + "_source") map { decode(_) } 25 | val hashes = readResource(name + "_hashes") 26 | assert(sources.size > 0) 27 | 28 | sources zip hashes foreach { case (source, hashAsString) => 29 | val hash = BigInt(hashAsString).toLong 30 | assert(hasher.hashKey(source) === hash) 31 | } 32 | } 33 | 34 | "KeyHasher" should { 35 | "correctly hash fnv1_32" in { 36 | testHasher("fnv1_32", KeyHasher.FNV1_32) 37 | } 38 | 39 | "correctly hash fnv1_64" in { 40 | testHasher("fnv1_64", KeyHasher.FNV1_64) 41 | } 42 | 43 | "correctly hash fnv1a_32" in { 44 | testHasher("fnv1a_32", KeyHasher.FNV1A_32) 45 | } 46 | 47 | "correctly hash fnv1a_64" in { 48 | testHasher("fnv1a_64", KeyHasher.FNV1A_64) 49 | } 50 | 51 | "correctly hash jenkins" in { 52 | testHasher("jenkins", KeyHasher.JENKINS) 53 | } 54 | 55 | "correctly hash crc32 itu" in { 56 | testHasher("crc32", KeyHasher.CRC32_ITU) 57 | } 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/TempFolder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.util 18 | 19 | import java.io.File 20 | 21 | import com.twitter.io.Files 22 | 23 | /** 24 | * Test mixin that creates a new folder for a block of code to execute in. 25 | * The folder is recursively deleted after the test. 26 | * 27 | * Note, the [[com.twitter.util.io]] package would be a better home for this trait. 28 | */ 29 | trait TempFolder { 30 | private val _folderName = new ThreadLocal[File] 31 | 32 | def withTempFolder(f: => Any) { 33 | val tempFolder = System.getProperty("java.io.tmpdir") 34 | // Note: If we were willing to have a dependency on Guava in util-core 35 | // we could just use `com.google.common.io.Files.createTempDir()` 36 | var folder: File = null 37 | do { 38 | folder = new File(tempFolder, "scala-test-" + System.currentTimeMillis) 39 | } while (! folder.mkdir()) 40 | _folderName.set(folder) 41 | 42 | try { 43 | f 44 | } finally { 45 | Files.delete(folder) 46 | } 47 | } 48 | 49 | def folderName = { _folderName.get.getPath } 50 | 51 | def canonicalFolderName = { _folderName.get.getCanonicalPath } 52 | } 53 | -------------------------------------------------------------------------------- /util-core/src/main/java/com/twitter/util/Closables.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | import scala.collection.JavaConversions; 4 | import scala.runtime.BoxedUnit; 5 | 6 | import java.util.Arrays; 7 | import java.util.concurrent.atomic.AtomicReference; 8 | 9 | /** 10 | * Java compatibility layer for {@link com.twitter.util.Closable}. 11 | */ 12 | public final class Closables { 13 | 14 | /** 15 | * @see com.twitter.util.Closable$#nop() 16 | */ 17 | public static final Closable NOP = Closable$.MODULE$.nop(); 18 | 19 | private Closables() { } 20 | 21 | /** 22 | * @see com.twitter.util.Closable$#all(scala.collection.Seq) 23 | */ 24 | public static Closable all(Closable... closables) { 25 | return Closable$.MODULE$.all( 26 | JavaConversions.asScalaBuffer(Arrays.asList(closables)) 27 | ); 28 | } 29 | 30 | /** 31 | * @see com.twitter.util.Closable$#sequence(scala.collection.Seq) 32 | */ 33 | public static Closable sequence(Closable... closables) { 34 | return Closable$.MODULE$.sequence( 35 | JavaConversions.asScalaBuffer(Arrays.asList(closables)) 36 | ); 37 | } 38 | 39 | /** 40 | * @see com.twitter.util.Closable$#make(scala.Function1) 41 | */ 42 | public static Closable newClosable(Function> function) { 43 | return Closable$.MODULE$.make(function); 44 | } 45 | 46 | /** 47 | * @see com.twitter.util.Closable$#ref(java.util.concurrent.atomic.AtomicReference) 48 | */ 49 | public static Closable newClosable(AtomicReference reference) { 50 | return Closable$.MODULE$.ref(reference); 51 | } 52 | 53 | /** 54 | * @see com.twitter.util.Closable$#closeOnCollect(Closable, Object) 55 | */ 56 | public static void closeOnCollect(Closable closable, Object object) { 57 | Closable$.MODULE$.closeOnCollect(closable, object); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/concurrent/ConcurrentPool.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Twitter, Inc. 2 | // 3 | // Concurrent object pool. 4 | 5 | package com.twitter.concurrent 6 | 7 | import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue} 8 | 9 | /** 10 | * The ConcurrentPool provides a concurrent object pool on top of the 11 | * java.util.concurrent primitives. 12 | * 13 | * The pool currently supports only FIFO ordering of items, and does 14 | * not yet clean up per-key object lists. 15 | */ 16 | @deprecated("use finagle's BufferingPool", "6.2.x") 17 | class ConcurrentPool[K, V] { 18 | type Queue = ConcurrentLinkedQueue[V] 19 | type Map = ConcurrentHashMap[K, Queue] 20 | 21 | val map = new Map 22 | val deathQueue = new ConcurrentLinkedQueue[Tuple2[K, Queue]] 23 | 24 | def doPut(k: K, v: V) { 25 | var objs = map.get(k) 26 | if (objs eq null) { 27 | map.putIfAbsent(k, new ConcurrentLinkedQueue[V]) 28 | objs = map.get(k) 29 | } 30 | 31 | assert(objs ne null) 32 | objs.offer(v) 33 | } 34 | 35 | def put(k: K, v: V) { 36 | doPut(k, v) 37 | 38 | // Queue repair. TODO: amortize these more? 39 | var item = deathQueue.poll() 40 | while (item != null) { 41 | val (key, queue) = item 42 | if (!queue.isEmpty) 43 | queue.toArray().asInstanceOf[Array[V]].foreach(doPut(k, _)) 44 | 45 | item = deathQueue.poll() 46 | } 47 | } 48 | 49 | def get(k: K): Option[V] = { 50 | val objs = map.get(k) 51 | if (objs eq null) { 52 | None 53 | } else { 54 | val obj = objs.poll() 55 | 56 | if (objs.isEmpty) { 57 | val deadMap = map.remove(k) 58 | if (deadMap ne null) 59 | deathQueue.offer((k, deadMap)) 60 | } 61 | 62 | if (obj == null) 63 | None 64 | else 65 | Some(obj) 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/Codec.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | trait EncoderCompanion { 4 | 5 | type Enc[T, S] <: Encoder[T, S] 6 | 7 | /** 8 | * Encode a value of type T into a serialized value of type S based on the implicit environment. 9 | */ 10 | def encode[T, S](t: T)(implicit enc: Enc[T, S]): S = enc.encode(t) 11 | } 12 | 13 | object Encoder extends EncoderCompanion { 14 | type Enc[T, S] = Encoder[T, S] 15 | } 16 | 17 | /** 18 | * A base trait for encoders of type T into a serialized form S 19 | */ 20 | trait Encoder[T, S] extends (T => S) { 21 | /* 22 | * Encode a T into an S 23 | */ 24 | def encode(t: T): S 25 | 26 | def apply(t: T) = encode(t) 27 | } 28 | 29 | trait DecoderCompanion { 30 | 31 | type Dec[T, S] <: Decoder[T, S] 32 | 33 | /** 34 | * Decode a value of type T from a serialized value of type S based on the implicit environment. 35 | */ 36 | def decode[T, S](s: S)(implicit dec: Dec[T, S]): T = dec.decode(s) 37 | } 38 | 39 | object Decoder extends DecoderCompanion { 40 | type Dec[T, S] = Decoder[T, S] 41 | } 42 | 43 | /** 44 | * A base trait for decoders for type T from a serialized form S 45 | */ 46 | trait Decoder[T, S] { 47 | /* 48 | * Decode a T from an S 49 | */ 50 | def decode(s: S): T 51 | 52 | def invert(s: S) = decode(s) 53 | } 54 | 55 | object Codec extends EncoderCompanion with DecoderCompanion { 56 | type Enc[T, S] = Codec[T, S] 57 | type Dec[T, S] = Codec[T, S] 58 | } 59 | 60 | /** 61 | * A base trait for all Codecs that translate a type T into a serialized form S 62 | */ 63 | trait Codec[T, S] extends Bijection[T, S] with Encoder[T, S] with Decoder[T, S] 64 | 65 | object BinaryCodec { 66 | def encode[T](t: T)(implicit enc: Codec[T, Array[Byte]]): Array[Byte] = enc.encode(t) 67 | def decode[T](a: Array[Byte])(implicit dec: Codec[T, Array[Byte]]): T = dec.decode(a) 68 | } 69 | 70 | -------------------------------------------------------------------------------- /util-collection/src/main/scala/com/twitter/util/LruMap.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import java.{util => ju} 4 | 5 | import scala.collection.JavaConverters._ 6 | import scala.collection.mutable.{Map, MapLike, SynchronizedMap} 7 | 8 | import org.apache.commons.collections.map.LRUMap 9 | 10 | trait JMapWrapperLike[A, B, +Repr <: MapLike[A, B, Repr] with Map[A, B]] extends Map[A, B] with MapLike[A, B, Repr] { 11 | def underlying: ju.Map[A, B] 12 | 13 | override def size = underlying.size 14 | 15 | override def get(k: A) = underlying.asScala.get(k) 16 | 17 | override def +=(kv: (A, B)): this.type = { underlying.put(kv._1, kv._2); this } 18 | override def -=(key: A): this.type = { underlying remove key; this } 19 | 20 | override def put(k: A, v: B): Option[B] = underlying.asScala.put(k, v) 21 | 22 | override def update(k: A, v: B) { underlying.put(k, v) } 23 | 24 | override def remove(k: A): Option[B] = underlying.asScala.remove(k) 25 | 26 | override def clear() = underlying.clear() 27 | 28 | override def empty: Repr = null.asInstanceOf[Repr] 29 | 30 | override def iterator = underlying.asScala.iterator 31 | } 32 | 33 | case class JMapWrapper[A, B](underlying : ju.Map[A, B]) extends JMapWrapperLike[A, B, JMapWrapper[A, B]] { 34 | override def empty = JMapWrapper(new ju.HashMap[A, B]) 35 | } 36 | 37 | object LruMap { 38 | def makeUnderlying[K, V](maxSize: Int) = new LRUMap(maxSize).asInstanceOf[ju.Map[K, V]] 39 | } 40 | 41 | class LruMap[K, V](val maxSize: Int, underlying: ju.Map[K, V]) 42 | extends JMapWrapper[K, V](underlying) 43 | { 44 | def this(maxSize: Int) = this(maxSize, LruMap.makeUnderlying(maxSize)) 45 | } 46 | 47 | class SynchronizedLruMap[K, V](maxSize: Int, underlying: ju.Map[K, V]) 48 | extends LruMap[K, V](maxSize, ju.Collections.synchronizedMap(underlying)) 49 | with SynchronizedMap[K, V] 50 | { 51 | def this(maxSize: Int) = this(maxSize, LruMap.makeUnderlying(maxSize)) 52 | } 53 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/CredentialsTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2011 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.util 18 | 19 | 20 | import org.junit.runner.RunWith 21 | import org.scalatest.WordSpec 22 | import org.scalatest.junit.JUnitRunner 23 | 24 | @RunWith(classOf[JUnitRunner]) 25 | class CredentialsTest extends WordSpec { 26 | "Credentials" should { 27 | "parse a simple auth file" in { 28 | val content = "username: root\npassword: hellokitty\n" 29 | assert(Credentials(content) === Map("username" -> "root", "password" -> "hellokitty")) 30 | } 31 | 32 | "parse a more complex auth file" in { 33 | val content = """# random comment 34 | 35 | username:root 36 | password : last_0f-the/international:playboys 37 | # more stuff 38 | moar :ok 39 | 40 | """ 41 | assert(Credentials(content) === Map( 42 | "username" -> "root", 43 | "password" -> "last_0f-the/international:playboys", 44 | "moar" -> "ok" 45 | )) 46 | } 47 | 48 | "work for java peeps too" in { 49 | val content = "username: root\npassword: hellokitty\n" 50 | val jmap = new Credentials().read(content) 51 | assert(jmap.size() === 2) 52 | assert(jmap.get("username") === "root") 53 | assert(jmap.get("password") === "hellokitty") 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | import org.junit.runner.RunWith 4 | import org.mockito.Mockito._ 5 | import org.scalatest.WordSpec 6 | import org.scalatest.junit.JUnitRunner 7 | import org.scalatest.mock.MockitoSugar 8 | 9 | import com.twitter.util.Future 10 | 11 | @RunWith(classOf[JUnitRunner]) 12 | class ConnectorTest extends WordSpec with MockitoSugar { 13 | "Connector.RoundRobin" should { 14 | "require underlying connections" in { 15 | intercept[Exception] { 16 | Connector.RoundRobin() 17 | } 18 | } 19 | 20 | "dispatch requests across underlying connectors" should { 21 | class ConnectorSpecHelper { 22 | def mockConnector = { 23 | val connector = mock[Connector] 24 | when(connector.apply()).thenReturn(Future.never) 25 | when(connector.release()).thenReturn(Future.never) 26 | connector 27 | } 28 | val nConnectors = 3 29 | val connectors = 1 to nConnectors map { _ => mockConnector } 30 | val connector = Connector.RoundRobin(connectors: _*) 31 | } 32 | 33 | "apply" in { 34 | val h = new ConnectorSpecHelper 35 | import h._ 36 | 37 | connectors foreach { x => 38 | assert(x.apply() === Future.never) 39 | } 40 | (1 to 2 * nConnectors) foreach { _ => 41 | connector() 42 | } 43 | connectors foreach { c => 44 | verify(c, times(3)).apply() 45 | } 46 | } 47 | 48 | "release" in { 49 | val h = new ConnectorSpecHelper 50 | import h._ 51 | 52 | connectors foreach { x => 53 | assert(x.release() === Future.never) 54 | } 55 | (1 to 2) foreach { _ => 56 | connector.release() 57 | } 58 | connectors foreach { c => 59 | verify(c, times(3)).release() 60 | } 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /util-core/src/test/java/com/twitter/util/AwaitableCompilationTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.util; 2 | 3 | import junit.framework.Assert; 4 | import org.junit.Test; 5 | import scala.runtime.BoxedUnit; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | import java.util.Collection; 10 | 11 | public class AwaitableCompilationTest { 12 | 13 | public static class MockAwaitable implements Awaitable { 14 | @Override 15 | public Awaitable ready(Duration timeout, CanAwait permit) throws TimeoutException { 16 | return this; 17 | } 18 | 19 | @Override 20 | public String result(Duration timeout, CanAwait permit) throws Exception { 21 | return "42"; 22 | } 23 | 24 | @Override 25 | public boolean isReady(CanAwait permit) { 26 | return true; 27 | } 28 | } 29 | 30 | public static class MockCloseAwaitably extends AbstractCloseAwaitably { 31 | @Override 32 | public Future onClose(Time deadline) { 33 | return Future.Done(); 34 | } 35 | 36 | public int fortyTwo() { 37 | return 42; 38 | } 39 | } 40 | 41 | @Test 42 | public void testAwaitable() throws Exception { 43 | Awaitable awaitable = new MockAwaitable(); 44 | Assert.assertEquals(awaitable, Await.ready(awaitable)); 45 | Assert.assertEquals("42", Await.result(awaitable)); 46 | } 47 | 48 | @Test 49 | public void testCloseAwaitable() throws Exception { 50 | MockCloseAwaitably m = new MockCloseAwaitably(); 51 | m.close(); 52 | Await.ready(m); 53 | Assert.assertEquals(42, m.fortyTwo()); 54 | } 55 | 56 | @Test 57 | public void testAll() throws TimeoutException, InterruptedException { 58 | MockAwaitable a = new MockAwaitable(); 59 | MockAwaitable b = new MockAwaitable(); 60 | MockAwaitable c = new MockAwaitable(); 61 | 62 | Await.all(a, b, c); 63 | 64 | Collection> all = new ArrayList>(Arrays.asList(a, b, c)); 65 | Await.all(all, Duration.Top()); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/concurrent/ConcurrentBijectionTest.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Twitter, Inc. 2 | 3 | package com.twitter.concurrent 4 | 5 | import org.junit.runner.RunWith 6 | import org.scalatest.WordSpec 7 | import org.scalatest.junit.JUnitRunner 8 | 9 | @RunWith(classOf[JUnitRunner]) 10 | class ConcurrentBijectionTest extends WordSpec { 11 | "maintain a bijective map" in { 12 | val b = new ConcurrentBijection[Int, Int] 13 | b ++= (1 -> 1) :: (2 -> 3) :: (100 -> 2) :: Nil 14 | 15 | assert(b.get(1) === Some(1)) 16 | assert(b.getReverse(1) === Some(1)) 17 | 18 | assert(b.get(2) === Some(3)) 19 | assert(b.getReverse(3) === Some(2)) 20 | 21 | assert(b.get(100) === Some(2)) 22 | assert(b.getReverse(2) === Some(100)) 23 | } 24 | 25 | "maintain the bijective property" in { 26 | val b = new ConcurrentBijection[Int, Int] 27 | 28 | b += (1 -> 2) 29 | 30 | assert(b.get(1) === Some(2)) 31 | assert(b.getReverse(2) === Some(1)) 32 | 33 | // Introduce a new forward mapping. This should delete the old 34 | // one. 35 | b += (1 -> 3) 36 | assert(b.getReverse(2) === None) 37 | assert(b.get(1) === Some(3)) 38 | assert(b.getReverse(3) === Some(1)) 39 | 40 | // Now, introduce a new reverse mapping for 3, which should kill 41 | // the existing 1 -> 3 mapping. 42 | b += (100 -> 3) 43 | assert(b.getReverse(3) === Some(100)) 44 | assert(b.get(1) === None) // the old forward mapping was killed. 45 | assert(b.get(100) === Some(3)) 46 | 47 | } 48 | 49 | "delete mappings" in { 50 | val b = new ConcurrentBijection[Int, Int] 51 | 52 | b += (1 -> 2) 53 | 54 | assert(b.isEmpty === false) 55 | b -= 1 56 | assert(b.isEmpty === true) 57 | } 58 | 59 | "iterate over mappings" in { 60 | val b = new ConcurrentBijection[Int, Int] 61 | 62 | for (i <- 0 until 100) 63 | b += (i -> i) 64 | 65 | assert(b.toSet === (for (i <- 0 until 100) yield (i, i)).toSet) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | import org.junit.runner.RunWith 4 | import org.scalatest.FunSuite 5 | import org.scalatest.concurrent.{Eventually, IntegrationPatience} 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | import com.twitter.util.Promise 9 | 10 | 11 | abstract class LocalSchedulerTest(lifo: Boolean) extends FunSuite { 12 | private val scheduler = new LocalScheduler(lifo) 13 | def submit(f: => Unit) = scheduler.submit(new Runnable { 14 | def run() = f 15 | }) 16 | 17 | val N = 100 18 | 19 | test("run the first submitter immediately") { 20 | var ok = false 21 | submit { 22 | ok = true 23 | } 24 | assert(ok) 25 | } 26 | 27 | test("run subsequent submits serially") { 28 | var n = 0 29 | submit { 30 | assert(n === 0) 31 | submit { 32 | assert(n === 1) 33 | submit { 34 | assert(n === 2) 35 | n += 1 36 | } 37 | n += 1 38 | } 39 | n += 1 40 | } 41 | 42 | assert(n === 3) 43 | } 44 | 45 | test("handle many submits") { 46 | var ran = Nil: List[Int] 47 | submit { 48 | for (which <- 0 until N) 49 | submit { 50 | ran ::= which 51 | } 52 | } 53 | if (lifo) 54 | assert(ran === (0 until N)) 55 | else 56 | assert(ran === (0 until N).reverse) 57 | } 58 | } 59 | 60 | @RunWith(classOf[JUnitRunner]) 61 | class LocalSchedulerFifoTest extends LocalSchedulerTest(false) 62 | 63 | @RunWith(classOf[JUnitRunner]) 64 | class LocalSchedulerLifoTest extends LocalSchedulerTest(true) 65 | 66 | @RunWith(classOf[JUnitRunner]) 67 | class ThreadPoolSchedulerTest extends FunSuite with Eventually with IntegrationPatience { 68 | test("works") { 69 | val p = new Promise[Unit] 70 | val scheduler = new ThreadPoolScheduler("test") 71 | scheduler.submit(new Runnable { 72 | def run() { p.setDone() } 73 | }) 74 | 75 | eventually { p.isDone } 76 | 77 | scheduler.shutdown() 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to get patches from you! 4 | 5 | ## Workflow 6 | 7 | The workflow that we support: 8 | 9 | 1. Fork util 10 | 2. Make a feature branch 11 | 3. Make your cool new feature or bugfix on your branch 12 | 4. Write a test for your change 13 | 5. From your branch, make a pull request against twitter/util/master 14 | 6. Work with repo maintainers to get merged in 15 | 7. Wait for your change to be pulled into twitter/util/master 16 | 8. Merge twitter/util/master into your origin master 17 | 9. Delete your feature branch 18 | 19 | ## Testing 20 | 21 | We've standardized on using the ScalaTest testing framework. A lot of our older 22 | tests are still on Specs, which we are slowly converting to ScalaTest. If you 23 | are looking for an easy way to start contributing, finding a few Specs tests to 24 | move over to ScalaTest is a great way to get started! 25 | 26 | Because ScalaTest has such a big surface area, we use a restricted subset of it 27 | in our tests to keep them easy to read. We use the "assert" api, and not 28 | the "matchers" one, and we use the FunSuite mixin, which supports xUnit-like 29 | semantics. 30 | 31 | ## Style 32 | 33 | We generally follow [Effective Scala][0], and the [Scala Style Guide][1]. When 34 | in doubt, look around the codebase and see how it's done elsewhere. 35 | 36 | Comments should be formatted to a width no greater than 80 columns. 37 | 38 | Files should be exempt of trailing spaces. 39 | 40 | We adhere to a specific format for commit messages. Please write your commit 41 | messages along these guidelines: 42 | 43 | One line description of your change (less than 72 characters) 44 | 45 | Problem 46 | Explain here the context, and why you're making that change. 47 | What is the problem you're trying to solve? 48 | 49 | Solution 50 | Describe the modifications you've done. 51 | 52 | Result 53 | After your change, what will change? 54 | 55 | [0]: http://twitter.github.io/effectivescala/ 56 | [1]: http://docs.scala-lang.org/style/scaladoc.html 57 | -------------------------------------------------------------------------------- /util-hashing/src/main/scala/com/twitter/hashing/KeyHasher.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.hashing 2 | 3 | /** 4 | * Hashes a key into a 32-bit or 64-bit number (depending on the algorithm). 5 | */ 6 | @deprecated("Prefer Hashable[Array[Byte],Long]", "6.12.1") 7 | trait KeyHasher { 8 | def hashKey(key: Array[Byte]): Long 9 | } 10 | 11 | /** 12 | * Commonly used key hashing algorithms. 13 | */ 14 | object KeyHasher { 15 | def fromHashableInt(hashable: Hashable[Array[Byte],Int]): KeyHasher = new KeyHasher { 16 | def hashKey(key: Array[Byte]) = hashable(key).toLong 17 | override def toString = hashable.toString 18 | } 19 | def fromHashableLong(hashable: Hashable[Array[Byte],Long]): KeyHasher = new KeyHasher { 20 | def hashKey(key: Array[Byte]) = hashable(key) 21 | override def toString = hashable.toString 22 | } 23 | 24 | val FNV1_32 = fromHashableInt(Hashable.FNV1_32) 25 | val FNV1A_32 = fromHashableInt(Hashable.FNV1A_32) 26 | val FNV1_64 = fromHashableLong(Hashable.FNV1_64) 27 | val FNV1A_64 = fromHashableLong(Hashable.FNV1A_64) 28 | /** 29 | * Ketama's default hash algorithm: the first 4 bytes of the MD5 as a little-endian int. 30 | * Wow, really? Who thought that was a good way to do it? :( 31 | */ 32 | val KETAMA = fromHashableInt(Hashable.MD5_LEInt) 33 | val CRC32_ITU = fromHashableInt(Hashable.CRC32_ITU) 34 | val HSIEH = fromHashableInt(Hashable.HSIEH) 35 | val JENKINS = fromHashableLong(Hashable.JENKINS) 36 | 37 | /** 38 | * Return one of the key hashing algorithms by name. This is used to configure a memcache 39 | * client from a config file. 40 | */ 41 | def byName(name: String): KeyHasher = { 42 | name match { 43 | case "fnv" => FNV1_32 44 | case "fnv1" => FNV1_32 45 | case "fnv1-32" => FNV1_32 46 | case "fnv1a-32" => FNV1A_32 47 | case "fnv1-64" => FNV1_64 48 | case "fnv1a-64" => FNV1A_64 49 | case "ketama" => KETAMA 50 | case "crc32-itu" => CRC32_ITU 51 | case "hsieh" => HSIEH 52 | case _ => throw new NoSuchElementException(name) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /util-core/src/main/java/com/twitter/concurrent/Offers.java: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent; 2 | 3 | import com.twitter.util.Duration; 4 | import com.twitter.util.Function0; 5 | import com.twitter.util.Future; 6 | import com.twitter.util.Timer; 7 | import scala.collection.JavaConversions; 8 | import scala.runtime.BoxedUnit; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Collection; 12 | 13 | /** 14 | * A Java adaptation of {@link com.twitter.concurrent.Offer} companion object. 15 | */ 16 | public final class Offers { 17 | private Offers() { } 18 | 19 | /** 20 | * @see Offer$#never() 21 | */ 22 | public static final Offer NEVER = Offer$.MODULE$.never(); 23 | 24 | /** 25 | * @see Offer$#const(scala.Function0) 26 | * 27 | * Note: Updates here must also be done at {@link com.twitter.concurrent.Offer$#const(scala.Function0)} 28 | */ 29 | public static Offer newConstOffer(final Function0 function) { 30 | return new AbstractOffer() { 31 | @Override 32 | public Future> prepare() { 33 | return Future.value(Txs.newConstTx(function.apply())); 34 | } 35 | }; 36 | } 37 | 38 | /** 39 | * Creates a new {@link Offer} from given constant {@code value}. 40 | * 41 | * Note: Updates here must also be done at {@link com.twitter.concurrent.Offer$#const(scala.Function0)} 42 | */ 43 | public static Offer newConstOffer(final T value) { 44 | return Offers.newConstOffer(new Function0() { 45 | @Override 46 | public T apply() { 47 | return value; 48 | } 49 | }); 50 | } 51 | 52 | /** 53 | * @see Offer$#timeout(com.twitter.util.Duration, com.twitter.util.Timer) 54 | */ 55 | public static Offer newTimeoutOffer(Duration duration, Timer timer) { 56 | return Offer$.MODULE$.timeout(duration, timer); 57 | } 58 | 59 | /** 60 | * @see Offer$#choose(scala.collection.Seq) 61 | */ 62 | public static Offer choose(Collection> offers) { 63 | return Offer$.MODULE$.choose(JavaConversions.asScalaBuffer(new ArrayList>(offers))); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.finagle.stats 2 | 3 | import java.io.PrintStream 4 | import scala.collection.mutable 5 | 6 | /** In-memory stats receiver for testing. */ 7 | class InMemoryStatsReceiver extends StatsReceiver { 8 | val repr = this 9 | 10 | val counters = new mutable.HashMap[Seq[String], Int] 11 | with mutable.SynchronizedMap[Seq[String], Int] 12 | val stats = new mutable.HashMap[Seq[String], Seq[Float]] 13 | with mutable.SynchronizedMap[Seq[String], Seq[Float]] 14 | val gauges = new mutable.WeakHashMap[Seq[String], () => Float] 15 | with mutable.SynchronizedMap[Seq[String], () => Float] 16 | 17 | def counter(name: String*): ReadableCounter = 18 | new ReadableCounter { 19 | 20 | def incr(delta: Int): Unit = synchronized { 21 | val oldValue = apply 22 | counters(name) = oldValue + delta 23 | } 24 | def apply(): Int = counters.get(name).getOrElse(0) 25 | } 26 | 27 | def stat(name: String*): ReadableStat = 28 | new ReadableStat { 29 | def add(value: Float) = synchronized { 30 | val oldValue = apply 31 | stats(name) = oldValue :+ value 32 | } 33 | def apply(): Seq[Float] = stats.get(name).getOrElse(Seq.empty) 34 | } 35 | 36 | def addGauge(name: String*)(f: => Float): Gauge = { 37 | val gauge = new Gauge { 38 | def remove() { 39 | gauges -= name 40 | } 41 | } 42 | gauges += name -> (() => f) 43 | gauge 44 | } 45 | 46 | def print(p: PrintStream) { 47 | for ((k, v) <- counters) 48 | p.printf("%s %d\n", k mkString "/", v: java.lang.Integer) 49 | for ((k, g) <- gauges) 50 | p.printf("%s %f\n", k mkString "/", g(): java.lang.Float) 51 | for ((k, s) <- stats if s.size > 0) 52 | p.printf("%s %f\n", k mkString "/", (s.sum / s.size): java.lang.Float) 53 | } 54 | } 55 | 56 | trait ReadableCounter extends Counter { 57 | def apply(): Int 58 | } 59 | 60 | trait ReadableStat extends Stat { 61 | def apply(): Seq[Float] 62 | } 63 | -------------------------------------------------------------------------------- /util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache 2 | 3 | import com.twitter.cache.guava.LoadingFutureCache 4 | import com.twitter.util.{Future, Throw} 5 | 6 | private[cache] class EvictingCache[K, V](underlying: FutureCache[K, V]) extends FutureCacheProxy[K, V](underlying) { 7 | private[this] def evictOnFailure(k: K, f: Future[V]): Future[V] = { 8 | f onFailure { case t: Throwable => 9 | evict(k, f) 10 | } 11 | f // we return the original future to make evict(k, f) easier to work with. 12 | } 13 | 14 | override def set(k: K, v: Future[V]) { 15 | super.set(k, v) 16 | evictOnFailure(k, v) 17 | } 18 | 19 | override def getOrElseUpdate(k: K)(v: => Future[V]): Future[V] = 20 | evictOnFailure(k, underlying.getOrElseUpdate(k) { 21 | v 22 | }) 23 | } 24 | 25 | private[cache] class LazilyEvictingCache[K, V]( 26 | underlying: LoadingFutureCache[K, V] 27 | ) extends FutureCacheProxy[K, V](underlying) { 28 | private[this] def invalidateLazily(k: K, f: Future[V]): Unit = { 29 | f.poll match { 30 | case Some(Throw(e)) => underlying.invalidate(k) 31 | case _ => 32 | } 33 | } 34 | 35 | override def get(k: K): Option[Future[V]] = { 36 | val result = super.get(k) 37 | result foreach { fut => 38 | invalidateLazily(k, fut) 39 | } 40 | result 41 | } 42 | 43 | override def getOrElseUpdate(k: K)(v: => Future[V]): Future[V] = { 44 | val result = super.getOrElseUpdate(k)(v) 45 | invalidateLazily(k, result) 46 | result 47 | } 48 | } 49 | 50 | object EvictingCache { 51 | /** 52 | * Wraps an underlying FutureCache, ensuring that failed Futures that are set in 53 | * the cache are evicted later. 54 | */ 55 | def apply[K, V](underlying: FutureCache[K, V]): FutureCache[K, V] = 56 | new EvictingCache[K, V](underlying) 57 | 58 | /** 59 | * Wraps an underlying FutureCache, ensuring that if a failed future 60 | * is fetched, we evict it. 61 | */ 62 | def lazily[K, V](underlying: LoadingFutureCache[K, V]): FutureCache[K, V] = 63 | new LazilyEvictingCache[K, V](underlying) 64 | } 65 | -------------------------------------------------------------------------------- /util-codec/src/main/scala/com/twitter/util/StringEncoder.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import java.io.{ByteArrayInputStream, ByteArrayOutputStream} 4 | import java.util.zip.{GZIPInputStream, GZIPOutputStream} 5 | 6 | import org.apache.commons.codec.binary.Base64 7 | 8 | import com.twitter.io.StreamIO 9 | 10 | trait StringEncoder { 11 | def encode(bytes: Array[Byte]): String = new String(bytes) 12 | def decode(str: String): Array[Byte] = str.getBytes 13 | } 14 | 15 | trait Base64StringEncoder extends StringEncoder { 16 | private[this] def codec = new Base64() 17 | 18 | override def encode(bytes: Array[Byte]): String = { 19 | codec.encodeToString(bytes) 20 | } 21 | 22 | override def decode(str: String): Array[Byte] = 23 | codec.decode(str) 24 | } 25 | 26 | object StringEncoder extends StringEncoder 27 | object Base64StringEncoder extends Base64StringEncoder 28 | 29 | /** 30 | * A collection of utilities for encoding strings and byte arrays to and decoding from strings 31 | * compressed from with gzip. 32 | * 33 | * This trait is thread-safe because there are no streams shared outside of method scope, and 34 | * therefore no contention for shared byte arrays. 35 | * 36 | * The encoding for strings is UTF-8. 37 | * 38 | * gzipping inherently includes base64 encoding (the GZIP utilities from java will complain 39 | * otherwise!) 40 | */ 41 | trait GZIPStringEncoder extends StringEncoder { 42 | override def encode(bytes: Array[Byte]): String = { 43 | val baos = new ByteArrayOutputStream 44 | val gos = new GZIPOutputStream(baos) 45 | gos.write(bytes) 46 | gos.finish() 47 | Base64StringEncoder.encode(baos.toByteArray) 48 | } 49 | 50 | def encodeString(str: String) = encode(str.getBytes("UTF-8")) 51 | 52 | override def decode(str: String): Array[Byte] = { 53 | val baos = new ByteArrayOutputStream 54 | StreamIO.copy(new GZIPInputStream(new ByteArrayInputStream(Base64StringEncoder.decode(str))), baos) 55 | 56 | baos.toByteArray 57 | } 58 | 59 | def decodeString(str: String): String = new String(decode(str), "UTF-8") 60 | } 61 | 62 | object GZIPStringEncoder extends GZIPStringEncoder 63 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/io/TempFile.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import java.io.{BufferedOutputStream, File, FileNotFoundException, FileOutputStream} 4 | 5 | object TempFile { 6 | /** 7 | * Create a temporary file from the given (resource) path. The 8 | * tempfile is deleted on JVM exit. 9 | * 10 | * Note, due to the usage of `File.deleteOnExit()` callers should 11 | * be careful using this as it can leak memory. 12 | * See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for 13 | * example. 14 | * 15 | * @param path the resource-relative path to make a temp file from 16 | * @return the temp File object 17 | */ 18 | def fromResourcePath(path: String): File = fromResourcePath(getClass, path) 19 | 20 | /** 21 | * Create a temporary file from the given (resource) path. The 22 | * tempfile is deleted on JVM exit. 23 | * 24 | * Note, due to the usage of `File.deleteOnExit()` callers should 25 | * be careful using this as it can leak memory. 26 | * See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for 27 | * example. 28 | * 29 | * @param klass the `Class` to use for getting the resource. 30 | * @param path the resource-relative path to make a temp file from 31 | * @return the temp File object 32 | */ 33 | def fromResourcePath(klass: Class[_], path: String): File = { 34 | val (basename, ext) = { 35 | val last = path.split(File.separatorChar).last 36 | last.split('.').reverse match { 37 | case Array(basename) => 38 | (basename, "") 39 | case Array(ext, base@_*) => 40 | (base.reverse.mkString("."), ext) 41 | } 42 | } 43 | 44 | klass.getResourceAsStream(path) match { 45 | case null => 46 | throw new FileNotFoundException(path) 47 | case stream => 48 | val file = File.createTempFile(basename, "." + ext) 49 | file.deleteOnExit() 50 | val fos = new BufferedOutputStream(new FileOutputStream(file), 1<<20) 51 | StreamIO.copy(stream, fos) 52 | fos.flush() 53 | fos.close() 54 | stream.close() 55 | file 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.finagle.stats; 2 | 3 | import com.twitter.util.Future$; 4 | import com.twitter.util.Future; 5 | 6 | import org.junit.Test; 7 | 8 | import java.lang.Integer; 9 | import java.util.concurrent.Callable; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | /** 13 | * Java compatibility layer for {@link com.twitter.finagle.stats.StatsReceiver}. 14 | */ 15 | public final class StatsReceiverCompilationTest { 16 | 17 | @Test 18 | public void testNullStatsReceiver() { 19 | StatsReceiver sr = NullStatsReceiver.get(); 20 | } 21 | 22 | @Test 23 | public void testStatMethods() { 24 | InMemoryStatsReceiver sr = new InMemoryStatsReceiver(); 25 | Stat stat = StatsReceivers.stat(sr, "hello", "world"); 26 | Callable callable = new Callable() { 27 | public Integer call() { 28 | return 3; 29 | } 30 | }; 31 | Callable> fCallable = new Callable>() { 32 | public Future call() { 33 | return Future.value(new Integer(3)); 34 | } 35 | }; 36 | TimeUnit unit = TimeUnit.MILLISECONDS; 37 | 38 | JStats.time(stat, callable); 39 | JStats.time(stat, callable, unit); 40 | JStats.timeFuture(stat, fCallable); 41 | JStats.timeFuture(stat, fCallable, unit); 42 | stat.add(0); 43 | } 44 | 45 | @Test 46 | public void testGaugeMethods() { 47 | InMemoryStatsReceiver sr = new InMemoryStatsReceiver(); 48 | Callable callable = new Callable() { 49 | public Float call() { 50 | return 3.0f; 51 | } 52 | }; 53 | Gauge gauge = StatsReceivers.addGauge(sr, callable, "hello", "world"); 54 | StatsReceivers.provideGauge(sr, callable, "hello", "world"); 55 | gauge.remove(); 56 | } 57 | 58 | @Test 59 | public void testCounterMethods() { 60 | InMemoryStatsReceiver sr = new InMemoryStatsReceiver(); 61 | sr.isNull(); 62 | Counter counter = StatsReceivers.counter(sr.scopeSuffix("bah").scope("foo"), "hello", "world"); 63 | counter.incr(); 64 | counter.incr(100); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /util-zk-common/src/test/scala/com/twitter/zk/CommonConnectorTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | import java.net.InetSocketAddress 4 | 5 | import scala.collection.JavaConverters._ 6 | 7 | import org.junit.runner.RunWith 8 | import org.scalatest.junit.JUnitRunner 9 | import org.scalatest.{BeforeAndAfter, WordSpec} 10 | 11 | import com.twitter.common.net.InetSocketAddressHelper 12 | import com.twitter.common.zookeeper.ZooKeeperClient 13 | import com.twitter.conversions.common.quantity._ 14 | import com.twitter.conversions.common.zookeeper._ 15 | import com.twitter.conversions.time._ 16 | import com.twitter.util.{Await, FuturePool, RandomSocket} 17 | 18 | @RunWith(classOf[JUnitRunner]) 19 | class CommonConnectorTest extends WordSpec with BeforeAndAfter { 20 | val timeout = 2.seconds 21 | val port = RandomSocket.nextPort() 22 | val addresses = new InetSocketAddress("localhost", port) :: Nil 23 | 24 | "CommonConnector" should { 25 | "initialize" should { 26 | "with addresses" in { 27 | implicit val pool = FuturePool.immediatePool 28 | assert(CommonConnector(addresses, timeout) != null) 29 | } 30 | 31 | "with a ZooKeeperClient instance" in { 32 | implicit val pool = FuturePool.immediatePool 33 | val zookeeper = new ZooKeeperClient(timeout.toIntAmount, addresses.asJava) 34 | val connector = CommonConnector(zookeeper, timeout) 35 | assert(connector.underlying === zookeeper) 36 | } 37 | } 38 | } 39 | 40 | // A simple live test 41 | Option { System.getProperty("com.twitter.zk.TEST_CONNECT") } foreach { connectString => 42 | val address = InetSocketAddressHelper.parse(connectString) 43 | 44 | "A live server @ %s".format(connectString) should { 45 | val commonClient: ZooKeeperClient = new ZooKeeperClient(timeout.toIntAmount, address) 46 | val zkClient = commonClient.toZkClient(timeout)(FuturePool.immediatePool) 47 | 48 | after { 49 | Await.ready(zkClient.release()) 50 | } 51 | 52 | "have 'zookeeper' in '/'" in { 53 | assert(Await.result(zkClient("/").getChildren(), timeout).children map { _.name } contains("zookeeper")) 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/concurrent/ConcurrentBijection.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Twitter, Inc. 2 | 3 | package com.twitter.concurrent 4 | 5 | import java.util.concurrent.ConcurrentHashMap 6 | 7 | import scala.collection.JavaConversions._ 8 | import scala.collection.mutable.{Map => MMap} 9 | 10 | // A bijection that may be modified and accessed simultaneously. Note 11 | // that we can allow only one modification at a time. Updates need to 12 | // be serialized to ensure that the bijective property is maintained. 13 | class ConcurrentBijection[A, B] extends MMap[A, B] { 14 | val forward = new ConcurrentHashMap[A, B] 15 | val reverse = new ConcurrentHashMap[B, A] 16 | 17 | def toOpt[T](x: T) = if (x == null) None else Some(x) 18 | 19 | def -=(key: A) = { 20 | synchronized { 21 | val value = forward.remove(key) 22 | if (value != null) 23 | reverse.remove(value) 24 | } 25 | this 26 | } 27 | 28 | def +=(elem: (A, B)) = { 29 | elem match { 30 | case (key, value) => update(key, value) 31 | } 32 | this 33 | } 34 | 35 | override def update(key: A, value: B) = synchronized { 36 | // We need to update: 37 | // 38 | // a -> b 39 | // b -> a 40 | // 41 | // There may be existing mappings: 42 | // 43 | // a -> b' 44 | // b' -> a 45 | // 46 | // or 47 | // 48 | // a' -> b 49 | // b -> a' 50 | // 51 | // So we need to ensure these are killed. 52 | val oldValue = forward.put(key, value) 53 | if (oldValue != value) { 54 | if (oldValue != null) { 55 | // Remove the old reverse mapping. 56 | val keyForOldValue = reverse.remove(oldValue) 57 | if (key != keyForOldValue) 58 | forward.remove(keyForOldValue) 59 | } 60 | 61 | val oldKeyForValue = reverse.put(value, key) 62 | if (oldKeyForValue != null) 63 | forward.remove(oldKeyForValue) 64 | } 65 | } 66 | 67 | override def size = forward.size 68 | 69 | def get(key: A) = toOpt(forward.get(key)) 70 | def getReverse(value: B) = toOpt(reverse.get(value)) 71 | 72 | def iterator = forward.entrySet.iterator.map(e => (e.getKey, e.getValue)) 73 | } 74 | -------------------------------------------------------------------------------- /util-events/src/main/scala/com/twitter/util/events/Sink.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util.events 2 | 3 | import com.twitter.util.events.Event.Type 4 | 5 | /** 6 | * Where runtime events such as logging, stats and tracing can be 7 | * sent to allow for analysis. 8 | * 9 | * '''Note:''' while the API is public it should be considered as experimental 10 | * and subject to changes. 11 | * 12 | * ===Design notes=== 13 | * - Implementations must be thread-safe. 14 | * - Implementations should have very low runtime overhead such that as 15 | * many events as possible can be sent here. In particular, object 16 | * allocations should be kept to a minimum. 17 | * - `event` is expected to be called many orders of magnitude 18 | * more frequently than `events`. 19 | */ 20 | trait Sink { 21 | 22 | /** 23 | * Event input is captured as individual fields in service of 24 | * avoiding an allocation to wrap the event. 25 | */ 26 | def event( 27 | etype: Event.Type, 28 | longVal: Long = Event.NoLong, 29 | objectVal: Object = Event.NoObject, 30 | doubleVal: Double = Event.NoDouble 31 | ): Unit 32 | 33 | /** 34 | * Returns all currently available events. 35 | * 36 | * '''Note:''' the events are not returned in any particular order. 37 | */ 38 | def events: Iterator[Event] 39 | 40 | } 41 | 42 | object Sink { 43 | 44 | /** 45 | * A sink that ignores all input. 46 | */ 47 | val Null: Sink = new Sink { 48 | override def event( 49 | etype: Type, 50 | longVal: Long, 51 | objectVal: Object, 52 | doubleVal: Double 53 | ): Unit = () 54 | 55 | override def events: Iterator[Event] = Iterator.empty 56 | } 57 | 58 | // exposed for testing 59 | private[events] def newDefault: Sink = { 60 | if (!sinkEnabled.apply()) { 61 | Null 62 | } else if (approxNumEvents() <= 0) { 63 | Null 64 | } else { 65 | SizedSink(approxNumEvents()) 66 | } 67 | } 68 | 69 | /** 70 | * The global default `Sink`. 71 | */ 72 | val default: Sink = newDefault 73 | 74 | /** 75 | * Returns whether or not any event capture is enabled. 76 | */ 77 | def enabled: Boolean = default ne Null 78 | 79 | } 80 | -------------------------------------------------------------------------------- /util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.jvm 2 | 3 | import java.lang.Thread.State._ 4 | import java.lang.management.{ManagementFactory, ThreadInfo} 5 | 6 | /** 7 | * A thread contention summary providing a brief overview of threads 8 | * that are [[http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#BLOCKED BLOCKED]], [[http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#WAITING WAITING]], or [[http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#TIMED_WAITING TIMED_WAITING]] 9 | * 10 | * While this could be an object, we use instantiation as a signal of 11 | * intent and enable contention monitoring. 12 | */ 13 | class ContentionSnapshot { 14 | ManagementFactory.getThreadMXBean.setThreadContentionMonitoringEnabled(true) 15 | 16 | case class Snapshot( 17 | blockedThreads: Seq[String], 18 | lockOwners: Seq[String], 19 | deadlocks: Seq[String]) 20 | 21 | private[this] object Blocked { 22 | def unapply(t: ThreadInfo): Option[ThreadInfo] = { 23 | t.getThreadState match { 24 | case BLOCKED | WAITING | TIMED_WAITING => Some(t) 25 | case _ => None 26 | } 27 | } 28 | } 29 | 30 | def snap(): Snapshot = { 31 | val bean = ManagementFactory.getThreadMXBean 32 | 33 | val blocked = bean.getThreadInfo(bean.getAllThreadIds, true, true) 34 | .filter(_ != null) 35 | .collect { case Blocked(info) => info } 36 | 37 | val ownerIds = blocked map(_.getLockOwnerId) filter(_ != -1) 38 | val owners = if (ownerIds.length == 0) Seq[String]() else 39 | bean.getThreadInfo(ownerIds.toArray, true, true).map(_.toString).toSeq 40 | 41 | val deadlockThreadIds = bean.findDeadlockedThreads() 42 | val deadlocks = if (deadlockThreadIds == null) Array.empty[ThreadInfo] else 43 | deadlockThreadIds.flatMap { id => 44 | blocked.find { threadInfo => 45 | threadInfo.getThreadId() == id 46 | } 47 | } 48 | 49 | Snapshot( 50 | blockedThreads = blocked.map(_.toString).toSeq, 51 | lockOwners = owners, 52 | deadlocks = deadlocks.map(_.toString).toSeq) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | import java.util.{List => JList} 4 | 5 | import scala.collection.JavaConverters._ 6 | 7 | import org.apache.zookeeper.data.Stat 8 | import org.apache.zookeeper.{AsyncCallback, KeeperException} 9 | 10 | import com.twitter.util.{Promise, Return, Throw} 11 | 12 | /** Mix-in to make an AsyncCallback a Promise */ 13 | trait AsyncCallbackPromise[T] extends Promise[T] { 14 | /** Process result iff rc is OK; otherwise throw a KeeperException. */ 15 | protected def process(rc: Int, path: String)(result: => T) { 16 | KeeperException.Code.get(rc) match { 17 | case KeeperException.Code.OK => updateIfEmpty(Return(result)) 18 | case code => updateIfEmpty(Throw(KeeperException.create(code, path))) 19 | } 20 | } 21 | } 22 | 23 | class StringCallbackPromise extends AsyncCallbackPromise[String] with AsyncCallback.StringCallback { 24 | def processResult(rc: Int, path: String, ctx: AnyRef, name: String) { 25 | process(rc, path) { name } 26 | } 27 | } 28 | 29 | class UnitCallbackPromise extends AsyncCallbackPromise[Unit] with AsyncCallback.VoidCallback { 30 | def processResult(rc: Int, path: String, ctx: AnyRef) { 31 | process(rc, path) { Unit } 32 | } 33 | } 34 | 35 | class ExistsCallbackPromise(znode: ZNode) extends AsyncCallbackPromise[ZNode.Exists] 36 | with AsyncCallback.StatCallback { 37 | def processResult(rc: Int, path: String, ctx: AnyRef, stat: Stat) { 38 | process(rc, path) { 39 | znode(stat) 40 | } 41 | } 42 | } 43 | 44 | class ChildrenCallbackPromise(znode: ZNode) extends AsyncCallbackPromise[ZNode.Children] 45 | with AsyncCallback.Children2Callback { 46 | def processResult(rc: Int, path: String, ctx: AnyRef, children: JList[String], stat: Stat) { 47 | process(rc, path) { 48 | znode(stat, children.asScala.toSeq) 49 | } 50 | } 51 | } 52 | 53 | class DataCallbackPromise(znode: ZNode) extends AsyncCallbackPromise[ZNode.Data] 54 | with AsyncCallback.DataCallback { 55 | def processResult(rc: Int, path: String, ctx: AnyRef, bytes: Array[Byte], stat: Stat) { 56 | process(rc, path) { 57 | znode(stat, bytes) 58 | } 59 | } 60 | } 61 | 62 | -------------------------------------------------------------------------------- /util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.cache 2 | 3 | import com.twitter.util.{Promise, Future} 4 | import java.util.concurrent.ConcurrentHashMap 5 | import org.junit.runner.RunWith 6 | import org.mockito.invocation.InvocationOnMock 7 | import org.mockito.Matchers.{anyString, anyObject} 8 | import org.mockito.Mockito.{verify, never, when} 9 | import org.mockito.stubbing.Answer 10 | import org.scalatest.FunSuite 11 | import org.scalatest.junit.JUnitRunner 12 | import org.scalatest.mock.MockitoSugar 13 | 14 | @RunWith(classOf[JUnitRunner]) 15 | class EvictingCacheTest extends FunSuite with MockitoSugar { 16 | test("EvictingCache should evict on failed futures for set") { 17 | val cache = mock[FutureCache[String, String]] 18 | val fCache = new EvictingCache(cache) 19 | val p = Promise[String] 20 | fCache.set("key", p) 21 | verify(cache).set("key", p) 22 | p.setException(new Exception) 23 | verify(cache).evict("key", p) 24 | } 25 | 26 | test("EvictingCache should keep satisfied futures for set") { 27 | val cache = mock[FutureCache[String, String]] 28 | val fCache = new EvictingCache(cache) 29 | val p = Promise[String] 30 | fCache.set("key", p) 31 | verify(cache).set("key", p) 32 | p.setValue("value") 33 | verify(cache, never).evict("key", p) 34 | } 35 | 36 | test("EvictingCache should evict on failed futures for getOrElseUpdate") { 37 | val map = new ConcurrentHashMap[String, Future[String]]() 38 | val cache = new ConcurrentMapCache(map) 39 | val fCache = new EvictingCache(cache) 40 | val p = Promise[String] 41 | assert(fCache.getOrElseUpdate("key")(p).poll === p.poll) 42 | p.setException(new Exception) 43 | assert(fCache.get("key") === None) 44 | } 45 | 46 | test("EvictingCache should keep satisfied futures for getOrElseUpdate") { 47 | val map = new ConcurrentHashMap[String, Future[String]]() 48 | val cache = new ConcurrentMapCache(map) 49 | val fCache = new EvictingCache(cache) 50 | val p = Promise[String] 51 | assert(fCache.getOrElseUpdate("key")(p).poll === p.poll) 52 | p.setValue("value") 53 | assert(fCache.get("key").map(_.poll) === Some(p.poll)) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/Bijection.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * This Bijection is slated to be replaced and is deprecated in favor of 3 | * com.twitter.bijection.Bijection (which lives at https://github.com/twitter/bijection). 4 | */ 5 | package com.twitter.util 6 | 7 | @deprecated("Prefer using com.twitter.bijection.Bijection", "6.17.1") 8 | object Bijection { 9 | implicit def identity[A]: Bijection[A, A] = new Bijection[A, A] { 10 | def apply(a: A) = a 11 | def invert(a: A) = a 12 | 13 | override def inverse = this 14 | override def andThen[T](g: Bijection[A, T]) = g 15 | override def compose[T](g: Bijection[T, A]) = g 16 | } 17 | } 18 | 19 | /** 20 | * A bijection is a function for which every value in its codomain 21 | * (set of all possible results) is equivalent to application of the 22 | * function on a unique value in its domain (all possible 23 | * inputs). This trait in particular provides an interface for 24 | * functions that can be 'unapplied' as well as applied. A codec that 25 | * can convert to and from a set of objects and their serialized form 26 | * is an example of a bijection. 27 | */ 28 | @deprecated("Prefer using com.twitter.bijection.Bijection", "2014-06-05") 29 | trait Bijection[A, B] extends (A => B) { self => 30 | def apply(a: A): B 31 | def invert(b: B): A 32 | def unapply(b: B) = Try(Some(invert(b))).getOrElse(None) 33 | 34 | /** 35 | * Return a Bijection that is the inverse of this one. 36 | */ 37 | def inverse: Bijection[B, A] = _inverse 38 | 39 | private lazy val _inverse = { 40 | new Bijection[B, A] { 41 | def apply(b: B) = self.invert(b) 42 | def invert(a: A) = self.apply(a) 43 | override def inverse = self 44 | } 45 | } 46 | 47 | /** 48 | * Composes two instances of Bijection in a new Bijection, with this one applied first. 49 | */ 50 | def andThen[C](g: Bijection[B, C]) = { 51 | new Bijection[A, C] { 52 | def apply(a: A) = g(self(a)) 53 | def invert(c: C) = self.invert(g.invert(c)) 54 | } 55 | } 56 | 57 | /** 58 | * Composes two instances of Bijection in a new Bijection, with this one applied last. 59 | */ 60 | def compose[T](g: Bijection[T, A]) = g andThen this 61 | } 62 | -------------------------------------------------------------------------------- /util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | 3 | import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream} 4 | 5 | import com.fasterxml.jackson.databind.MappingJsonFactory 6 | import org.apache.thrift.TBase 7 | import org.apache.thrift.protocol.{TBinaryProtocol, TCompactProtocol, TProtocolFactory, TSimpleJSONProtocol} 8 | import org.apache.thrift.transport.TIOStreamTransport 9 | 10 | trait ThriftSerializer extends StringEncoder { 11 | def protocolFactory: TProtocolFactory 12 | 13 | def toBytes(obj: TBase[_, _], bufSize: Int): Array[Byte] = { 14 | val baos = new ByteArrayOutputStream(bufSize) 15 | obj.write(protocolFactory.getProtocol(new TIOStreamTransport(baos))) 16 | baos.toByteArray 17 | } 18 | 19 | def toBytes(obj: TBase[_, _]): Array[Byte] = { 20 | toBytes(obj, 32) // default initial size of ByteArrayOutputStream 21 | } 22 | 23 | def fromInputStream(obj: TBase[_, _], stream: InputStream): Unit = 24 | obj.read(protocolFactory.getProtocol(new TIOStreamTransport(stream))) 25 | 26 | def fromBytes(obj: TBase[_, _], bytes: Array[Byte]): Unit = 27 | fromInputStream(obj, new ByteArrayInputStream(bytes)) 28 | 29 | def toString(obj: TBase[_, _], bufSize: Int): String = encode(toBytes(obj, bufSize)) 30 | 31 | def toString(obj: TBase[_, _]): String = encode(toBytes(obj)) 32 | 33 | def fromString(obj: TBase[_, _], str: String): Unit = fromBytes(obj, decode(str)) 34 | } 35 | 36 | class JsonThriftSerializer extends ThriftSerializer { 37 | override def protocolFactory = new TSimpleJSONProtocol.Factory 38 | 39 | override def fromBytes(obj: TBase[_, _], bytes: Array[Byte]): Unit = { 40 | val binarySerializer = new BinaryThriftSerializer 41 | val newObj = new MappingJsonFactory().createJsonParser(bytes).readValueAs(obj.getClass) 42 | binarySerializer.fromBytes(obj, binarySerializer.toBytes(newObj.asInstanceOf[TBase[_, _]])) 43 | } 44 | } 45 | 46 | class BinaryThriftSerializer extends ThriftSerializer with Base64StringEncoder { 47 | override def protocolFactory = new TBinaryProtocol.Factory 48 | } 49 | 50 | class CompactThriftSerializer extends ThriftSerializer with Base64StringEncoder { 51 | override def protocolFactory = new TCompactProtocol.Factory 52 | } 53 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/conversions/time.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter 18 | package conversions 19 | 20 | import java.util.concurrent.TimeUnit 21 | 22 | import com.twitter.util.Duration 23 | 24 | object time { 25 | class RichWholeNumber(wrapped: Long) { 26 | def nanoseconds = Duration(wrapped, TimeUnit.NANOSECONDS) 27 | def nanosecond = nanoseconds 28 | def microseconds = Duration(wrapped, TimeUnit.MICROSECONDS) 29 | def microsecond = microseconds 30 | def milliseconds = Duration(wrapped, TimeUnit.MILLISECONDS) 31 | def millisecond = milliseconds 32 | def millis = milliseconds 33 | def seconds = Duration(wrapped, TimeUnit.SECONDS) 34 | def second = seconds 35 | def minutes = Duration(wrapped, TimeUnit.MINUTES) 36 | def minute = minutes 37 | def hours = Duration(wrapped, TimeUnit.HOURS) 38 | def hour = hours 39 | def days = Duration(wrapped, TimeUnit.DAYS) 40 | def day = days 41 | } 42 | 43 | private val ZeroRichWholeNumber = new RichWholeNumber(0) { 44 | override def nanoseconds = Duration.Zero 45 | override def microseconds = Duration.Zero 46 | override def milliseconds = Duration.Zero 47 | override def seconds = Duration.Zero 48 | override def minutes = Duration.Zero 49 | override def hours = Duration.Zero 50 | override def days = Duration.Zero 51 | } 52 | 53 | implicit def intToTimeableNumber(i: Int): RichWholeNumber = 54 | if (i == 0) ZeroRichWholeNumber else new RichWholeNumber(i) 55 | implicit def longToTimeableNumber(l: Long): RichWholeNumber = 56 | if (l == 0) ZeroRichWholeNumber else new RichWholeNumber(l) 57 | } 58 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/concurrent/TxTest.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent 2 | 3 | 4 | import org.junit.runner.RunWith 5 | import org.scalatest.WordSpec 6 | import org.scalatest.junit.JUnitRunner 7 | 8 | import com.twitter.util.Return 9 | 10 | @RunWith(classOf[JUnitRunner]) 11 | class TxTest extends WordSpec { 12 | "Tx.twoParty" should { 13 | "commit when everything goes dandy" in { 14 | val (stx, rtx) = Tx.twoParty(123) 15 | val sf = stx.ack() 16 | assert(sf.poll === None) 17 | val rf = rtx.ack() 18 | assert(sf.poll === Some(Return(Tx.Commit(())))) 19 | assert(rf.poll === Some(Return(Tx.Commit(123)))) 20 | } 21 | 22 | "abort when receiver nacks" in { 23 | val (stx, rtx) = Tx.twoParty(123) 24 | val sf = stx.ack() 25 | assert(sf.poll === None) 26 | rtx.nack() 27 | assert(sf.poll === Some(Return(Tx.Abort))) 28 | } 29 | 30 | "abort when sender nacks" in { 31 | val (stx, rtx) = Tx.twoParty(123) 32 | val rf = rtx.ack() 33 | assert(rf.poll === None) 34 | stx.nack() 35 | assert(rf.poll === Some(Return(Tx.Abort))) 36 | } 37 | 38 | "complain on ack ack" in { 39 | val (stx, rtx) = Tx.twoParty(123) 40 | rtx.ack() 41 | 42 | assert(intercept[Exception] { 43 | rtx.ack() 44 | } === Tx.AlreadyAckd) 45 | } 46 | 47 | "complain on ack nack" in { 48 | val (stx, rtx) = Tx.twoParty(123) 49 | rtx.ack() 50 | 51 | assert(intercept[Exception] { 52 | rtx.nack() 53 | } === Tx.AlreadyAckd) 54 | } 55 | 56 | "complain on nack ack" in { 57 | val (stx, rtx) = Tx.twoParty(123) 58 | rtx.nack() 59 | 60 | assert(intercept[Exception] { 61 | rtx.ack() 62 | } === Tx.AlreadyNackd) 63 | } 64 | 65 | "complain on nack nack" in { 66 | val (stx, rtx) = Tx.twoParty(123) 67 | rtx.nack() 68 | 69 | assert(intercept[Exception] { 70 | rtx.nack() 71 | } === Tx.AlreadyNackd) 72 | } 73 | 74 | "complain when already done" in { 75 | val (stx, rtx) = Tx.twoParty(123) 76 | stx.ack() 77 | rtx.ack() 78 | 79 | assert(intercept[Exception] { 80 | stx.ack() 81 | } === Tx.AlreadyDone) 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2010 Twitter, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may 5 | * not use this file except in compliance with the License. You may obtain 6 | * a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.twitter.util 18 | 19 | import org.junit.runner.RunWith 20 | import org.scalatest.WordSpec 21 | import org.scalatest.junit.JUnitRunner 22 | 23 | import com.twitter.conversions.string._ 24 | 25 | @RunWith(classOf[JUnitRunner]) 26 | class StringConversionsTest extends WordSpec { 27 | "string" should { 28 | "quoteC" in { 29 | assert("nothing".quoteC === "nothing") 30 | assert("name\tvalue\t\u20acb\u00fcllet?\u20ac".quoteC === "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac") 31 | assert("she said \"hello\"".quoteC === "she said \\\"hello\\\"") 32 | assert("\\backslash".quoteC === "\\\\backslash") 33 | } 34 | 35 | "unquoteC" in { 36 | assert("nothing".unquoteC === "nothing") 37 | assert("name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac".unquoteC === "name\tvalue\t\u20acb\u00fcllet?\u20ac") 38 | assert("she said \\\"hello\\\"".unquoteC === "she said \"hello\"") 39 | assert("\\\\backslash".unquoteC === "\\backslash") 40 | assert("real\\$dollar".unquoteC === "real\\$dollar") 41 | assert("silly\\/quote".unquoteC === "silly/quote") 42 | } 43 | 44 | "hexlify" in { 45 | assert("hello".getBytes.slice(1, 4).hexlify === "656c6c") 46 | assert("hello".getBytes.hexlify === "68656c6c6f") 47 | } 48 | 49 | "unhexlify" in { 50 | assert("656c6c".unhexlify.toList === "hello".getBytes.slice(1, 4).toList) 51 | assert("68656c6c6f".unhexlify.toList === "hello".getBytes.toList) 52 | "5".unhexlify 53 | assert("5".unhexlify.hexlify.toInt === 5) 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /util-benchmark/src/main/scala/com/twitter/util/benchmark/CpuProfile.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util.benchmark 2 | 3 | import java.lang.management.ManagementFactory 4 | 5 | import scala.util.Random 6 | 7 | import com.google.caliper.SimpleBenchmark 8 | 9 | class CpuProfileBenchmark extends SimpleBenchmark { 10 | 11 | // TODO: change dynamically. bounce up & down 12 | // the stack. μ and σ are for *this* stack. 13 | class Stack(rng: Random, μ: Int, σ: Int) { 14 | def apply() = { 15 | val depth = math.max(1, μ + (rng.nextGaussian*σ).toInt) 16 | dive(depth) 17 | } 18 | 19 | private def dive(depth: Int): Int = { 20 | if (depth == 0) { 21 | while (true) { 22 | Thread.sleep(10<<20) 23 | } 24 | 1 25 | } else 26 | 1+dive(depth-1) // make sure we don't tail recurse 27 | } 28 | } 29 | 30 | var stackMeanSize = 40 31 | var stackStddev = 10 32 | var nthreads = 16 33 | var threads = Seq[Thread]() 34 | var rngSeed = 131451732492626L 35 | 36 | override protected def setUp() { 37 | val stack = new Stack(new Random(rngSeed), stackMeanSize, stackStddev) 38 | threads = for (_ <- 0 until nthreads) yield new Thread { 39 | override def run() { 40 | try stack() catch { 41 | case _: InterruptedException => 42 | } 43 | } 44 | } 45 | 46 | threads foreach { t => t.start() } 47 | } 48 | 49 | override protected def tearDown() { 50 | threads foreach { t => t.interrupt() } 51 | threads foreach { t => t.join() } 52 | threads = Seq() 53 | } 54 | 55 | def timeThreadGetStackTraces(nreps: Int) { 56 | var i = 0 57 | while (i < nreps) { 58 | Thread.getAllStackTraces() 59 | i += 1 60 | } 61 | } 62 | 63 | def timeThreadInfoBare(nreps: Int) { 64 | val bean = ManagementFactory.getThreadMXBean() 65 | var i = 0 66 | while (i < nreps) { 67 | bean.dumpAllThreads(false, false) 68 | i += 1 69 | } 70 | } 71 | 72 | // Note: we should actually simulate some contention, too. 73 | def timeThreadInfoFull(nreps: Int) { 74 | val bean = ManagementFactory.getThreadMXBean() 75 | var i = 0 76 | while (i < nreps) { 77 | bean.dumpAllThreads(true, true) 78 | i += 1 79 | } 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/io/BufInputStream.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.io 2 | 3 | import java.io.InputStream 4 | 5 | class BufInputStream(val buf: Buf) extends InputStream { 6 | /** 7 | * The slice of `buf` that hasn't been consumed. 8 | */ 9 | private[this] var rest: Buf = buf 10 | 11 | /** 12 | * A saved version of `rest` 13 | */ 14 | private[this] var mrk: Buf = buf 15 | 16 | // Returns an estimate of the number of bytes that can be read (or 17 | // skipped over) from this input stream without blocking by the next 18 | // invocation of a method for this input stream. 19 | override def available(): Int = synchronized { rest.length } 20 | 21 | // Closing a BufInputStream has no effect. 22 | override def close() {} 23 | 24 | // Marks the current position in this input stream. 25 | override def mark(readlimit: Int) = synchronized { mrk = rest } 26 | 27 | // Tests if this input stream supports the mark and reset methods. 28 | override def markSupported(): Boolean = true 29 | 30 | // Reads the next byte of data from the input stream. 31 | def read(): Int = synchronized { 32 | if (rest.length <= 0) 33 | return -1 34 | 35 | val b = new Array[Byte](1) 36 | rest.slice(0, 1).write(b, 0) 37 | rest = rest.slice(1, rest.length) 38 | b(0) & 0xFF 39 | } 40 | 41 | /** 42 | * Reads up to len bytes of data from the input stream into an 43 | * array of bytes. 44 | */ 45 | override def read(b: Array[Byte], off: Int, len: Int): Int = synchronized { 46 | if (rest.length <= 0) 47 | return -1 48 | 49 | if (len == 0) 50 | return 0 51 | 52 | val n = len min rest.length 53 | rest.slice(0, n).write(b, off) 54 | rest = rest.slice(n, rest.length) 55 | n 56 | } 57 | 58 | /** 59 | * Repositions this stream to the position at the time the mark 60 | * method was last called on this input stream. 61 | */ 62 | override def reset() = synchronized { rest = mrk } 63 | 64 | /** 65 | * Skips over and discards n bytes of data from this input stream. 66 | */ 67 | override def skip(n: Long): Long = synchronized { 68 | if (n <= 0) 69 | return 0 70 | 71 | val skipped = n min rest.length 72 | rest = rest.slice(skipped.toInt, rest.length) 73 | skipped 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java: -------------------------------------------------------------------------------- 1 | package com.twitter.concurrent; 2 | 3 | import com.twitter.util.Await; 4 | import com.twitter.util.Future; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | import scala.collection.JavaConversions; 8 | 9 | import java.util.Arrays; 10 | import java.util.Collection; 11 | 12 | public class SpoolCompilationTest { 13 | private static class OwnSpool extends AbstractSpool { 14 | @Override 15 | public boolean isEmpty() { 16 | return false; 17 | } 18 | 19 | @Override 20 | public Future> tail() { 21 | return Future.value(Spools.newEmptySpool()); 22 | } 23 | 24 | @Override 25 | public String head() { 26 | return "spool"; 27 | } 28 | } 29 | 30 | @Test 31 | public void testOwnSpool() { 32 | Spool a = new OwnSpool(); 33 | Assert.assertFalse(a.isEmpty()); 34 | Assert.assertEquals("spool", a.head()); 35 | } 36 | 37 | @Test 38 | public void testSpoolCreation() { 39 | Spool a = Spools.newEmptySpool(); 40 | Spool b = Spools.EMPTY; 41 | Spool c = Spools.newSpool(Arrays.asList("a", "b")); 42 | 43 | Assert.assertNotNull(a); 44 | Assert.assertNotNull(b); 45 | Assert.assertNotNull(c); 46 | } 47 | 48 | @Test 49 | public void testSpoolConcat() throws Exception { 50 | Spool a = Spools.newSpool(Arrays.asList("a")); 51 | Spool b = Spools.newSpool(Arrays.asList("b")); 52 | Spool c = Spools.newSpool(Arrays.asList("c")); 53 | 54 | Spool ab = a.concat(b); 55 | Spool abNothing = ab.concat(Spools.newEmptySpool()); 56 | Spool abc = Await.result(ab.concat(Future.value(c))); 57 | 58 | Collection listA = JavaConversions.seqAsJavaList(Await.result(ab.toSeq())); 59 | Collection listB = JavaConversions.seqAsJavaList(Await.result(abNothing.toSeq())); 60 | Collection listC = JavaConversions.seqAsJavaList(Await.result(abc.toSeq())); 61 | 62 | Assert.assertArrayEquals(new String[] { "a", "b"}, listA.toArray()); 63 | Assert.assertArrayEquals(new String[] { "a", "b"}, listB.toArray()); 64 | Assert.assertArrayEquals(new String[] { "a", "b", "c"}, listC.toArray()); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.zk 2 | 3 | import org.apache.zookeeper.KeeperException 4 | 5 | import com.twitter.conversions.time._ 6 | import com.twitter.util.{Duration, Future, Timer} 7 | 8 | /** Pluggable retry strategy. */ 9 | trait RetryPolicy { 10 | def apply[T](op: => Future[T]): Future[T] 11 | } 12 | 13 | /** Matcher for connection-related KeeperExceptions. */ 14 | object KeeperConnectionException { 15 | def unapply(e: KeeperException) = e match { 16 | case e: KeeperException.ConnectionLossException => Some(e) 17 | case e: KeeperException.SessionExpiredException => Some(e) 18 | case e: KeeperException.SessionMovedException => Some(e) 19 | case e: KeeperException.OperationTimeoutException => Some(e) 20 | case e => None 21 | } 22 | } 23 | 24 | object RetryPolicy { 25 | /** Retries an operation a fixed number of times without back-off. */ 26 | case class Basic(retries: Int) extends RetryPolicy { 27 | def apply[T](op: => Future[T]): Future[T] = { 28 | def retry(tries: Int): Future[T] = { 29 | op rescue { case KeeperConnectionException(_) if (tries > 0) => 30 | retry(tries - 1) 31 | } 32 | } 33 | retry(retries) 34 | } 35 | } 36 | 37 | /** 38 | * Retries an operation indefinitely until success, with a delay that increases exponentially. 39 | * 40 | * @param base initial value that is multiplied by factor every time; should be > 0 41 | * @param factor should be >= 1 so the retries do not become more aggressive 42 | */ 43 | case class Exponential( 44 | base: Duration, 45 | factor: Double = 2.0, 46 | maximum: Duration = 30.seconds 47 | )(implicit timer: Timer) extends RetryPolicy { 48 | require(base > 0.seconds) 49 | require(factor >= 1) 50 | 51 | def apply[T](op: => Future[T]): Future[T] = { 52 | def retry(delay: Duration): Future[T] = { 53 | op rescue { case KeeperConnectionException(_) => 54 | timer.doLater(delay) { 55 | retry((delay.inNanoseconds * factor).toLong.nanoseconds min maximum) 56 | }.flatten 57 | } 58 | } 59 | retry(base) 60 | } 61 | } 62 | 63 | /** A single try */ 64 | object None extends RetryPolicy { 65 | def apply[T](op: => Future[T]) = op 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /util-core/src/main/scala/com/twitter/util/LastWriteWinsQueue.scala: -------------------------------------------------------------------------------- 1 | package com.twitter.util 2 | import java.util.Collection 3 | import java.util.concurrent.atomic.AtomicReference 4 | 5 | /** 6 | * An implementation of java.util.Queue that has LIFO order and a maximum capacity of 1 7 | * When the Queue is full, a push replaces the item. 8 | */ 9 | class LastWriteWinsQueue[A] extends java.util.Queue[A] { 10 | val item = new AtomicReference[Option[A]](None) 11 | 12 | def clear() { 13 | item.set(None) 14 | } 15 | 16 | def retainAll(p1: Collection[_]) = throw new UnsupportedOperationException 17 | 18 | def removeAll(p1: Collection[_]) = throw new UnsupportedOperationException 19 | 20 | def addAll(p1: Collection[_ <: A]) = throw new UnsupportedOperationException 21 | 22 | def containsAll(p1: Collection[_]) = 23 | p1.size == 1 && item.get == p1.iterator.next() 24 | 25 | def remove(candidate: AnyRef) = { 26 | val contained = item.get 27 | val containsCandidate = contained.map(_ == candidate).getOrElse(false) 28 | if (containsCandidate) { 29 | item.compareAndSet(contained, None) 30 | } 31 | containsCandidate 32 | } 33 | 34 | def toArray[T](array: Array[T with java.lang.Object]): Array[T with java.lang.Object] = { 35 | val contained = item.get 36 | if (contained.isDefined && array.length > 0) { 37 | array(0) = contained.get.asInstanceOf[T with java.lang.Object] 38 | array 39 | } else if (contained.isDefined) { 40 | Array[Any](contained.get).asInstanceOf[Array[T with java.lang.Object]] 41 | } else Array[Any]().asInstanceOf[Array[T with java.lang.Object]] 42 | } 43 | 44 | def toArray = toArray(new Array[AnyRef](0)) 45 | 46 | def iterator = null 47 | 48 | def contains(p1: AnyRef) = false 49 | 50 | def isEmpty = item.get.isDefined 51 | 52 | def size = if (item.get.isDefined) 1 else 0 53 | 54 | def peek = item.get.getOrElse(null.asInstanceOf[A]) 55 | 56 | def element = item.get.getOrElse(throw new NoSuchElementException) 57 | 58 | def poll = item.getAndSet(None).getOrElse(null.asInstanceOf[A]) 59 | 60 | def remove = item.getAndSet(None).getOrElse(throw new NoSuchElementException) 61 | 62 | def offer(p1: A) = { 63 | item.set(Some(p1)) 64 | true 65 | } 66 | 67 | def add(p1: A) = { 68 | item.set(Some(p1)) 69 | true 70 | } 71 | } 72 | --------------------------------------------------------------------------------