├── project ├── build.properties └── plugins.sbt ├── version.sbt ├── .travis.yml ├── README ├── README.md ├── .gitignore ├── NOTICE ├── src ├── main │ └── scala │ │ └── com │ │ └── metamx │ │ └── common │ │ ├── scala │ │ ├── net │ │ │ ├── curator │ │ │ │ ├── package.scala │ │ │ │ ├── Curator.scala │ │ │ │ ├── CuratorUtils.scala │ │ │ │ ├── Discotheque.scala │ │ │ │ └── Disco.scala │ │ │ ├── finagle │ │ │ │ ├── DiscoResolver.scala │ │ │ │ └── InetAddressResolver.scala │ │ │ └── uri.scala │ │ ├── counters │ │ │ ├── CountersMonitor.scala │ │ │ ├── Counters.scala │ │ │ ├── MapCounters.scala │ │ │ └── NumericCounters.scala │ │ ├── collection │ │ │ ├── implicits.scala │ │ │ ├── concurrent │ │ │ │ ├── PermanentByTypesMap.scala │ │ │ │ ├── atomic.scala │ │ │ │ ├── PermanentMap.scala │ │ │ │ ├── SizeBoundedQueue.scala │ │ │ │ ├── BlockingQueue.scala │ │ │ │ └── ByteBufferQueue.scala │ │ │ ├── mutable.scala │ │ │ └── package.scala │ │ ├── timekeeper.scala │ │ ├── Paths.scala │ │ ├── nio.scala │ │ ├── db │ │ │ ├── DBConfig.scala │ │ │ ├── MySQLDB.scala │ │ │ └── DB.scala │ │ ├── Env.scala │ │ ├── concurrent │ │ │ ├── Implicits.scala │ │ │ ├── locks.scala │ │ │ └── package.scala │ │ ├── process.scala │ │ ├── config.scala │ │ ├── Abort.scala │ │ ├── junit.scala │ │ ├── chaincast.scala │ │ ├── option.scala │ │ ├── iteration.scala │ │ ├── event │ │ │ ├── package.scala │ │ │ ├── Emitting.scala │ │ │ ├── emit.scala │ │ │ ├── AlertAggregator.scala │ │ │ └── Metric.scala │ │ ├── Logging.scala │ │ ├── Algorithms.scala │ │ ├── lifecycle.scala │ │ ├── pretty.scala │ │ ├── gz.scala │ │ ├── LateVal.scala │ │ ├── Yaml.scala │ │ ├── time │ │ │ ├── package.scala │ │ │ └── Intervals.scala │ │ ├── control.scala │ │ ├── threads.scala │ │ ├── Jackson.scala │ │ ├── Predef.scala │ │ ├── Math.scala │ │ ├── Walker.scala │ │ ├── exception.scala │ │ ├── untyped.scala │ │ └── Logger.scala │ │ ├── concurrent │ │ ├── RepeatingLoggingThread.scala │ │ └── RepeatingThread.scala │ │ └── Backoff.scala └── test │ └── scala │ └── com │ └── metamx │ └── common │ └── scala │ ├── LoggerTest.scala │ ├── net │ ├── UriTest.scala │ └── curator │ │ └── DiscothequeTest.scala │ ├── PredefTest.scala │ ├── collection │ ├── concurrent │ │ ├── PermanentMapTest.scala │ │ └── BlockingQueueTest.scala │ ├── MapLikeOpsTest.scala │ └── MutableTest.scala │ ├── ChaincastTest.scala │ ├── LoggingTest.scala │ ├── JacksonTest.scala │ ├── UntypedTest.scala │ ├── GzTest.scala │ ├── PrettyTest.scala │ ├── WalkerTest.scala │ ├── ExceptionTest.scala │ ├── counters │ └── NumericCountersTest.scala │ ├── RetryOnErrorTest.scala │ ├── event │ ├── MetricTest.scala │ └── AlertAggregatorTest.scala │ └── CollectionTest.scala └── LICENSE /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.13 2 | -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | version in ThisBuild := "1.14.2-SNAPSHOT" 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.10.6 5 | - 2.11.8 6 | - 2.12.1 7 | 8 | jdk: 9 | - oraclejdk8 10 | 11 | sudo: false 12 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | com.metamx.common.scala.* Written in scala, best used by scala client code 2 | com.metamx.common.* Written in scala, usable from any jvm language 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | scala-util 2 | ========== 3 | [![Build Status](https://travis-ci.org/metamx/scala-util.svg?branch=master)](https://travis-ci.org/metamx/scala-util) 4 | 5 | Scala stuff -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.ipr 2 | *.iws 3 | *.iml 4 | target 5 | dist 6 | *.jar 7 | *.tar 8 | *.zip 9 | *.iml 10 | *.ipr 11 | *.sublime-project 12 | *.sublime-workspace 13 | .idea 14 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This product contains a modified version of Daniel Lundin's loglady library 2 | * LICENSE: 3 | * https://github.com/dln/loglady/blob/master/LICENSE (Apache License, Version 2.0) 4 | * HOMEPAGE: 5 | * https://github.com/dln/loglady 6 | * COMMIT TAG: 7 | * https://github.com/dln/loglady/commit/cd607af3 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | resolvers ++= Seq( 2 | "Central" at "https://oss.sonatype.org/content/repositories/releases/" 3 | ) 4 | 5 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2") 6 | 7 | addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.4") 8 | 9 | addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") 10 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/curator/package.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net 2 | 3 | import org.apache.curator.x.discovery.{ServiceProvider, ServiceInstance} 4 | 5 | package object curator 6 | { 7 | implicit def ServiceInstanceOps[T](x: ServiceInstance[T]) = new ServiceInstanceOps[T](x) 8 | implicit def ServiceProviderOps[T](x: ServiceProvider[T]) = new ServiceProviderOps[T](x) 9 | } 10 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/LoggerTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.simple.simplespec.Matchers 4 | import org.junit.Test 5 | 6 | class LoggerTest extends Matchers 7 | { 8 | @Test 9 | def testLogVariable() { 10 | val obj = new SimpleExample 11 | obj.getLog must beA[Logger] 12 | } 13 | 14 | } 15 | 16 | class SimpleExample extends Logging { 17 | def getLog = log 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/counters/CountersMonitor.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.counters 2 | 3 | import com.metamx.emitter.service.ServiceEmitter 4 | import com.metamx.metrics.AbstractMonitor 5 | 6 | /** 7 | * Periodically emits deltas based off a Counters object. 8 | */ 9 | class CountersMonitor(counters: Counters) extends AbstractMonitor 10 | { 11 | def doMonitor(emitter: ServiceEmitter) = { 12 | counters.snapshotAndReset() foreach emitter.emit 13 | true 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/net/UriTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net 2 | 3 | import com.metamx.common.scala.net.uri._ 4 | import com.simple.simplespec.Matchers 5 | import org.junit.Test 6 | import scala.collection.immutable.ListMap 7 | 8 | class UriTest extends Matchers 9 | { 10 | 11 | @Test 12 | def testToQueryString() 13 | { 14 | Seq(("a", 2), ("a", 3), ("b", "foo")).toQueryString must be("a=2&a=3&b=foo") 15 | ListMap("a" -> 2, "b" -> "foo").toQueryString must be("a=2&b=foo") 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/counters/Counters.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.counters 2 | 3 | import com.metamx.common.scala.event.Metric 4 | import com.metamx.metrics.Monitor 5 | 6 | /** 7 | * Snapshottable counters. Useful for both streaming metrics (snapshot periodically, emit) and batch 8 | * metrics (snapshot once at the end of a task). 9 | */ 10 | trait Counters 11 | { 12 | def snapshotAndReset(): Counters.Snapshot 13 | 14 | def monitor: Monitor = new CountersMonitor(this) 15 | } 16 | 17 | object Counters 18 | { 19 | type Snapshot = Seq[Metric] 20 | } 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2012 Metamarkets Group Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/implicits.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.collection 2 | 3 | import scala.collection.{MapLike, TraversableLike} 4 | 5 | object implicits 6 | { 7 | implicit def TraversableOnceOps[X, F[Y] <: TraversableOnce[Y]](xs: F[X]) = new TraversableOnceOps[X,F](xs) 8 | implicit def TraversableLikeOps[X, F[Y] <: TraversableLike[Y, F[Y]]](xs: F[X]) = new TraversableLikeOps[X,F](xs) 9 | implicit def IteratorOps[X](xs: Iterator[X]) = new IteratorOps[X](xs) 10 | implicit def MapLikeOps[A, B, Repr <: MapLike[A, B, Repr] with scala.collection.Map[A, B]](m: MapLike[A, B, Repr]) = new MapLikeOps[A, B, Repr](m) 11 | } 12 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/PredefTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.metamx.common.scala.Predef._ 4 | import com.simple.simplespec.Matchers 5 | import org.junit.Test 6 | 7 | class PredefTest extends Matchers { 8 | 9 | @Test def testRequiringTwoArgThrow() { 10 | val retval = try { 11 | 1 requiring(_ > 3, "str") 12 | } catch { 13 | case e: Exception => 0 14 | } 15 | 16 | retval must be(0) 17 | } 18 | 19 | @Test def testRequiringTwoArgNoThrow() { 20 | val retval = try { 21 | 5 requiring(_ > 3, "str") 22 | } catch { 23 | case e: Exception => 0 24 | } 25 | 26 | retval must be(5) 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/collection/concurrent/PermanentMapTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.collection.concurrent 2 | 3 | import com.simple.simplespec.Matchers 4 | import org.junit.Test 5 | 6 | class PermanentMapTest extends Matchers 7 | { 8 | 9 | @Test 10 | def testSimple() 11 | { 12 | // Not testing the concurrency parts, just basic functionality from a single thread's perspective. 13 | val m = new PermanentMap[String, Int] 14 | m.getOrElseUpdate("foo", 3) must be(3) 15 | m.getOrElseUpdate("foo", 5) must be(3) 16 | m.getOrElseUpdate("bar", 5) must be(5) 17 | m.get("bar") must be(Some(5)) 18 | m.get("baz") must be(None) 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/ChaincastTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.metamx.common.scala.chaincast._ 4 | import com.metamx.common.scala.untyped.Dict 5 | import com.simple.simplespec.Matchers 6 | import org.junit.Test 7 | 8 | class ChaincastTest extends Matchers 9 | { 10 | 11 | @Test 12 | def testSimple() { 13 | val thing = Jackson.parse[Dict]("""{"results":[{"k": 1, "k2": 2}, {"k": 2}]}""").chainCast 14 | thing("results").asList.map(kv => kv.asDict.apply("k").asInt) must be(Seq(1, 2)) 15 | thing("results").asList.flatMap(kv => kv.asDict.get("k2").map(_.asInt)) must be(Seq(2)) 16 | thing("results").asList.head.apply("k").asInt must be(1) 17 | thing("results").asList.headOption.map(_.apply("k").asInt) must be(Some(1)) 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/concurrent/RepeatingLoggingThread.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.concurrent 2 | 3 | import com.metamx.common.scala.Logging 4 | import org.joda.time.Duration 5 | 6 | class RepeatingLoggingThread(delay: Duration, name: String, body: => Any) extends Logging 7 | { 8 | val thread = new RepeatingThread(delay, new Runnable { 9 | def run() { 10 | try { 11 | body 12 | } catch { 13 | case e: InterruptedException => 14 | throw e 15 | 16 | case e: Exception => 17 | log.error(e, "[%s] - Exception while performing operation".format(name)) 18 | } 19 | } 20 | }) 21 | thread.setName(name) 22 | 23 | def start() = thread.start() 24 | 25 | def cancel() = thread.cancel() 26 | 27 | def repeatNow() = thread.repeatNow() 28 | } 29 | 30 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/timekeeper.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import org.joda.time.DateTime 4 | 5 | object timekeeper { 6 | /** 7 | * Timekeepers are a more testable alternative to `DateTime.now` or `new DateTime()`. Most applications will use 8 | * SystemTimekeeper in production and TestingTimekeeper in time-sensitive unit tests. 9 | */ 10 | trait Timekeeper 11 | { 12 | def now: DateTime 13 | } 14 | 15 | class SystemTimekeeper extends Timekeeper with Serializable 16 | { 17 | def now = new DateTime() 18 | } 19 | 20 | class TestingTimekeeper extends Timekeeper 21 | { 22 | @volatile private[this] var _now: Option[DateTime] = None 23 | 24 | def now = _now getOrElse { 25 | throw new IllegalStateException("Time not set!") 26 | } 27 | 28 | def now_=(dt: DateTime) { 29 | _now = Some(dt) 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/counters/MapCounters.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.counters 2 | 3 | import com.metamx.common.scala.collection.mutable.ConcurrentMap 4 | import java.util.concurrent.atomic.AtomicLong 5 | import com.metamx.common.scala.event.Metric 6 | 7 | /** 8 | * Use this when too lazy to create a domain-specific Counters class. Uses memory linear in the number of keys 9 | * ever seen. 10 | */ 11 | class MapCounters(prototype: Metric) extends Counters 12 | { 13 | private[this] val counters = ConcurrentMap[String, AtomicLong]() 14 | 15 | def increment(key: String) { 16 | add(key, 1) 17 | } 18 | 19 | def add(key: String, value: Long) { 20 | counters.putIfAbsent(key, new AtomicLong(0)) 21 | counters(key).addAndGet(value) 22 | } 23 | 24 | def snapshotAndReset() = { 25 | (counters.keys map { 26 | k => 27 | Metric(k, counters(k).getAndSet(0)) + prototype 28 | }).toIndexedSeq 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Paths.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | object Paths { 20 | def apply(sep: String) = new { 21 | 22 | class Path(a: String) { 23 | def / (b: String) = a + sep + b 24 | } 25 | implicit def Path(s: String): Path = new Path(s) 26 | 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/nio.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import java.nio.ByteBuffer 20 | 21 | object nio { 22 | 23 | def byteBufferToString(b: ByteBuffer) = new String({ 24 | val bytes = new Array[Byte](b.remaining) 25 | b.get(bytes) 26 | bytes 27 | }) 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/LoggingTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.metamx.common.scala.Predef._ 4 | import com.simple.simplespec.Matchers 5 | import java.io.{ObjectInputStream, ByteArrayInputStream, ByteArrayOutputStream, ObjectOutputStream} 6 | import org.junit.{Ignore, Test} 7 | 8 | @Ignore 9 | class SerializableLogging(val n: Int) extends Logging with Serializable 10 | 11 | class LoggingTest extends Matchers 12 | { 13 | 14 | @Test 15 | def testJavaSerialization() 16 | { 17 | val x = new SerializableLogging(1) 18 | x.log.trace("Hello!") 19 | 20 | val xBytes = (new ByteArrayOutputStream() withEffect { 21 | baos => 22 | new ObjectOutputStream(baos).writeObject(x) 23 | }).toByteArray 24 | 25 | val y = new ObjectInputStream(new ByteArrayInputStream(xBytes)).readObject().asInstanceOf[SerializableLogging] 26 | y.log.trace("World!") 27 | 28 | x.n must be(y.n) 29 | x eq y must be(false) 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/db/DBConfig.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.db 18 | 19 | import org.joda.time.Duration 20 | 21 | trait DBConfig { 22 | def uri: String 23 | def user: String 24 | def password: String 25 | def queryTimeout: Duration 26 | def batchSize: Int 27 | def fetchSize: Int 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Env.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import scala.collection.JavaConverters._ 20 | 21 | object Env { 22 | val env: _root_.scala.collection.Map[String,String] = System.getenv.asScala 23 | def apply (k: String) = env(k) 24 | def apply (k: String, v: String) = env.getOrElse(k,v) 25 | def get (k: String) = env.get(k) 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/concurrent/Implicits.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.concurrent 18 | 19 | import java.util.concurrent.Callable 20 | 21 | object Implicits { 22 | 23 | implicit def functionToRunnable[X](f: () => X): Runnable = 24 | new Runnable { def run = f() } 25 | 26 | implicit def functionToCallable[X](f: () => X): Callable[X] = 27 | new Callable[X] { def call = f() } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/concurrent/locks.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.concurrent 18 | 19 | import java.util.concurrent.locks.Lock 20 | 21 | object locks { 22 | 23 | class LockOps(l: Lock) { 24 | def apply[X](x: => X) = { 25 | l.lock 26 | try x finally { 27 | l.unlock 28 | } 29 | } 30 | } 31 | 32 | implicit def LockOps(l: Lock): LockOps = new LockOps(l) 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/process.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import scala.collection.mutable.ListBuffer 4 | import scala.sys.process.{Process, ProcessLogger} 5 | 6 | object process extends Logging { 7 | 8 | def backtick(cmd: String*): Seq[String] = { 9 | log.trace("Running: %s" format (cmd mkString " ")) 10 | val buf = ListBuffer[String]() 11 | val status = Process(cmd) ! 12 | ProcessLogger(out => buf += "%s" format out, err => log.warn("%s: %s", cmd mkString " ", err)) 13 | if (status != 0) { 14 | throw new ProcessFailureException("Command failed: %s (status = %d)" format (cmd mkString " ", status)) 15 | } else { 16 | buf.toSeq 17 | } 18 | } 19 | 20 | def system(cmd: String*) { 21 | log.trace("Running: %s" format (cmd mkString " ")) 22 | val status = Process(cmd) ! ProcessLogger(line => log.warn("%s: %s", cmd mkString " ", line)) 23 | if (status != 0) { 24 | throw new ProcessFailureException("Command failed: %s (status = %d)" format (cmd mkString " ", status)) 25 | } 26 | } 27 | 28 | class ProcessFailureException(msg: String) extends Exception(msg) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/config.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import org.skife.config.ConfigurationObjectFactory 20 | import scala.reflect.runtime.universe.TypeTag 21 | 22 | object config { 23 | 24 | class ConfigOps(configs: ConfigurationObjectFactory) { 25 | implicit def apply[X](implicit tag: TypeTag[X]): X = configs.build(tag.mirror.runtimeClass(tag.tpe).asInstanceOf[Class[X]]) 26 | } 27 | implicit def ConfigOps(configs: ConfigurationObjectFactory) = new ConfigOps(configs) 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Abort.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | object Abort extends Logging { 20 | 21 | def apply(e: Throwable): Nothing = { 22 | log.error("Aborting: " + e) 23 | e.printStackTrace 24 | Runtime.getRuntime.halt(1) // (Avoid System.exit hangs) 25 | throw new Exception("Unreachable") 26 | } 27 | 28 | def apply(msg: String): Nothing = { 29 | log.error("Aborting: " + msg) 30 | Runtime.getRuntime.halt(1) // (Avoid System.exit hangs) 31 | throw new Exception("Unreachable") 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/JacksonTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.google.common.collect.ImmutableList 4 | import com.metamx.common.scala.untyped.Dict 5 | import com.simple.simplespec.Matchers 6 | import org.junit.Test 7 | import scala.collection.JavaConverters._ 8 | 9 | class JacksonTest extends Matchers 10 | { 11 | @Test 12 | def testSimple() 13 | { 14 | val json = """{"hey":"what"}""" 15 | Jackson.parse[Dict](json) must be(Map("hey" -> "what")) 16 | Jackson.parse[Dict](json.getBytes) must be(Map("hey" -> "what")) 17 | Jackson.parse[java.util.Map[String, AnyRef]](json) must be(Map[String, AnyRef]("hey" -> "what").asJava) 18 | Jackson.generate(Jackson.parse[AnyRef](json)) must be(json) 19 | Jackson.bytes(Jackson.parse[AnyRef](json)) must be(json.getBytes) 20 | Jackson.normalize[Dict]( 21 | Map( 22 | "hey" -> ImmutableList.of("what"), 23 | "foo" -> None 24 | ) 25 | ) must be( 26 | Dict( 27 | "hey" -> Seq("what"), 28 | "foo" -> null 29 | ) 30 | ) 31 | } 32 | 33 | @Test 34 | def testNulls() 35 | { 36 | val json = """{"hey":[null]}""" 37 | Jackson.parse[Dict](json) must be(Map("hey" -> Seq(null))) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/junit.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.exception._ 20 | import com.metamx.common.scala.Predef._ 21 | 22 | object junit { 23 | 24 | // Dump context `xs' to stdout if any test (i.e. assertion) in `body' fails 25 | def inContext[X](xs: Any*)(body: => X): X = body mapException { 26 | case e: AssertionError => e withEffect { _ => 27 | println("In context: %s" format (xs mkString ", ")) 28 | } 29 | } 30 | 31 | def contextually[X,Y](f: X => Y): X => Y = { 32 | x => inContext(x) { f(x) } 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/chaincast.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.metamx.common.scala.untyped._ 4 | 5 | /** 6 | * Works with "untyped" to make it easier to extract specific things from nested, untyped structures. 7 | * 8 | * Somewhat experimental API; if this proves useful, we'll keep it. 9 | */ 10 | object chaincast 11 | { 12 | implicit def chainCast[A](o: A) = new StartChainable(o) 13 | 14 | class StartChainable[A](o: A) 15 | { 16 | def chainCast = new Chainable(o) 17 | } 18 | 19 | class Chainable(o: Any) 20 | { 21 | def apply(s: String): Chainable = new Chainable(dict(o).apply(s)) 22 | 23 | def get(s: String): Option[Chainable] = dict(o).get(s).map(new Chainable(_)) 24 | 25 | def getOrElse[B](s: String, default: => B): Chainable = new Chainable(get(s).getOrElse(default)) 26 | 27 | def asList: Seq[Chainable] = list(o).map(new Chainable(_)) 28 | 29 | def asDict: Map[String, Chainable] = dict(o).map(kv => (kv._1, new Chainable(kv._2))) 30 | 31 | def asString = str(o) 32 | 33 | def asInt = int(o) 34 | 35 | def asLong = long(o) 36 | 37 | def asBool = bool(o) 38 | 39 | def asDouble = double(o) 40 | 41 | def asFloat = float(o) 42 | 43 | def unwrap = o 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/collection/MapLikeOpsTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Metamarkets Group Inc. (Metamarkets) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. Metamarkets licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package com.metamx.common.scala.collection 21 | 22 | import com.simple.simplespec.Matchers 23 | import org.junit.Test 24 | 25 | class MapLikeOpsTest extends Matchers 26 | { 27 | @Test def extractPrefixed() { 28 | val m = Map("foo" -> 1, "bar" -> 2, "foo.baz" -> 3) 29 | val res = m.extractPrefixed("foo") 30 | res must be(Map("baz" -> 3)) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/option.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | object option { 20 | 21 | class OptionOps[X](x: Option[X]) { 22 | 23 | def ifEmpty (f: => Any): Option[X] = { if (x.isEmpty) f; x } // Like TraversableOnceOps 24 | def ifNonEmpty (f: => Any): Option[X] = { if (x.nonEmpty) f; x } // Like TraversableOnceOps 25 | def ifDefined (f: X => Any): Option[X] = { if (x.isDefined) f(x.get); x } // Not in TraversableOnceOps 26 | 27 | def andThen[Y](f: X => Option[Y]): Option[Y] = x flatMap f 28 | 29 | } 30 | implicit def OptionOps[X](x: Option[X]): OptionOps[X] = new OptionOps(x) 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/curator/Curator.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net.curator 2 | 3 | import com.metamx.common.lifecycle.Lifecycle 4 | import com.metamx.common.lifecycle.Lifecycle.Handler 5 | import org.joda.time.Duration 6 | import org.apache.curator.framework.{CuratorFrameworkFactory, CuratorFramework} 7 | import org.apache.curator.retry.ExponentialBackoffRetry 8 | 9 | object Curator 10 | { 11 | def create(zkConnect: String, zkTimeout: Duration, lifecycle: Lifecycle): CuratorFramework = { 12 | val curator = CuratorFrameworkFactory 13 | .builder() 14 | .connectString(zkConnect) 15 | .sessionTimeoutMs(zkTimeout.getMillis.toInt) 16 | .retryPolicy(new ExponentialBackoffRetry(1000, 30)) 17 | .build() 18 | 19 | // Allow to create curator even if lifecycle is already created 20 | lifecycle.addMaybeStartHandler( 21 | new Handler 22 | { 23 | def start() { 24 | curator.start() 25 | } 26 | 27 | def stop() { 28 | curator.close() 29 | } 30 | } 31 | ) 32 | 33 | curator 34 | } 35 | 36 | def create(config: CuratorConfig, lifecycle: Lifecycle): CuratorFramework = { 37 | create(config.zkConnect, config.zkTimeout, lifecycle) 38 | } 39 | } 40 | 41 | trait CuratorConfig 42 | { 43 | def zkConnect: String 44 | 45 | def zkTimeout: Duration 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/iteration.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | object iteration { 20 | 21 | implicit def toTimes(n: Int) = new { 22 | def times(f: => Any) { 23 | var i = n 24 | while (i > 0) { 25 | f 26 | i -= 1 27 | } 28 | } 29 | } 30 | 31 | class IterablePairOps[K,V](self: Iterable[(K,V)]) { 32 | def toManyMap: Map[K, Iterable[V]] = 33 | self groupBy (_._1) mapValues (_.map(_._2)) 34 | } 35 | implicit def IterablePairOps[K,V](xs: Iterable[(K,V)]) = new IterablePairOps(xs) 36 | 37 | // Need higher kinds to do this right 38 | // class IterableLikeOps[X, Y, Repr](xs: IterableLike[(X,Y), Repr]) { 39 | // def toManyMap: Map[X, Repr[Y]] = ... 40 | // } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/event/package.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.metamx.emitter.service.AlertEvent.Severity._ 4 | import com.metamx.emitter.service._ 5 | import com.metamx.emitter.core.Event 6 | import scala.collection.JavaConverters._ 7 | 8 | package object event 9 | { 10 | val WARN = ANOMALY 11 | val ERROR = COMPONENT_FAILURE 12 | 13 | class ServiceMetricEventOps(e: ServiceMetricEvent) extends ServiceEventOps(e) 14 | { 15 | def userDims = e.getUserDims.asScala 16 | def metric = e.getMetric 17 | def value = e.getValue 18 | } 19 | implicit def ServiceMetricEventOps(e: ServiceMetricEvent) = new ServiceMetricEventOps(e) 20 | 21 | class AlertEventOps(e: AlertEvent) extends ServiceEventOps(e) 22 | { 23 | def severity = e.getSeverity 24 | def description = e.getDescription 25 | def dataMap = e.getDataMap.asScala 26 | } 27 | implicit def AlertEventOps(e: AlertEvent) = new AlertEventOps(e) 28 | 29 | class ServiceEventOps(e: ServiceEvent) extends EventOps(e) 30 | { 31 | def host = e.getHost 32 | def service = e.getService 33 | } 34 | implicit def ServiceEventOps(e: ServiceEvent) = new ServiceEventOps(e) 35 | 36 | class EventOps(e: Event) 37 | { 38 | def feed = e.getFeed 39 | def createdTime = e.getCreatedTime 40 | def safeToBuffer = e.isSafeToBuffer 41 | } 42 | implicit def EventOps(e: Event) = new EventOps(e) 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Logging.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | trait Logging { 20 | // Initialize boolean variable by default value to retain the same behaviour for deserialized class 21 | @transient @volatile private var initialized: Boolean = _ 22 | @transient private var logger: Logger = _ 23 | 24 | // We emulate behaviour of lazy val because scala 2.12.1 has bug with transient lazy val 25 | // https://issues.scala-lang.org/browse/SI-10244 26 | def log: Logger = { 27 | if (initialized) { 28 | logger 29 | } else { 30 | compute 31 | } 32 | } 33 | 34 | private def compute: Logger = { 35 | synchronized { 36 | if (!initialized) { 37 | logger = Logger(getClass) 38 | initialized = true 39 | } 40 | logger 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Algorithms.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | object Algorithms { 20 | 21 | // From com.metamx.druid.utils.BufferUtils.binarySearch translated to Scala 22 | def binarySearch[A](seq: IndexedSeq[A], minIndex: Int, maxIndex: Int, value: A)(implicit ordering: Ordering[A]): Int = { 23 | var _minIndex = minIndex 24 | var _maxIndex = maxIndex 25 | while (_minIndex < _maxIndex) { 26 | val currIndex = (_minIndex + _maxIndex - 1) >>> 1 27 | val currValue = seq(currIndex) 28 | val comparison = ordering.compare(currValue, value) 29 | if (comparison == 0) { 30 | return currIndex 31 | } 32 | if (comparison < 0) { 33 | _minIndex = currIndex + 1 34 | } else { 35 | _maxIndex = currIndex 36 | } 37 | } 38 | return -(_minIndex + 1) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/Backoff.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common 18 | 19 | import _root_.scala.util.Random 20 | import java.util.concurrent.atomic.AtomicLong 21 | 22 | class Backoff(start: Long, growth: Double, max: Long, fuzz: Double) 23 | { 24 | def this(start: Long, growth: Double, max: Long) = this(start, growth, max, .2) // (Java doesn't speak default args) 25 | 26 | private[this] val _next: AtomicLong = new AtomicLong(fuzzy(start)) 27 | 28 | def next = _next.get() 29 | 30 | def incr() { 31 | _next.set(fuzzy(math.min(max, (_next.get() * growth).toLong))) 32 | } 33 | 34 | def sleep() { 35 | Thread.sleep(next) 36 | incr() 37 | } 38 | 39 | def reset() { 40 | _next.set(start) 41 | } 42 | 43 | def fuzzy(x: Long): Long = (math.max(1 + fuzz * Random.nextGaussian, 0) * x).toLong 44 | 45 | } 46 | 47 | object Backoff 48 | { 49 | def standard() = new Backoff(200, 2, 30000) 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/lifecycle.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.lifecycle.Lifecycle 20 | import com.metamx.common.lifecycle.Lifecycle.Handler 21 | 22 | object lifecycle { 23 | 24 | class LifecycleOps(lifecycle: Lifecycle) { 25 | def apply[X](x: X): X = lifecycle.addManagedInstance(x) 26 | 27 | def onStart(f: => Any) = { 28 | lifecycle.addHandler( 29 | new Handler { 30 | def start() { 31 | f 32 | } 33 | 34 | def stop() {} 35 | } 36 | ) 37 | lifecycle 38 | } 39 | 40 | def onStop(f: => Any) = { 41 | lifecycle.addHandler( 42 | new Handler { 43 | def start() {} 44 | 45 | def stop() { 46 | f 47 | } 48 | } 49 | ) 50 | lifecycle 51 | } 52 | } 53 | implicit def LifecycleOps(x: Lifecycle) = new LifecycleOps(x) 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/pretty.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | object pretty { 20 | 21 | // Truncate a string with "...". Useful for stuffing potentially huge .toString's into log lines. 22 | def truncate(s: String, n: Int = 500): String = if (s.length > n) s.take(n) + "..." else s 23 | 24 | // e.g. parseBytes("5 GB") == 5L * 1024*1024*1024 25 | def parseBytes(s: String): Long = ("""\s*(\d+)\s*([A-Z]+)?\s*""".r.unapplySeq(s) : @unchecked) match { 26 | case None => throw new IllegalArgumentException("Can't parse bytes: %s" format s) 27 | case Some(List(n, null)) => parseBytes(s + "B") 28 | case Some(List(n, suf)) => Seq("B","KB","MB","GB","TB","PB","EB","ZB","YB","BB").indexOf(suf) match { 29 | case -1 => throw new IllegalArgumentException("Unknown bytes suffix %s in: %s" format (suf, s)) 30 | case exp => n.toLong * math.pow(1024L, exp).toLong 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/concurrent/RepeatingThread.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.concurrent 18 | 19 | import com.github.nscala_time.time.Imports._ 20 | import com.metamx.common.scala.Logging 21 | 22 | class RepeatingThread(delay: Duration, runnable: Runnable) 23 | extends Thread with Logging { 24 | 25 | setDaemon(true) 26 | 27 | @volatile 28 | private var cancelled = false 29 | 30 | override def run() { 31 | try { 32 | while (!cancelled) { 33 | runnable.run() 34 | try { 35 | Thread.sleep(delay.millis) 36 | } catch { 37 | case _: InterruptedException => 38 | // Continue 39 | } 40 | } 41 | } catch { 42 | case e: Throwable => 43 | log.error(e, "Killed by exception") 44 | } 45 | } 46 | 47 | def repeatNow() { 48 | interrupt() 49 | } 50 | 51 | def cancel() { 52 | cancelled = true 53 | interrupt() 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/concurrent/PermanentByTypesMap.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection.concurrent 18 | 19 | import com.metamx.common.scala.Predef._ 20 | import com.metamx.common.scala.collection.mutable.ConcurrentMap 21 | 22 | class PermanentByTypesMap[K, T, V] 23 | { 24 | private val lock = new AnyRef 25 | private val valuesByKey = ConcurrentMap[K, V]() 26 | private val valuesByType = ConcurrentMap[T, V]() 27 | 28 | def apply(key: K, typeByKeyFn: K => T, makeFn: T => V): V = { 29 | valuesByKey.get(key) match { 30 | case Some(x) => x 31 | case None => lock.synchronized { 32 | valuesByKey.get(key) match { 33 | case Some(x) => x 34 | case None => 35 | val typ = typeByKeyFn(key) 36 | valuesByType.getOrElseUpdate(typ, makeFn(typ)) withEffect { 37 | value => valuesByKey.put(key, value) 38 | } 39 | } 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/concurrent/atomic.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection.concurrent 18 | 19 | import scala.collection.mutable 20 | import scala.collection.mutable.ArrayBuffer 21 | import com.metamx.common.scala.Predef._ 22 | 23 | @deprecated("Consider java.util.concurrent.* as an alternative.", "1.13.0") 24 | object atomic 25 | { 26 | @deprecated("SynchronizedMap is deprecated since scala 2.11.0. Consider java.util.concurrent.ConcurrentHashMap as an alternative.", "1.13.0") 27 | class AtomicMap[A, B] extends mutable.HashMap[A, B] with mutable.SynchronizedMap[A, B] 28 | 29 | @deprecated("SynchronizedBuffer are deprecated since scala 2.11.0. Consider java.util.concurrent.ConcurrentLinkedQueue as an alternative.", "1.13.0") 30 | class AtomicBuffer[A] extends ArrayBuffer[A] with mutable.SynchronizedBuffer[A] 31 | { 32 | def drain() = synchronized { 33 | toList withEffect { 34 | _ => clear() 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/collection/MutableTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Metamarkets Group Inc. (Metamarkets) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. Metamarkets licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package com.metamx.common.scala.collection 21 | 22 | import com.metamx.common.scala.collection.mutable._ 23 | import com.simple.simplespec.Matchers 24 | import org.junit.Test 25 | 26 | class MutableTest extends Matchers 27 | { 28 | 29 | @Test def concurrentMap { 30 | val m = ConcurrentMap("foo" -> 3) 31 | m.putIfAbsent("foo", 4) 32 | m.putIfAbsent("bar", 4) 33 | m.toMap must be(Map("foo" -> 3, "bar" -> 4)) 34 | } 35 | 36 | @Test def multiMap { 37 | val m = MultiMap("foo" -> 3, "foo" -> 4, "bar" -> 5) 38 | m.entryExists("foo", _ == 3) must be(true) 39 | m.entryExists("foo", _ == 4) must be(true) 40 | m.entryExists("bar", _ == 5) must be(true) 41 | m.entryExists("bar", _ == 6) must be(false) 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/mutable.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection 18 | 19 | import com.metamx.common.scala.Predef.EffectOps 20 | import java.util.concurrent.ConcurrentHashMap 21 | import scala.collection.JavaConverters._ 22 | import scala.collection.mutable.HashMap 23 | import scala.collection.{mutable => _mutable} 24 | 25 | object mutable { 26 | 27 | // A scala-friendly way to create a ConcurrentMap backed by a juc.ConcurrentHashMap 28 | object ConcurrentMap { 29 | def apply[K,V](xs: (K,V)*): ConcurrentMap[K,V] = new ConcurrentHashMap().asScala withEffect { 30 | _ ++= xs 31 | } 32 | } 33 | type ConcurrentMap[K,V] = scala.collection.concurrent.Map[K,V] 34 | 35 | // A more friendly way to create a MultiMap 36 | object MultiMap { 37 | def apply[K,V](xs: (K,V)*): MultiMap[K,V] = new HashMap[K, _mutable.Set[V]] with MultiMap[K,V] withEffect { m => 38 | xs foreach { case (k,v) => m.addBinding(k,v) } 39 | } 40 | } 41 | type MultiMap[K,V] = _mutable.MultiMap[K,V] 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/gz.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.google.common.io.ByteStreams 20 | import com.metamx.common.scala.Predef.EffectOps 21 | import java.io.ByteArrayInputStream 22 | import java.io.ByteArrayOutputStream 23 | import java.util.zip.GZIPInputStream 24 | import java.util.zip.GZIPOutputStream 25 | 26 | object gz { 27 | 28 | def gzip(bytes: Array[Byte]): Array[Byte] = { 29 | new ByteArrayOutputStream withEffect { out => 30 | new GZIPOutputStream(out) withEffect { gz => 31 | gz.write(bytes) 32 | gz.close 33 | } 34 | } toByteArray 35 | } 36 | 37 | def gunzip(bytes: Array[Byte]): Array[Byte] = { 38 | ByteStreams.toByteArray(new GZIPInputStream(new ByteArrayInputStream(bytes))) 39 | } 40 | 41 | def gunzipIfNecessary(bytes: Array[Byte]): Array[Byte] = { 42 | if (isGzip(bytes)) gunzip(bytes) else bytes 43 | } 44 | 45 | def isGzip(bytes: Array[Byte]): Boolean = { 46 | bytes.length >= 2 && 47 | bytes(0) == (GZIPInputStream.GZIP_MAGIC >> 0).toByte && 48 | bytes(1) == (GZIPInputStream.GZIP_MAGIC >> 8).toByte 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/curator/CuratorUtils.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net.curator 2 | 3 | import com.metamx.common.scala.Logging 4 | import org.apache.curator.framework.CuratorFramework 5 | import org.apache.zookeeper.CreateMode 6 | import org.apache.zookeeper.KeeperException.NodeExistsException 7 | 8 | object CuratorUtils extends Logging 9 | { 10 | /** 11 | * Creates or updates data in the given path. This operation is not atomic, so you need proper synchronization 12 | * if multiple clients are going to modify the same path. 13 | */ 14 | def createOrUpdate( 15 | curator: CuratorFramework, 16 | path: String, 17 | create: => Array[Byte], 18 | update: (Array[Byte]) => Array[Byte] 19 | ) { 20 | if (curator.checkExists().forPath(path) == null) { 21 | curator.create().forPath(path, create) 22 | } else { 23 | val data = curator.getData.forPath(path) 24 | curator.setData().forPath(path, update(data)) 25 | } 26 | } 27 | 28 | /** 29 | * Recursively creates path if it doesn't exist. This operation is atomic if all clients trying to 30 | * create the same path use identical mode, otherwise path might be created with different mode 31 | * than asked by client. 32 | */ 33 | def createRecursiveIfNotExists( 34 | curator: CuratorFramework, 35 | path: String, 36 | createMode: CreateMode = CreateMode.PERSISTENT 37 | ) { 38 | try { 39 | if (curator.checkExists().forPath(path) == null) { 40 | val builder = curator 41 | .create() 42 | .creatingParentsIfNeeded() 43 | .withMode(createMode) 44 | .forPath(path) 45 | } 46 | } catch { 47 | case e: NodeExistsException => log.info("Concurrent path creation: %s".format(path)) 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/concurrent/PermanentMap.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection.concurrent 18 | 19 | import com.metamx.common.scala.collection.mutable.ConcurrentMap 20 | 21 | /** 22 | * PermanentMaps are like ConcurrentMaps that only support get and getOrElseUpdate, and for which getOrElseUpdate is 23 | * guaranteed to execute the "update" function at most once, even if called concurrently from multiple threads. 24 | */ 25 | class PermanentMap[K, V] 26 | { 27 | private val keyLocks = ConcurrentMap[K, AnyRef]() // Want fine-grained locking, per key. 28 | private val backingMap = ConcurrentMap[K, V]() 29 | 30 | def apply(key: K, makeFn: () => V): V = getOrElseUpdate(key, makeFn()) 31 | 32 | def get(key: K): Option[V] = backingMap.get(key) 33 | 34 | def getOrElseUpdate(key: K, op: => V): V = { 35 | backingMap.get(key) match { 36 | case Some(x) => x 37 | case None => 38 | keyLocks.putIfAbsent(key, new AnyRef) // Atomic 39 | keyLocks(key).synchronized { 40 | backingMap.getOrElseUpdate(key, op) // Non-atomic, needs to be synchronized 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/counters/NumericCounters.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.counters 2 | 3 | import com.metamx.common.scala.counters.Counters.Snapshot 4 | import com.metamx.common.scala.event.Metric 5 | import scala.collection.mutable 6 | 7 | /** 8 | * Use this when too lazy to create a domain-specific Counters class. 9 | * Aggregates values per each metric & dimensions pair. 10 | */ 11 | class NumericCounters[A](converter: (Numeric[A], A) => Number, feed: String = null)(implicit numeric: Numeric[A]) extends Counters 12 | { 13 | private[this] val lock = new AnyRef 14 | private[this] val map = mutable.Map[(String, Map[String, Iterable[String]]), A]() 15 | 16 | def inc(metric: String, dims: Map[String, Iterable[String]]) { 17 | add(metric, dims, numeric.one) 18 | } 19 | 20 | def add(metric: String, dims: Map[String, Iterable[String]], value: A) { 21 | lock.synchronized { 22 | val key = metric -> dims 23 | val curr = map.get(key) match { 24 | case Some(x) => x 25 | case None => numeric.zero 26 | } 27 | val updated = numeric.plus(curr, value) 28 | map.put(key, updated) 29 | } 30 | } 31 | 32 | def del(metric: String, dims: Map[String, Iterable[String]]) { 33 | lock.synchronized { 34 | val key = metric -> dims 35 | map.remove(key) 36 | } 37 | } 38 | 39 | override def snapshotAndReset(): Snapshot = { 40 | lock.synchronized { 41 | val snapshot = map.map { 42 | case ((metric, dims), value) => 43 | new Metric( 44 | metric = metric, 45 | value = converter(numeric, value), 46 | userDims = dims, 47 | created = null, 48 | feed 49 | ) 50 | }.toList 51 | map.clear() 52 | snapshot 53 | } 54 | } 55 | } 56 | 57 | class LongCounters(feed: String = null) extends NumericCounters[Long](converter = (numeric, x) => numeric.toLong(x), feed) 58 | 59 | class DoubleCounters(feed: String = null) extends NumericCounters[Double](converter = (numeric, x) => numeric.toDouble(x), feed) 60 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/event/Emitting.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.event 2 | 3 | import com.metamx.common.scala.Logging 4 | import com.metamx.common.scala.untyped.Dict 5 | import Emitting._ 6 | import com.metamx.common.scala.LateVal.LateVal 7 | import com.metamx.emitter.EmittingLogger 8 | 9 | // TODO Implicits or cake to convert these runtime exceptions into compile-time errors? 10 | 11 | @deprecated("Setting Emitting.emitter is annoying.", "Sometime") 12 | trait Emitting extends Logging 13 | { 14 | val WARN = com.metamx.common.scala.event.WARN 15 | val ERROR = com.metamx.common.scala.event.ERROR 16 | 17 | Emitting.requireEmitter() 18 | 19 | def emitter: ServiceEmitter = Emitting.emitter 20 | 21 | def emitAlert(e: Throwable, severity: Severity, description: String, data: Dict) 22 | { 23 | emit.emitAlert(e, log, emitter, severity, description, data) 24 | } 25 | 26 | def emitAlert(severity: Severity, description: String, data: Dict) 27 | { 28 | emit.emitAlert(null, log, emitter, severity, description, data) 29 | } 30 | } 31 | 32 | object Emitting 33 | { 34 | type ServiceEmitter = com.metamx.emitter.service.ServiceEmitter 35 | type Severity = com.metamx.emitter.service.AlertEvent.Severity 36 | 37 | private[this] val _emitter = new LateVal[ServiceEmitter] 38 | 39 | // Same as `emitter`, but emphasizes that an exception will be thrown if the emitter is not found 40 | @deprecated("Setting Emitting.emitter is annoying.", "Sometime") 41 | def requireEmitter(): ServiceEmitter = _emitter.derefOption getOrElse { 42 | throw new IllegalStateException("Emitter not set! (Try Emitting.emitter = ...)") 43 | } 44 | 45 | // Same as `requireEmitter()`, but less verbose and emphasizes the return value over the exception behavior 46 | @deprecated("Setting Emitting.emitter is annoying.", "Sometime") 47 | def emitter: ServiceEmitter = requireEmitter() 48 | 49 | @deprecated("Setting Emitting.emitter is annoying.", "Sometime") 50 | def emitter_=(_emitter: ServiceEmitter) { 51 | this._emitter.assignIfEmpty(_emitter) 52 | EmittingLogger.registerEmitter(_emitter) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/concurrent/SizeBoundedQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection.concurrent 18 | 19 | import java.util.concurrent.atomic.AtomicLong 20 | import java.{util => ju} 21 | 22 | class SizeBoundedQueue[E](sizeF: E => Long, size: Long, queue: ju.Queue[E]) extends BlockingQueue[E] 23 | { 24 | // Used space size 25 | private val _used = new AtomicLong 26 | 27 | // Returns used space in bytes 28 | def used() = _used.get() 29 | 30 | protected def enqueue(elem: E): Boolean = { 31 | if (elem == null) { 32 | throw new NullPointerException("Can't put null element") 33 | } 34 | 35 | val elemTotalLength = sizeF(elem) 36 | if (size < elemTotalLength) { 37 | // There is no way to get this element ever appended so let's fail fast 38 | throw new IllegalStateException("Element too big to enqueue") 39 | } 40 | 41 | if (size - _used.get() < elemTotalLength) { 42 | false 43 | } else { 44 | 45 | queue.add(elem) 46 | 47 | _used.addAndGet(elemTotalLength) 48 | _count.incrementAndGet() 49 | 50 | true 51 | } 52 | } 53 | 54 | protected def dequeue(): Option[E] = { 55 | if (_used.get() == 0) { 56 | None 57 | } else { 58 | 59 | val elem = queue.remove() 60 | 61 | val elemTotalLength = sizeF(elem) 62 | 63 | _used.addAndGet(-elemTotalLength) 64 | _count.decrementAndGet() 65 | 66 | Some(elem) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/UntypedTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.Predef._ 20 | import com.metamx.common.scala.untyped._ 21 | import com.simple.simplespec.Matchers 22 | import java.{util => ju} 23 | import org.junit.Test 24 | import scala.collection.JavaConverters._ 25 | 26 | class UntypedTest extends Matchers { 27 | 28 | def jList[X] (xs: X*) = new ju.ArrayList[X] withEffect { _.asScala ++= xs } 29 | def jMap[K,V] (xs: (K,V)*) = new ju.HashMap[K,V] withEffect { _.asScala ++= xs } 30 | 31 | val obj = new { val x = 0 } 32 | 33 | // These need to be by-name (def) instead of by-value (val) because of Iterator, which isn't pure 34 | def messy = jList(jMap("a" -> Seq(Map("b" -> Iterable(Iterator(Array(1, "c", false, (2,true), obj))))))) 35 | def norm = List(Map("a" -> List(Map("b" -> List(List(List(1, "c", false, (2,true), obj))))))) 36 | def jnorm = jList(jMap("a" -> jList(jMap("b" -> jList(jList(jList(1, "c", false, (2,true), obj))))))) 37 | 38 | @Test def normalizing { 39 | normalize(messy) must be(norm) 40 | } 41 | 42 | @Test def normalizingJava { 43 | normalizeJava(messy).toString must be(jnorm.toString) 44 | } 45 | 46 | @Test def normalizingRepeatedly { 47 | normalize(normalize(messy)) must be(normalize(messy)) 48 | normalize(normalizeJava(messy)) must be(normalize(messy)) 49 | normalizeJava(normalize(messy)).toString must be(normalizeJava(messy).toString) 50 | normalizeJava(normalizeJava(messy)).toString must be(normalizeJava(messy).toString) 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/GzTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.gz._ 20 | import com.simple.simplespec.Matchers 21 | import java.io.IOException 22 | import org.junit.Test 23 | import scala.util.Random 24 | 25 | class GzTest extends Matchers { 26 | 27 | def str(bytes: Array[Byte]) = new String(bytes) 28 | 29 | @Test def empty { 30 | 31 | isGzip(Array()) must be(false) 32 | isGzip(gzip(Array())) must be(true) 33 | 34 | str(gunzip(gzip(Array()))) must be("") 35 | evaluating { gunzip(Array()) } must throwAn[IOException] 36 | 37 | str(gunzipIfNecessary(Array())) must be("") 38 | 39 | } 40 | 41 | @Test def nonEmpty { 42 | val bytes = Stream.fill(1024) { Random.nextPrintableChar }.mkString("").getBytes 43 | 44 | isGzip(bytes) must be(false) 45 | isGzip(gzip(bytes)) must be(true) 46 | isGzip(gunzip(gzip(bytes))) must be(false) 47 | evaluating { gunzip(bytes) } must throwAn[IOException] 48 | 49 | str(gunzip(gzip(bytes))) must be(str(bytes)) 50 | 51 | str(gunzipIfNecessary(bytes)) must be(str(bytes)) 52 | str(gunzipIfNecessary(gzip(bytes))) must be(str(bytes)) 53 | str(gunzipIfNecessary(gunzipIfNecessary(bytes))) must be(str(bytes)) 54 | 55 | } 56 | 57 | @Test def doubleZip { 58 | val bytes = Stream.fill(1024) { Random.nextPrintableChar }.mkString("").getBytes 59 | 60 | str(gunzip(gunzip(gzip(gzip(bytes))))) must be(str(bytes)) 61 | str(gunzip(gzip(gzip(bytes)))) must be(str(gzip(bytes))) 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/PrettyTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.simple.simplespec.Matchers 20 | import org.junit.Test 21 | 22 | class PrettyTest extends Matchers { 23 | 24 | @Test def parseBytes { 25 | 26 | pretty.parseBytes("0") must be(0) 27 | pretty.parseBytes("0") must be(0) 28 | pretty.parseBytes("0KB") must be(0) 29 | pretty.parseBytes("0MB") must be(0) 30 | pretty.parseBytes("0PB") must be(0) 31 | 32 | pretty.parseBytes("5") must be(5L) 33 | pretty.parseBytes("5KB") must be(5L*1024) 34 | pretty.parseBytes("5MB") must be(5L*1024*1024) 35 | pretty.parseBytes("5GB") must be(5L*1024*1024*1024) 36 | 37 | pretty.parseBytes("1038524239") must be(1038524239L) 38 | pretty.parseBytes("1038524239 KB") must be(1038524239L*1024) 39 | 40 | pretty.parseBytes("5 GB") must be(5L*1024*1024*1024) 41 | pretty.parseBytes(" \t5\tGB ") must be(5L*1024*1024*1024) 42 | 43 | evaluating { pretty.parseBytes("") } must throwAn[IllegalArgumentException] 44 | evaluating { pretty.parseBytes("\t") } must throwAn[IllegalArgumentException] 45 | evaluating { pretty.parseBytes("3.2") } must throwAn[IllegalArgumentException] 46 | evaluating { pretty.parseBytes("5G") } must throwAn[IllegalArgumentException] 47 | evaluating { pretty.parseBytes("5MBB") } must throwAn[IllegalArgumentException] 48 | evaluating { pretty.parseBytes("foo") } must throwAn[IllegalArgumentException] 49 | 50 | evaluating { pretty.parseBytes("5BB") } must not(throwAn[IllegalArgumentException]) 51 | evaluating { pretty.parseBytes("5XB") } must throwAn[IllegalArgumentException] 52 | 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/finagle/DiscoResolver.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net.finagle 2 | 3 | import com.metamx.common.lifecycle.Lifecycle 4 | import com.metamx.common.scala.Logging 5 | import com.metamx.common.scala.net.curator.Disco 6 | import com.twitter.finagle.{Addr, Address, Resolver} 7 | import com.twitter.util.{Closable, Future, Time, Var} 8 | import java.net.InetSocketAddress 9 | import org.apache.curator.framework.CuratorFramework 10 | import org.apache.curator.framework.state.ConnectionState 11 | import org.apache.curator.x.discovery.details.ServiceCacheListener 12 | import scala.collection.JavaConverters._ 13 | 14 | /** 15 | * Bridges Finagle with Curator-based service discovery. 16 | * 17 | * @param disco Service discovery environment 18 | */ 19 | class DiscoResolver(disco: Disco) extends Resolver with Logging 20 | { 21 | val scheme = "disco" 22 | 23 | def bind(service: String) = Var.async[Addr](Addr.Pending) { 24 | updatable => 25 | val lifecycle = new Lifecycle 26 | val serviceCache = disco.cacheFor(service, lifecycle) 27 | def doUpdate() { 28 | val newInstances = serviceCache.getInstances.asScala.toSet 29 | log.info("Updating instances for service[%s] to %s", service, newInstances) 30 | val newSocketAddresses: Set[Address] = newInstances map 31 | (instance => Address(new InetSocketAddress(instance.getAddress, 32 | if (instance.getSslPort != null && instance.getSslPort > 0) instance.getSslPort else instance.getPort))) 33 | updatable.update(Addr.Bound(newSocketAddresses)) 34 | } 35 | serviceCache.addListener( 36 | new ServiceCacheListener 37 | { 38 | def cacheChanged() { 39 | doUpdate() 40 | } 41 | 42 | def stateChanged(curator: CuratorFramework, state: ConnectionState) { 43 | doUpdate() 44 | } 45 | } 46 | ) 47 | lifecycle.start() 48 | try { 49 | doUpdate() 50 | new Closable 51 | { 52 | def close(deadline: Time) = Future { 53 | log.info("No longer monitoring service[%s]", service) 54 | lifecycle.stop() 55 | } 56 | } 57 | } 58 | catch { 59 | case e: Exception => 60 | log.warn(e, "Failed to bind to service[%s]", service) 61 | lifecycle.stop() 62 | throw e 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/net/curator/DiscothequeTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net.curator 2 | 3 | import com.metamx.common.ISE 4 | import com.metamx.common.lifecycle.Lifecycle 5 | import com.metamx.common.scala.Predef.EffectOps 6 | import com.metamx.common.scala.control.ifException 7 | import com.metamx.common.scala.control.retryOnErrors 8 | import com.simple.simplespec.Matchers 9 | import java.net.BindException 10 | import org.apache.curator.test.TestingCluster 11 | import org.hamcrest.CoreMatchers._ 12 | import org.junit.Test 13 | 14 | class DiscothequeTest extends Matchers { 15 | 16 | @Test 17 | def testTheSameInstance(): Unit = { 18 | var lifecycle: Lifecycle = null 19 | var cluster: TestingCluster = null 20 | 21 | try { 22 | lifecycle = new Lifecycle().withEffect(_.start()) 23 | cluster = newTestingZkCluster().withEffect(_.start()) 24 | 25 | val discotheque = new Discotheque(lifecycle) 26 | 27 | val disco1 = discotheque.disco(cluster.getConnectString, "/test/discoPath") 28 | val disco2 = discotheque.disco(cluster.getConnectString, "/test/discoPath") 29 | 30 | disco1 must be(sameInstance(disco2)) 31 | } finally { 32 | Option(lifecycle).foreach(_.stop()) 33 | Option(cluster).foreach(_.stop()) 34 | } 35 | } 36 | 37 | @Test 38 | def testAnnounce(): Unit = { 39 | var lifecycle: Lifecycle = null 40 | var cluster: TestingCluster = null 41 | 42 | try { 43 | lifecycle = new Lifecycle().withEffect(_.start()) 44 | cluster = newTestingZkCluster().withEffect(_.start()) 45 | 46 | val discotheque = new Discotheque(lifecycle) 47 | 48 | val announceConfig1 = DiscoAnnounceConfig("test:instance", 8080, false) 49 | 50 | val disco1 = discotheque.disco(cluster.getConnectString, "/test/discoPath", Some(announceConfig1)) 51 | val disco2 = discotheque.disco(cluster.getConnectString, "/test/discoPath") 52 | 53 | disco1 must be(sameInstance(disco2)) 54 | 55 | evaluating { 56 | discotheque.disco(cluster.getConnectString, "/test/discoPath", Some(announceConfig1.copy(port = 8081))) 57 | } must throwAn[ISE] 58 | } finally { 59 | Option(lifecycle).foreach(_.stop()) 60 | Option(cluster).foreach(_.stop()) 61 | } 62 | } 63 | 64 | private def newTestingZkCluster(size: Int = 1): TestingCluster = { 65 | retryOnErrors(ifException[BindException]) { new TestingCluster(size) } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/LateVal.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | /** 20 | * A late val is a single-assignment val that can be assigned after definition. Subsequent 21 | * assignments raise an error, and dereferencing a late val before assignment also raises an error. 22 | * 23 | * val x = new LateVal[Int] 24 | * ... 25 | * x.deref // BAD 26 | * x.assign(3) // Good 27 | * x.deref // Good 28 | * x.assign(4) // BAD 29 | * 30 | * A LateVal[X] can be used as an X, via the implicit conversion LateVal. Moreover, the method 31 | * names LateVal.assign and LateVal.deref are chosen to minimize shadowing whatever methods will be 32 | * available on X; in particular, LateVal.set and LateVal.get would shadow methods for many common 33 | * choices of X. 34 | * 35 | * val x = new LateVal[Int] 36 | * x.assign(3) 37 | * x + 1 38 | * 39 | * val x = new LateVal[Map[Int, String]] 40 | * x.assign(Map(1 -> "one")) 41 | * x.get(1) 42 | * 43 | */ 44 | object LateVal { 45 | 46 | class LateVal[X] { 47 | 48 | val monitor = new AnyRef 49 | @volatile private var v: Option[X] = None 50 | 51 | def assign(x: X) { 52 | monitor synchronized { 53 | if (v == None) { 54 | v = Some(x) 55 | } else { 56 | throw new IllegalStateException("LateVal(%s) already defined: assign(%s)" format (v.get, x)) 57 | } 58 | } 59 | } 60 | 61 | def assignIfEmpty(x: X) { 62 | monitor synchronized { 63 | if (v == None) { 64 | v = Some(x) 65 | } 66 | } 67 | } 68 | 69 | def deref: X = v getOrElse { 70 | throw new IllegalArgumentException("Undefined LateVal") 71 | } 72 | 73 | def derefOption: Option[X] = v 74 | 75 | } 76 | implicit def LateVal[X](x: LateVal[X]): X = x.deref 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/finagle/InetAddressResolver.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net.finagle 2 | 3 | import com.google.common.util.concurrent.ThreadFactoryBuilder 4 | import com.metamx.common.scala.Logging 5 | import com.twitter.finagle.util.{DefaultTimer, InetSocketAddressUtil} 6 | import com.twitter.finagle.{Addr, Address, Resolver} 7 | import com.twitter.util.{Closable, Future, FuturePool, Timer, Var, Duration => TwitterDuration} 8 | import java.util.concurrent.Executors 9 | import java.util.concurrent.atomic.AtomicBoolean 10 | 11 | /** 12 | * Like the InetResolver in Finagle, but periodically re-resolves names. (The built-in InetResolver does not, at least 13 | * as of Finagle 6.16.0). Like the built-in resolver, initial resolution is synchronous. Re-resolution occurs in the 14 | * background. 15 | * 16 | * Names are resolved roughly every '''networkaddress.cache.ttl''' seconds. 17 | * 18 | * @param ttl How often to re-resolve names 19 | * @param futurePool FuturePool used for background name resolution 20 | * @param timer Timer used to schedule background name resolution 21 | */ 22 | class InetAddressResolver(ttl: TwitterDuration, futurePool: FuturePool, timer: Timer) 23 | extends Resolver with Logging 24 | { 25 | override val scheme = "inetaddr" 26 | 27 | override def bind(arg: String) = Var.async[Addr](Addr.Pending) { 28 | updatable => 29 | updatable.update(resolveString(arg)) 30 | val again = new AtomicBoolean(true) 31 | def schedule() { 32 | timer.doLater(ttl) { 33 | futurePool { 34 | updatable.update(resolveString(arg)) 35 | } ensure { 36 | if (again.get()) { 37 | schedule() 38 | } 39 | } 40 | } 41 | } 42 | schedule() 43 | Closable.make { 44 | deadline => 45 | again.set(false) 46 | Future.Done 47 | } 48 | } 49 | 50 | private def resolveString(arg: String) = Addr.Bound(InetSocketAddressUtil.parseHosts(arg).map(Address(_)): _*) 51 | } 52 | 53 | object InetAddressResolver 54 | { 55 | def default = DefaultInetAddressResolver 56 | } 57 | 58 | object DefaultInetAddressResolver extends InetAddressResolver( 59 | TwitterDuration.fromSeconds(60), 60 | FuturePool( 61 | Executors.newSingleThreadExecutor( 62 | new ThreadFactoryBuilder() 63 | .setNameFormat("InetAddressResolver-Default") 64 | .setDaemon(true) 65 | .build() 66 | ) 67 | ), 68 | DefaultTimer.twitter 69 | ) 70 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Yaml.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.Predef._ 20 | import java.io.InputStream 21 | import java.io.Reader 22 | import java.io.Writer 23 | import java.{util => ju} 24 | import org.yaml.snakeyaml.DumperOptions 25 | import org.yaml.{snakeyaml => snake} 26 | 27 | import untyped.normalizeJava 28 | 29 | object Yaml { 30 | 31 | def load(in: InputStream) : Any = load(in, create) 32 | def load(in: Reader) : Any = load(in, create) 33 | def load(in: String) : Any = load(in, create) 34 | def load(in: InputStream, yaml: snake.Yaml) : Any = yaml.load(in) 35 | def load(in: Reader, yaml: snake.Yaml) : Any = yaml.load(in) 36 | def load(in: String, yaml: snake.Yaml) : Any = yaml.load(in) 37 | 38 | def dump(x: Any) : String = dump(x, create) 39 | def dump(x: Any, out: Writer) : Unit = dump(x, out, create) 40 | def dump(x: Any, yaml: snake.Yaml) : String = yaml.dump(normalizeJava(stripSharing(x))).stripLineEnd 41 | def dump(x: Any, out: Writer, yaml: snake.Yaml) : Unit = yaml.dump(normalizeJava(stripSharing(x)), out) 42 | 43 | def pretty(x: Any) : String = dump(x, createPretty) 44 | def pretty(x: Any, out: Writer) : Unit = dump(x, out, createPretty) 45 | 46 | def create = new snake.Yaml 47 | def createPretty = new snake.Yaml(new DumperOptions withEffect { opts => 48 | opts.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK) 49 | }) 50 | 51 | // HACK: snakeyaml dumps shared structure (e.g. { val x = new ju.ArrayList; List(x,x) }) using 52 | // anchors (&*), which is never what we want in our use cases. I can't find anything in 53 | // DumperOptions to control this, so we simply strip all shared structure by converting to json 54 | // and back. 55 | def stripSharing(x: Any): Any = Jackson.normalize(x) 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/time/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.timekeeper.Timekeeper 20 | 21 | package object time { 22 | 23 | import com.github.nscala_time.time.Imports._ 24 | import org.joda.time.ReadableDateTime 25 | import org.joda.time.ReadableDuration 26 | import org.joda.time.ReadableInterval 27 | import org.joda.time.ReadablePeriod 28 | 29 | def timed[X](f: => X): (Long, X) = { 30 | val start = System.currentTimeMillis() 31 | val x = f 32 | val end = System.currentTimeMillis() 33 | (end - start, x) 34 | } 35 | 36 | def timed[X](timekeeper: Timekeeper)(f: => X): (Long, X) = { 37 | val start = timekeeper.now 38 | val x = f 39 | (timekeeper.now.getMillis - start.getMillis, x) 40 | } 41 | 42 | class DateTimeOps(t: ReadableDateTime) { 43 | 44 | def min(u: ReadableDateTime) = new DateTime(t.millis min u.millis) 45 | def max(u: ReadableDateTime) = new DateTime(t.millis max u.millis) 46 | 47 | } 48 | implicit def DateTimeOps(t: ReadableDateTime) = new DateTimeOps(t) 49 | 50 | class DurationOps(a: ReadableDuration) { 51 | 52 | def min(b: ReadableDuration) = new Duration(a.millis min b.millis) 53 | def max(b: ReadableDuration) = new Duration(a.millis max b.millis) 54 | 55 | def at (t: DateTime) : Interval = t to t+a 56 | def until (t: DateTime) : Interval = t-a to t 57 | 58 | def isEmpty = a.getMillis == 0 59 | def nonEmpty = !isEmpty 60 | 61 | } 62 | implicit def DurationOps(d: ReadableDuration) = new DurationOps(d) 63 | 64 | class PeriodOps(a: ReadablePeriod) { 65 | 66 | def at (t: DateTime) : Interval = t to t+a 67 | def until (t: DateTime) : Interval = t-a to t 68 | 69 | } 70 | implicit def PeriodOps(p: ReadablePeriod) = new PeriodOps(p) 71 | 72 | class IntervalOps(a: ReadableInterval) { 73 | 74 | def isEmpty = a.toDurationMillis == 0 75 | def nonEmpty = !isEmpty 76 | 77 | } 78 | implicit def IntervalOps(i: ReadableInterval) = new IntervalOps(i) 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/event/emit.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.event 2 | 3 | import com.github.nscala_time.time.Imports._ 4 | import com.google.common.base.Throwables 5 | import com.metamx.common.scala.Jackson 6 | import com.metamx.common.scala.Logger 7 | import com.metamx.common.scala.Predef._ 8 | import com.metamx.common.scala.untyped._ 9 | import com.metamx.emitter.service.AlertEvent.Severity 10 | import com.metamx.emitter.service.AlertEvent.Severity._ 11 | import com.metamx.emitter.service.AlertBuilder 12 | import com.metamx.emitter.service.ServiceEmitter 13 | import org.codehaus.jackson.map.ObjectMapper 14 | import scala.compat.Platform 15 | 16 | object emit 17 | { 18 | 19 | def emitAlert(log: Logger, emitter: ServiceEmitter, severity: Severity, description: String, data: Dict): Unit = { 20 | emitAlert(null, log, emitter, severity, description, data) 21 | } 22 | 23 | def emitAlert( 24 | e: Throwable, 25 | log: Logger, 26 | emitter: ServiceEmitter, 27 | severity: Severity, 28 | description: String, 29 | data: Dict 30 | ): Unit = { 31 | ((if (severity == ANOMALY) log.warn(_,_) else log.error(_,_)): (Throwable, String) => Unit)( 32 | e, "Emitting alert: [%s] %s\n%s" format (severity, description, Jackson.pretty(data)) 33 | ) 34 | 35 | emitter.emit( 36 | AlertBuilder.create(description).severity(severity).withEffect { 37 | x => 38 | (dict(normalizeJavaViaJson(data)) ++ 39 | Option(e).map( 40 | e => Dict( 41 | "exceptionType" -> e.getClass.getName, 42 | "exceptionMessage" -> e.getMessage, 43 | "exceptionStackTrace" -> Throwables.getStackTraceAsString(e) 44 | ) 45 | ).getOrElse(Dict())) foreach { 46 | case (k, v) => x.addData(k, v) 47 | } 48 | } 49 | ) 50 | } 51 | 52 | def emitMetricTimed[T]( 53 | emitter: ServiceEmitter, 54 | metric: Metric 55 | )(action: => T): T = { 56 | val t0 = Platform.currentTime 57 | val res = action 58 | val t = Platform.currentTime - t0 59 | emitter.emit(metric + Metric(value = t, created = new DateTime())) 60 | res 61 | } 62 | 63 | // HACK: Map scala-native types to java types by writing out through jackson and reading back in through jackson. 64 | // This will not only normalize scala collections, which untyped.normalizeJava knows how to do, but also things like 65 | // Option and Either, which untyped.normalizeJava doesn't and shouldn't know how to do. 66 | def normalizeJavaViaJson(x: Any): Any = _jacksonMapper.readValue(Jackson.generate(x), classOf[Any]) 67 | private lazy val _jacksonMapper = new ObjectMapper 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/WalkerTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.simple.simplespec.Matchers 4 | import org.junit.Test 5 | 6 | class WalkerTest extends Matchers 7 | { 8 | 9 | def newWalker(): Walker[String] = { 10 | new Walker[String] { 11 | override def foreach(f: String => Unit) = List("hey", "there") foreach f 12 | } 13 | } 14 | 15 | @Test 16 | def testSimple() 17 | { 18 | val walker = newWalker() 19 | walker.toList must be(List("hey", "there")) 20 | walker.toList must be(List("hey", "there")) 21 | walker.foldLeft(0)(_ + _.size) must be(8) 22 | walker.foldLeft(0)(_ + _.size) must be(8) 23 | } 24 | 25 | @Test 26 | def testMap() 27 | { 28 | val walker = newWalker().map(_.size) 29 | walker.toList must be(List(3, 5)) 30 | walker.toList must be(List(3, 5)) 31 | } 32 | 33 | @Test 34 | def testFlatMap() 35 | { 36 | val walker = newWalker().flatMap(x => x) 37 | walker.toList must be(List('h', 'e', 'y', 't', 'h', 'e', 'r', 'e')) 38 | walker.toList must be(List('h', 'e', 'y', 't', 'h', 'e', 'r', 'e')) 39 | } 40 | 41 | @Test 42 | def testFilter() 43 | { 44 | val walker = newWalker().flatMap(x => x).filter(_ == 'e') 45 | walker.toList must be(List('e', 'e', 'e')) 46 | walker.toList must be(List('e', 'e', 'e')) 47 | } 48 | 49 | @Test 50 | def testPlusPlus() 51 | { 52 | val walker = newWalker() ++ newWalker().map(_.substring(0, 1)) 53 | walker.toList must be(List("hey", "there", "h", "t")) 54 | walker.toList must be(List("hey", "there", "h", "t")) 55 | } 56 | 57 | @Test 58 | def testSize() 59 | { 60 | val walker = newWalker().flatMap(x => x) 61 | walker.size must be(8) 62 | walker.size must be(8) 63 | } 64 | 65 | @Test 66 | def testToSet() 67 | { 68 | val walker = newWalker().flatMap(x => x) 69 | walker.toSet must be(Set('h', 'e', 'y', 't', 'r')) 70 | walker.toSet must be(Set('h', 'e', 'y', 't', 'r')) 71 | } 72 | 73 | @Test 74 | def testFromIterable() 75 | { 76 | val walker = Walker(Seq(1, 2, 3, 3)) 77 | walker.toSet must be(Set(1, 2, 3)) 78 | walker.toSet must be(Set(1, 2, 3)) 79 | } 80 | 81 | @Test 82 | def testFromForeachFunction() 83 | { 84 | val walker = Walker((f: Int => Unit) => Seq(1, 2, 3).foreach(f)) 85 | walker.toList must be(List(1, 2, 3)) 86 | walker.toList must be(List(1, 2, 3)) 87 | } 88 | 89 | @Test 90 | def testFromForeachFunctionOnce() 91 | { 92 | val walker = Walker.once((f: Int => Unit) => Seq(1, 2, 3).foreach(f)) 93 | walker.toList must be(List(1, 2, 3)) 94 | evaluating { 95 | walker.toList 96 | } must throwAn[IllegalStateException]("""Cannot walk more than once""") 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/control.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.github.nscala_time.time.Imports._ 20 | import com.metamx.common.Backoff 21 | import com.metamx.common.scala.option.OptionOps 22 | import scala.annotation.tailrec 23 | import scala.reflect.ClassTag 24 | 25 | object control extends Logging { 26 | 27 | @tailrec 28 | def untilSome[X](x: => Option[X]): X = x match { 29 | case Some(y) => y 30 | case None => untilSome(x) 31 | } 32 | 33 | def retryOnError[E <: Exception](isTransient: E => Boolean) = new { 34 | def apply[X](x: => X)(implicit ct: ClassTag[E]) = retryOnErrors( 35 | (e: Exception) => ct.runtimeClass.isAssignableFrom(e.getClass) && isTransient(e.asInstanceOf[E]) 36 | )(x) 37 | } 38 | 39 | // FIXME When used with untilPeriod, the last sleep is useless 40 | def retryOnErrors[X](isTransients: (Exception => Boolean)*)(x: => X): X = { 41 | withBackoff { backoff => 42 | try Some(x) catch { 43 | case e: Exception if isTransients.find(_(e)).isDefined => 44 | log.warn(e, "Transient error, retrying after %s ms", backoff.next) 45 | None 46 | } 47 | } 48 | } 49 | 50 | def withBackoff[X](f: Backoff => Option[X]): X = { 51 | val backoff = Backoff.standard() 52 | untilSome { 53 | f(backoff) ifEmpty { 54 | backoff.sleep 55 | } 56 | } 57 | } 58 | 59 | def ifException[E <: Exception](implicit ct: ClassTag[E]) = (e: Exception) => 60 | ct.runtimeClass.isAssignableFrom(e.getClass) 61 | 62 | def ifExceptionSatisfies[E <: Exception](pred: E => Boolean)(implicit ct: ClassTag[E]) = (e: Exception) => 63 | ct.runtimeClass.isAssignableFrom(e.getClass) && pred(e.asInstanceOf[E]) 64 | 65 | class PredicateOps[A](f: A => Boolean) 66 | { 67 | def untilCount(count: Int) = { 68 | var n = 0 69 | (a: A) => if (n < count) { 70 | val x = f(a) 71 | n += 1 72 | x 73 | } else { 74 | false 75 | } 76 | } 77 | 78 | def untilPeriod(period: Period) = { 79 | val end = DateTime.now + period 80 | (a: A) => if (DateTime.now < end) f(a) else false 81 | } 82 | } 83 | 84 | implicit def PredicateOps[A](f: A => Boolean) = new PredicateOps(f) 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/event/AlertAggregator.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.event 2 | 3 | import com.metamx.common.scala.Logger 4 | import com.metamx.common.scala.untyped.Dict 5 | import com.metamx.emitter.service.ServiceEmitter 6 | import com.metamx.metrics.{AbstractMonitor, Monitor} 7 | import java.util.concurrent.atomic.AtomicReference 8 | import scala.collection.mutable 9 | import scala.util.Random 10 | 11 | /** 12 | * Aggregates alerts together and can periodically emit and log aggregated alerts. Uses memory linear in the 13 | * number of (description, exception class) pairs seen between alert emissions. This is meant to make it feasible to 14 | * report exceptions that may occur at very high rates. 15 | * 16 | * To control memory use and noisiness of alerting, it is important to avoid using a wide variety of descriptions. 17 | * 18 | * @param log Used to log exceptions 19 | */ 20 | class AlertAggregator(log: Logger, private val rand: Random = new Random) 21 | { 22 | // (description, exception class) 23 | type AlertKey = (String, Option[String]) 24 | 25 | private val lock = new AnyRef 26 | private val alerts = new AtomicReference(mutable.HashMap[AlertKey, AggregatedAlerts]()) 27 | 28 | def put(e: Throwable, description: String, data: Dict) { 29 | put(Option(e), description, data) 30 | } 31 | 32 | def put(description: String, data: Dict) { 33 | put(None, description, data) 34 | } 35 | 36 | private def put(e: Option[Throwable], description: String, data: Dict) { 37 | val key = (description, e.map(_.getClass.getName)) 38 | lock.synchronized { 39 | alerts.get().getOrElseUpdate(key, new AggregatedAlerts(description)).put(e, data) 40 | } 41 | } 42 | 43 | lazy val monitor: Monitor = new AbstractMonitor { 44 | override def doMonitor(emitter: ServiceEmitter) = { 45 | // Swap alert buffers and emit the old ones. 46 | val snapshot = lock.synchronized { 47 | alerts.getAndSet(mutable.HashMap()) 48 | } 49 | for (((description, _), aa) <- snapshot) { 50 | // emitAlert can handle null exceptions 51 | emit.emitAlert( 52 | e = aa.e.orNull, 53 | log = log, 54 | emitter = emitter, 55 | severity = WARN, 56 | description = description, 57 | data = Dict( 58 | "sampleDetails" -> aa.data, 59 | "alertCount" -> aa.count 60 | ) 61 | ) 62 | } 63 | true 64 | } 65 | } 66 | 67 | private class AggregatedAlerts(val description: String) 68 | { 69 | var count = 0 70 | var data: Dict = Map.empty 71 | var e: Option[Throwable] = None 72 | 73 | def put(e: Option[Throwable], data: Dict) { 74 | // Try to get representative data instead of just the first ones. 75 | count += 1 76 | if (rand.nextInt(count) == 0) { 77 | this.data = data 78 | this.e = e 79 | } 80 | } 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/ExceptionTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.metamx.common.scala.exception.{causeMatches, causedBy, causes} 4 | import com.simple.simplespec.Matchers 5 | import java.io.IOException 6 | import org.junit.Test 7 | 8 | class ExceptionTest extends Matchers { 9 | 10 | @Test 11 | def testCausesEmpty() { 12 | val e = new RuntimeException("foo") 13 | causes(e).toList.map(_.getMessage) must be(List("foo")) 14 | } 15 | 16 | @Test 17 | def testCausesSingle() { 18 | val e = new RuntimeException("foo", new RuntimeException("bar")) 19 | causes(e).toList.map(_.getMessage) must be(List("foo", "bar")) 20 | } 21 | 22 | @Test 23 | def testCausesMulti() { 24 | val e = new RuntimeException("foo", new RuntimeException("bar", new RuntimeException("baz"))) 25 | causes(e).toList.map(_.getMessage) must be(List("foo", "bar", "baz")) 26 | } 27 | 28 | @Test 29 | def testCauseMatches() { 30 | val e = new UnsupportedOperationException("foo", new IOException("bar", new ArrayIndexOutOfBoundsException("baz"))) 31 | causeMatches(e) { case _ => true } must be(true) 32 | causeMatches(e) { case _ => false } must be(false) 33 | causeMatches(e) { case _: RuntimeException => false } must be(false) 34 | causeMatches(e) { case _: IOException => true } must be(true) 35 | causeMatches(e) { case _: UnsupportedOperationException => false; case _: IOException => true } must be(true) 36 | causeMatches(e) { case x: IOException if x.getMessage == "rofl" => true } must be(false) 37 | causeMatches(e) { case _: ArrayIndexOutOfBoundsException => true } must be(true) 38 | causeMatches(e) { case _: NoSuchElementException => true } must be(false) 39 | } 40 | 41 | @Test 42 | def testCausedBy() { 43 | val e = new UnsupportedOperationException("foo", new IOException("bar", new ArrayIndexOutOfBoundsException("baz"))) 44 | causedBy[RuntimeException](e) must be(true) 45 | causedBy[IOException](e) must be(true) 46 | causedBy[UnsupportedOperationException](e) must be(true) 47 | causedBy[ArrayIndexOutOfBoundsException](e) must be(true) 48 | causedBy[NoSuchElementException](e) must be(false) 49 | } 50 | 51 | @Test 52 | def testCausedByTryCatch() { 53 | val x = try { 54 | throw new UnsupportedOperationException("foo", new IOException("bar", new ArrayIndexOutOfBoundsException("baz"))) 55 | } catch { 56 | case e if causedBy[ArrayIndexOutOfBoundsException](e) => 57 | true 58 | case _ => 59 | false 60 | } 61 | x must be(true) 62 | } 63 | 64 | @Test 65 | def testCausedByTryCatchNoMatch() { 66 | val x = try { 67 | throw new UnsupportedOperationException("foo", new IOException("bar", new ArrayIndexOutOfBoundsException("baz"))) 68 | } catch { 69 | case e if causedBy[NoSuchElementException](e) => 70 | true 71 | case _ => 72 | false 73 | } 74 | x must be(false) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/threads.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.github.nscala_time.time.Imports._ 4 | import com.metamx.common.scala.Predef._ 5 | import com.metamx.common.scala.concurrent._ 6 | import scala.util.control.NonFatal 7 | 8 | object threads extends Logging 9 | { 10 | class RunnerThread(name: String, quietPeriod: Option[Period], f: => Any) extends Thread with Logging 11 | { 12 | setName(name) 13 | setDaemon(true) 14 | 15 | @volatile private var terminated = false 16 | 17 | override def run(): Unit = { 18 | try { 19 | val quietMillis = quietPeriod.map(_.toStandardDuration.getMillis) 20 | 21 | while (!terminated && !isInterrupted) { 22 | try { 23 | val startMillis = System.currentTimeMillis() 24 | try { 25 | f 26 | } catch { 27 | case NonFatal(e) => log.error(e, "Exception while running thread [%s]".format(name)) 28 | } 29 | 30 | quietMillis match { 31 | case Some(m) => 32 | val waitMillis = startMillis + m - System.currentTimeMillis() 33 | if (waitMillis > 0) { 34 | Thread.sleep(waitMillis) 35 | } 36 | 37 | case None => // Don't need to sleep 38 | } 39 | } catch { 40 | case e: InterruptedException => 41 | if (terminated) { 42 | log.info("Thread [%s] terminated") 43 | } else { 44 | log.info("Thread [%s] interrupted") 45 | } 46 | interrupt() 47 | } 48 | } 49 | } catch { 50 | case e: Throwable => log.error(e, "Thread [%s] killed by exception".format(name)) 51 | } 52 | } 53 | 54 | def terminate() { 55 | terminated = true 56 | interrupt() 57 | } 58 | } 59 | 60 | def startHaltingThread(body: => Any, name: String) = daemonThread { abortingRunnable { 61 | try body catch { 62 | case e: Throwable => 63 | log.error(e, "Halting") 64 | Runtime.getRuntime.halt(1) 65 | } 66 | }} withEffect { 67 | t => 68 | t.setName(name) 69 | t.start() 70 | } 71 | 72 | def runnerThread(name: String, f: => Any): RunnerThread = { 73 | runnerThread(name, None, f) 74 | } 75 | 76 | def runnerThread(name: String, quietPeriod: Period, f: => Any): RunnerThread = { 77 | runnerThread(name, Some(quietPeriod), f) 78 | } 79 | 80 | def initRunnerThread(name: String, f: => Any): RunnerThread = { 81 | new RunnerThread(name, None, f) 82 | } 83 | 84 | def initRunnerThread(name: String, quietPeriod: Period, f: => Any): RunnerThread = { 85 | new RunnerThread(name, Some(quietPeriod), f) 86 | } 87 | 88 | private def runnerThread(name: String, quietPeriod: Option[Period], f: => Any): RunnerThread = { 89 | val thread = new RunnerThread(name, quietPeriod, f) 90 | thread.start() 91 | thread 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/curator/Discotheque.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net.curator 2 | 3 | import com.metamx.common.ISE 4 | import com.metamx.common.lifecycle.Lifecycle 5 | import com.metamx.common.scala.collection.concurrent.PermanentMap 6 | import org.apache.curator.framework.CuratorFramework 7 | import org.joda.time.Duration 8 | 9 | /** 10 | * Collection of Disco instances to avoid multiple instances creation for the same endpoint (zkConnect, discoPath). 11 | * 12 | * If DiscoAnnounceConfig is defined and disco for this endpoint already created for different announce config then 13 | * [[IllegalStateException]] will be thrown. 14 | * 15 | * All methods are thread safe. 16 | */ 17 | class Discotheque(lifecycle: Lifecycle) 18 | { 19 | private val curators = new PermanentMap[CuratorConfig, CuratorFramework]() 20 | private val discos = new PermanentMap[DiscoEndpoint, (Disco, Option[DiscoAnnounceConfig])]() 21 | 22 | def disco(config: CuratorConfig with DiscoConfig): Disco = { 23 | disco(config.zkConnect, config.discoPath, config.discoAnnounce, config.zkTimeout) 24 | } 25 | 26 | def disco(curatorConfig: CuratorConfig, discoConfig: DiscoConfig): Disco = { 27 | disco(curatorConfig.zkConnect, discoConfig.discoPath, discoConfig.discoAnnounce, curatorConfig.zkTimeout) 28 | } 29 | 30 | def disco( 31 | discoZkConnect: String, 32 | discoPath: String, 33 | discoAnnounceConfig: Option[DiscoAnnounceConfig] = None, 34 | discoZkTimeout: Duration = Duration.parse("PT15S") 35 | ): Disco = { 36 | val endpoint = DiscoEndpoint(discoZkConnect, discoPath) 37 | 38 | val (disco, announceConfig) = discos.getOrElseUpdate(endpoint, { 39 | val curatorConfig = endpoint.toCuratorConfig(discoZkTimeout) 40 | val curator = curators.getOrElseUpdate(curatorConfig, Curator.create(curatorConfig, lifecycle)) 41 | 42 | val discoConfig = endpoint.toDiscoConfig(discoAnnounceConfig) 43 | (lifecycle.addMaybeStartManagedInstance(new Disco(curator, discoConfig)), discoAnnounceConfig) 44 | }) 45 | 46 | if (discoAnnounceConfig.isDefined && discoAnnounceConfig != announceConfig) { 47 | throw new ISE("Failed to create announce [%s] for zkConnect[%s] and path[%s]. Already announced to [%s]".format( 48 | discoAnnounceConfig.toString, 49 | discoZkConnect, 50 | discoPath, 51 | announceConfig.map(_.toString).getOrElse("None") 52 | )) 53 | } 54 | 55 | disco 56 | } 57 | 58 | private case class DiscoEndpoint(discoZkConnect: String, discoPath: String) 59 | { 60 | def toCuratorConfig(zookeeperTimeout: Duration): CuratorConfig = new CuratorConfig 61 | { 62 | override def zkTimeout: Duration = zookeeperTimeout 63 | 64 | override def zkConnect: String = DiscoEndpoint.this.discoZkConnect 65 | } 66 | 67 | def toDiscoConfig(discoAnnounceConfig: Option[DiscoAnnounceConfig]) = new DiscoConfig 68 | { 69 | def discoAnnounce = discoAnnounceConfig 70 | 71 | def discoPath = DiscoEndpoint.this.discoPath 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Jackson.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.fasterxml.jackson.core.{JsonFactory, JsonGenerator, JsonParser} 4 | import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} 5 | import com.fasterxml.jackson.datatype.joda.JodaModule 6 | import com.fasterxml.jackson.module.scala.DefaultScalaModule 7 | import com.metamx.common.scala.Predef._ 8 | import java.io.{FilterOutputStream, FilterWriter, InputStream, OutputStream, Reader, Writer} 9 | import scala.reflect.ClassTag 10 | 11 | object Jackson extends Jackson 12 | 13 | trait Jackson 14 | { 15 | private val objectMapper = newObjectMapper() 16 | 17 | def parse[A: ClassTag](s: String): A = { 18 | objectMapper.readValue(s, implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]]) 19 | } 20 | 21 | def parse[A: ClassTag](bs: Array[Byte]): A = { 22 | objectMapper.readValue(bs, implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]]) 23 | } 24 | 25 | def parse[A: ClassTag](reader: Reader): A = { 26 | objectMapper.readValue(reader, implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]]) 27 | } 28 | 29 | def parse[A: ClassTag](stream: InputStream): A = { 30 | objectMapper.readValue(stream, implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]]) 31 | } 32 | 33 | def parse[A: ClassTag](jp: JsonParser): A = { 34 | objectMapper.readValue(jp, implicitly[ClassTag[A]].runtimeClass.asInstanceOf[Class[A]]) 35 | } 36 | 37 | def generate[A](a: A): String = objectMapper.writeValueAsString(a) 38 | 39 | def generate[A](a: A, writer: Writer) { 40 | objectMapper.writeValue(writer, a) 41 | } 42 | 43 | def generate[A](a: A, stream: OutputStream) { 44 | objectMapper.writeValue(stream, a) 45 | } 46 | 47 | def generate[A](a: A, jg: JsonGenerator) { 48 | objectMapper.writeValue(jg, a) 49 | } 50 | 51 | def bytes[A](a: A): Array[Byte] = objectMapper.writeValueAsBytes(a) 52 | 53 | def pretty[A](a: A): String = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(a) 54 | 55 | def pretty[A](a: A, writer: Writer) { 56 | objectMapper.writerWithDefaultPrettyPrinter().writeValue(writer, a) 57 | } 58 | 59 | def pretty[A](a: A, stream: OutputStream) { 60 | objectMapper.writerWithDefaultPrettyPrinter().writeValue(stream, a) 61 | } 62 | 63 | def normalize[A : ClassTag](a: A) = parse[A](generate(a)) 64 | 65 | def newObjectMapper(): ObjectMapper = newObjectMapper(null) 66 | 67 | def newObjectMapper(jsonFactory: JsonFactory): ObjectMapper = { 68 | new ObjectMapper(jsonFactory) withEffect { 69 | jm => 70 | jm.registerModule(new JodaModule) 71 | jm.registerModule(DefaultScalaModule) 72 | jm.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) 73 | } 74 | } 75 | } 76 | 77 | // Jackson.generate and Jackson.pretty automatically close their JsonGenerator. Use these to prevent that. 78 | class NoCloseWriter (x: Writer) extends FilterWriter(x) { override def close {} } 79 | class NoCloseOutputStream (x: OutputStream) extends FilterOutputStream(x) { override def close {} } 80 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/concurrent/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | package object concurrent { 20 | 21 | import com.github.nscala_time.time.Imports._ 22 | import com.metamx.common.scala.Predef._ 23 | import com.metamx.common.scala.exception._ 24 | import java.util.concurrent.{Callable, ExecutionException, Executors} 25 | import scala.util.Random 26 | 27 | // loggingRunnable and abortingRunnable are almost always better than these 28 | //implicit def asRunnable(f: () => Unit) = new Runnable { def run = f } 29 | //implicit def asCallable[X](f: () => X) = new Callable[X] { def call = f() } 30 | 31 | def loggingRunnable(body: => Any) = new Runnable with Logging { 32 | override def run { 33 | try body catch { 34 | case e: Throwable => log.error(e, "Killed by exception") 35 | } 36 | } 37 | } 38 | 39 | def abortingRunnable(body: => Any) = new Runnable { 40 | override def run { 41 | try body catch { 42 | case e: Throwable => Abort(e) 43 | } 44 | } 45 | } 46 | 47 | def loggingThread(body: => Any) = daemonThread(loggingRunnable(body)) 48 | def abortingThread(body: => Any) = daemonThread(abortingRunnable(body)) 49 | def daemonThread(f: Runnable) = new Thread(f) withEffect { _ setDaemon true } 50 | 51 | def callable[X](x: => X) = new Callable[X] { def call = x } 52 | 53 | def numCores = Runtime.getRuntime.availableProcessors 54 | def par[X,Y](xs: Iterable[X], threads: Int = numCores)(f: X => Y): Vector[Y] = { 55 | // FIXME Creating a new pool for each call is dangerous... Should every caller really maintain their own pool...? 56 | Executors.newFixedThreadPool(threads).withFinally(_.shutdown) { exec => 57 | Vector() ++ xs map { x => exec.submit(callable { f(x) }) } map (_.get) 58 | } mapException { 59 | case e: ExecutionException => e.getCause // Wrapped exceptions are anti-useful; unwrap them 60 | } 61 | } 62 | 63 | // TODO Avoid drift 64 | def everyFuzzy(period: Duration, fuzz: Double, delay: Boolean = true)(f: => Unit) { 65 | if (delay) Thread.sleep((period.millis * Random.nextDouble).toLong) // Randomize phase 66 | forever { 67 | f 68 | Thread.sleep((period.millis * math.max(1 + fuzz * Random.nextGaussian, 0)).toLong) // Fuzz period 69 | } 70 | } 71 | 72 | def spawn(body: => Any) { 73 | loggingThread(body).start 74 | } 75 | 76 | def after(millis: Long)(f: => Unit) { 77 | spawn { 78 | Thread sleep millis 79 | f 80 | } 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/concurrent/BlockingQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection.concurrent 18 | 19 | import java.util.concurrent.atomic.AtomicInteger 20 | import java.util.concurrent.locks.ReentrantLock 21 | import scala.collection.mutable.ListBuffer 22 | import com.metamx.common.scala.concurrent.locks.LockOps 23 | 24 | abstract class BlockingQueue[E] 25 | { 26 | // Total elements in queue 27 | protected val _count = new AtomicInteger 28 | 29 | // Protects put/take and offer/poll 30 | private val lock = new ReentrantLock() 31 | 32 | // Signals waiting put/offer and take/poll 33 | private val condition = lock.newCondition() 34 | 35 | // Tries to insert an element non-blocking way, returns whether the insert was successful. 36 | def offer(elem: E): Boolean = { 37 | lock { 38 | val result = enqueue(elem) 39 | condition.signalAll() 40 | result 41 | } 42 | } 43 | 44 | // Tries to fetch an element non-blocking way, returns None if there is no elements in the queue and 45 | // Some(element) if an element was retrieved. 46 | def poll(): Option[E] = { 47 | lock { 48 | val result = dequeue() 49 | condition.signalAll() 50 | result 51 | } 52 | } 53 | 54 | // Puts element blocking way. 55 | def put(elem: E) { 56 | lock { 57 | while (!enqueue(elem)) { 58 | condition.await() 59 | } 60 | condition.signalAll() 61 | } 62 | } 63 | 64 | // Takes element blocking way, returns fetched element. 65 | def take(): E = { 66 | lock { 67 | var result: Option[E] = None 68 | while (result.isEmpty) { 69 | result = dequeue() 70 | if (result.isEmpty) { 71 | condition.await() 72 | } 73 | } 74 | condition.signalAll() 75 | result.get 76 | } 77 | } 78 | 79 | // Returns total elements count in the queue 80 | def count() = _count.get() 81 | 82 | // Removes all elements from the queue and returns them 83 | def drain(maxElements: Int = Int.MaxValue): List[E] = { 84 | val size = math.min(maxElements, _count.get()) 85 | 86 | val buf = new ListBuffer[E]() 87 | buf.sizeHint(size) 88 | 89 | var elem: Option[E] = None 90 | var i: Int = 0 91 | 92 | while ({ 93 | if (i < size) { 94 | elem = poll() 95 | elem.isDefined 96 | } else { 97 | false 98 | } 99 | }) { 100 | buf.append(elem.get) 101 | i += 1 102 | } 103 | 104 | buf.result() 105 | } 106 | 107 | protected def enqueue(elem: E): Boolean 108 | 109 | protected def dequeue(): Option[E] 110 | } 111 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/db/MySQLDB.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.db 18 | 19 | import com.metamx.common.scala.Predef._ 20 | 21 | // TODO Extract not-mysql-specific stuff from createIsTransient, push up to DB. Must: 22 | // - be easy to combine: `new DB(config) with MySQLErrors' or `new DB(config, mySqlErrors)' 23 | // - not be error prone 24 | // - support recursive isTransient test 25 | 26 | class MySQLDB(config: DBConfig) extends DB(config) { 27 | 28 | override def createIsTransient: Throwable => Boolean = { 29 | 30 | var timeoutLimit = 3 31 | 32 | def isTransient(e: Throwable): Boolean = e match { 33 | 34 | // If our query repeatedly fails to finish, then we should probably stop doing it 35 | case e: com.mysql.jdbc.exceptions.MySQLTimeoutException => 36 | log.info("DB query timed out: timeoutLimit = %s", timeoutLimit) 37 | (timeoutLimit > 1) andThen { 38 | timeoutLimit -= 1 39 | } 40 | 41 | // Anything marked "transient" 42 | case e: java.sql.SQLTransientException => true 43 | case e: com.mysql.jdbc.exceptions.MySQLTransientException => true 44 | 45 | // IO errors from jdbc look like this [are we responsible for force-closing the connection in this case?] 46 | case e: java.sql.SQLRecoverableException => true 47 | 48 | // Specific errors from jdbi with no useful supertype 49 | case e: org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException => true 50 | //case e: org.skife.jdbi.v2.exceptions.UnableToCloseResourceException => true // TODO Include this one? 51 | 52 | // MySQL ER_QUERY_INTERRUPTED "Query execution was interrupted" [ETL-153] 53 | case e: java.sql.SQLException if e.getErrorCode == 1317 => true 54 | 55 | // MySQL ER_LOCK_WAIT_TIMEOUT "Lock wait timeout exceeded; try restarting transaction" 56 | case e: java.sql.SQLException if e.getErrorCode == 1205 => true 57 | 58 | // Unwrap nested exceptions from jdbc and jdbi 59 | case e: java.sql.SQLException if isTransient(e.getCause) => true 60 | case e: org.skife.jdbi.v2.exceptions.DBIException if isTransient(e.getCause) => true 61 | 62 | // Nothing else, including nulls 63 | case _ => false 64 | 65 | } 66 | 67 | isTransient _ 68 | 69 | } 70 | 71 | override def createTable(table: String, decls: Seq[String]) { 72 | execute("create table %s (%s) engine=innodb charset=utf8" format (table, decls mkString ", ")) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Predef.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | object Predef { 20 | 21 | def forever(f: => Unit) { 22 | while (true) { 23 | f 24 | } 25 | } 26 | 27 | // `x into f into g == g(f(x))', like F#'s pipeline operator: 28 | // http://debasishg.blogspot.com/2009/09/thrush-combinator-in-scala.html 29 | // http://en.wikibooks.org/wiki/F_Sharp_Programming/Higher_Order_Functions#The_.7C.3E_Operator 30 | class IntoOps[X](x: X) { 31 | def into[Y](f: X => Y): Y = f(x) 32 | } 33 | implicit def IntoOps[X](x: X) = new IntoOps(x) 34 | 35 | class NullOps[X](x: X) { 36 | def mapNull (x0 : => X) : X = if (x == null) x0 else x 37 | def mapNonNull[Y >: Null] (f : X => Y) : Y = if (x == null) null else f(x) 38 | @deprecated("Use mapNull", "0.5.0") def ifNull (x0 : => X) : X = mapNull(x0) 39 | @deprecated("Use mapNonNull", "0.5.0") def unlessNull[Y >: Null] (f : X => Y) : Y = mapNonNull(f) 40 | } 41 | implicit def NullOps[X](x: X) = new NullOps(x) 42 | 43 | class EffectOps[X](x: X) { 44 | def withEffect(f: X => Unit): X = { f(x); x } 45 | } 46 | implicit def EffectOps[X](x: X) = new EffectOps(x) 47 | 48 | 49 | class RequiringOps[X](x: X){ 50 | def requiring(f: X => (Boolean, String)) : X = { val (y, msg) = f(x); require(y, msg); x } 51 | def requiring(f: X => Boolean, message: => Any): X = { require(f(x), message); x } 52 | } 53 | implicit def RequiringOps[X](x: X) = new RequiringOps(x) 54 | 55 | class FinallyOps[X](x: X) { 56 | def withFinally[Y](close: X => Unit) = (f: X => Y) => try f(x) finally close(x) 57 | } 58 | implicit def FinallyOps[X](x: X) = new FinallyOps(x) 59 | 60 | class BooleanOps(x: Boolean) { 61 | def orElse (f: => Unit): Boolean = { if (!x) f; x } 62 | def andThen (f: => Unit): Boolean = { if (x) f; x } 63 | } 64 | implicit def BooleanOps(x: Boolean) = new BooleanOps(x) 65 | 66 | class TraversableOnceBooleanOps(xs: TraversableOnce[Boolean]) { 67 | def any: Boolean = xs exists identity 68 | def all: Boolean = xs forall identity 69 | } 70 | implicit def TraversableOnceBooleanOps(xs: TraversableOnce[Boolean]) = new TraversableOnceBooleanOps(xs) 71 | 72 | // Disjunction types à la Miles Sabin (http://stackoverflow.com/a/6312508/397334), e.g. 73 | // 74 | // def size[X : (Int Or String)#F](x: X): Int = x match { 75 | // case n: Int => n 76 | // case s: String => s.length 77 | // } 78 | // 79 | type Or[A,B] = { 80 | type not[X] = X => Nothing 81 | type and[X,Y] = X with Y 82 | type or[X,Y] = not[not[X] and not[Y]] 83 | type F[X] = not[not[X]] <:< (A or B) 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/concurrent/ByteBufferQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection.concurrent 18 | 19 | import com.google.common.primitives.Ints 20 | import java.nio.ByteBuffer 21 | import java.util.concurrent.atomic.AtomicLong 22 | 23 | class ByteBufferQueue(buffer: ByteBuffer) extends BlockingQueue[Array[Byte]] 24 | { 25 | // Header (which is simply an integer length) size 26 | private val headerLength = 4 27 | 28 | // Maximum buffer size 29 | private val size = buffer.capacity() 30 | 31 | // Index of the oldest element 32 | private var start: Int = 0 33 | 34 | // Index of the next write position 35 | private var end: Int = 0 36 | 37 | // Used space size 38 | private val _used = new AtomicLong 39 | 40 | // Returns used space in bytes 41 | def used() = _used.get() 42 | 43 | override protected def enqueue(elem: Array[Byte]): Boolean = { 44 | if (elem == null) { 45 | throw new NullPointerException("Can't put null element") 46 | } 47 | 48 | val elemTotalLength = headerLength + elem.length 49 | if (size < elemTotalLength) { 50 | // There is no way to get this element ever appended so let's fail fast 51 | throw new IllegalStateException("Element too big to enqueue") 52 | } 53 | 54 | if (size - _used.get() < elemTotalLength) { 55 | false 56 | } else { 57 | 58 | val hdr = Ints.toByteArray(elem.length) 59 | 60 | writeArray(end, hdr) 61 | writeArray(end + headerLength, elem) 62 | 63 | _used.addAndGet(elemTotalLength) 64 | _count.incrementAndGet() 65 | 66 | end = (end + elemTotalLength) % size 67 | 68 | true 69 | } 70 | } 71 | 72 | override protected def dequeue(): Option[Array[Byte]] = { 73 | if (_used.get() == 0) { 74 | None 75 | } else { 76 | 77 | val hdr = readArray(start, headerLength) 78 | val length = Ints.fromByteArray(hdr) 79 | val elem = readArray(start + headerLength, length) 80 | 81 | val elemTotalLength = headerLength + elem.length 82 | 83 | _used.addAndGet(-elemTotalLength) 84 | _count.decrementAndGet() 85 | 86 | start = (start + elemTotalLength) % size 87 | 88 | Some(elem) 89 | } 90 | } 91 | 92 | private def writeArray(offset: Int, arr: Array[Byte]) { 93 | for (i <- 0 until arr.length) { 94 | buffer.put((offset + i) % size, arr(i)) 95 | } 96 | } 97 | 98 | private def readArray(offset: Int, length: Int): Array[Byte] = { 99 | val arr = new Array[Byte](length) 100 | for (i <- 0 until arr.length) { 101 | arr(i) = buffer.get((offset + i) % size) 102 | } 103 | arr 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Math.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | // Implicits for everything in scala.math that isn't already in runtime.Rich* 20 | 21 | object Math { 22 | 23 | class RichInt(a: Int) { 24 | 25 | def signum = math signum a 26 | 27 | def pow (b: Int) = math pow (a,b) toInt 28 | def pow (b: Long) = math pow (a,b) toLong 29 | def pow (b: Float) = math pow (a,b) toFloat 30 | def pow (b: Double) = math pow (a,b) toDouble 31 | def ** (b: Int) = math pow (a,b) toInt 32 | def ** (b: Long) = math pow (a,b) toLong 33 | def ** (b: Float) = math pow (a,b) toFloat 34 | def ** (b: Double) = math pow (a,b) toDouble 35 | 36 | } 37 | 38 | class RichLong(a: Long) { 39 | 40 | def signum = math signum a 41 | 42 | def pow (b: Long) = math pow (a,b) toLong 43 | def pow (b: Float) = math pow (a,b) toFloat 44 | def pow (b: Double) = math pow (a,b) toDouble 45 | def ** (b: Long) = math pow (a,b) toLong 46 | def ** (b: Float) = math pow (a,b) toFloat 47 | def ** (b: Double) = math pow (a,b) toDouble 48 | 49 | } 50 | 51 | class RichFloat(a: Float) { 52 | 53 | def signum = math signum a 54 | def ulp = math ulp a 55 | 56 | def pow (b: Float) = math pow (a,b) toFloat 57 | def pow (b: Double) = math pow (a,b) toDouble 58 | def ** (b: Float) = math pow (a,b) toFloat 59 | def ** (b: Double) = math pow (a,b) toDouble 60 | 61 | } 62 | 63 | class RichDouble(a: Double) { 64 | 65 | def sin = math sin a 66 | def cos = math cos a 67 | def tan = math tan a 68 | def asin = math asin a 69 | def acos = math acos a 70 | def atan = math atan a 71 | def exp = math exp a 72 | def log = math log a 73 | def sqrt = math sqrt a 74 | def rint = math rint a 75 | def signum = math signum a 76 | def log10 = math log10 a 77 | def cbrt = math cbrt a 78 | def ulp = math ulp a 79 | def sinh = math sinh a 80 | def cosh = math cosh a 81 | def tanh = math tanh a 82 | def expm1 = math expm1 a 83 | def log1p = math log1p a 84 | 85 | def IEEEremainder (b: Double) = math IEEEremainder (a,b) 86 | def atan2 (b: Double) = math atan2 (a,b) 87 | def pow (b: Double) = math pow (a,b) 88 | def ** (b: Double) = math pow (a,b) 89 | def hypot (b: Double) = math hypot (a,b) 90 | 91 | } 92 | 93 | implicit def RichInt (a: Int) = new RichInt (a) 94 | implicit def RichLong (a: Long) = new RichLong (a) 95 | implicit def RichFloat (a: Float) = new RichFloat (a) 96 | implicit def RichDouble (a: Double) = new RichDouble (a) 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Walker.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import java.util.concurrent.atomic.AtomicBoolean 4 | import scala.collection.GenTraversableOnce 5 | 6 | /** 7 | * Walkers provide a mechanism for walking through an underlying listish thing, exposed as a "foreach" method. They 8 | * do not return iterators, nor do they allow random access. This allows them to guarantee post-iteration cleanup 9 | * actions on the underlying resource, which will occur even if exceptions are thrown while walking. 10 | * 11 | * Walkers can be constructed such that the "foreach" method can only be called once. In that case, subsequent calls 12 | * should throw an IllegalStateException. Walkers can also be constructed with "foreach" methods that can be called 13 | * multiple times. In that case, each run should create and then clean up the resource-- saving state across runs is 14 | * usually counterproductive. 15 | */ 16 | trait Walker[+A] 17 | { 18 | self => 19 | 20 | def foreach(f: A => Unit) 21 | 22 | def map[B](f: A => B): Walker[B] = new Walker[B] { 23 | override def foreach(g: B => Unit) = self.foreach(a => g(f(a))) 24 | } 25 | 26 | def flatMap[B](f: A => GenTraversableOnce[B]): Walker[B] = new Walker[B] { 27 | override def foreach(g: B => Unit) = self.foreach(a => f(a) foreach g) 28 | } 29 | 30 | def filter(p: A => Boolean): Walker[A] = new Walker[A] { 31 | override def foreach(g: A => Unit) = self foreach { 32 | a => 33 | if (p(a)) { 34 | g(a) 35 | } 36 | } 37 | } 38 | 39 | def withFilter(p: A => Boolean): Walker[A] = filter(p) 40 | 41 | def ++[B >: A](other: Walker[B]): Walker[B] = new Walker[B] { 42 | override def foreach(g: B => Unit) { 43 | self foreach g 44 | other foreach g 45 | } 46 | } 47 | 48 | def toList: List[A] = { 49 | val builder = List.newBuilder[A] 50 | foreach(builder += _) 51 | builder.result() 52 | } 53 | 54 | def toSet[B >: A]: Set[B] = { 55 | val builder = Set.newBuilder[B] 56 | foreach(builder += _) 57 | builder.result() 58 | } 59 | 60 | def foldLeft[B](zero: B)(combine: (B, A) => B): B = { 61 | var current = zero 62 | foreach { 63 | a => 64 | current = combine(current, a) 65 | } 66 | current 67 | } 68 | 69 | def size: Long = { 70 | var count = 0L 71 | foreach(_ => count += 1) 72 | count 73 | } 74 | } 75 | 76 | object Walker 77 | { 78 | def empty[A]: Walker[A] = Walker[A](Nil) 79 | 80 | def apply[A](xs: Iterable[A]): Walker[A] = new Walker[A] { 81 | override def foreach(f: A => Unit) = xs foreach f 82 | } 83 | 84 | def apply[A](foreachFn: (A => Unit) => Unit): Walker[A] = new Walker[A] { 85 | override def foreach(f: A => Unit) { 86 | foreachFn(f) 87 | } 88 | } 89 | 90 | def once[A](foreachFn: (A => Unit) => Unit): Walker[A] = new Walker[A] { 91 | val finished = new AtomicBoolean(false) 92 | 93 | override def foreach(f: A => Unit) { 94 | val wasFinished = finished.getAndSet(true) 95 | if (wasFinished) { 96 | throw new IllegalStateException("Cannot walk more than once") 97 | } else { 98 | foreachFn(f) 99 | } 100 | } 101 | } 102 | 103 | class WalkerTuple2Ops[A, B](walker: Walker[(A, B)]) 104 | { 105 | def toMap: Map[A, B] = { 106 | val builder = Map.newBuilder[A, B] 107 | walker.foreach(builder += _) 108 | builder.result() 109 | } 110 | } 111 | 112 | implicit def WalkerTuple2Ops[A, B](walker: Walker[(A, B)]) = new WalkerTuple2Ops(walker) 113 | } 114 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/counters/NumericCountersTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.counters 2 | 3 | import com.simple.simplespec.Matchers 4 | import org.junit.Test 5 | 6 | class NumericCountersTest extends Matchers 7 | { 8 | private def makeMetrics(counters: LongCounters) = { 9 | counters.snapshotAndReset().map { 10 | m => 11 | val key = 12 | "%s: %s & (%s)".format( 13 | m.build("test", "localhost").getFeed, 14 | m.metric, 15 | m.userDims.toSeq.map{ 16 | case (dim, values) => "%s -> %s".format(dim, values.mkString(", ")) 17 | }.sorted.mkString("; ") 18 | ) 19 | key -> m.value.toString 20 | }.toMap 21 | } 22 | 23 | private def makeMetrics(counters: DoubleCounters) = { 24 | counters.snapshotAndReset().map { 25 | m => 26 | val key = 27 | "%s: %s & (%s)".format( 28 | m.build("test", "localhost").getFeed, 29 | m.metric, 30 | m.userDims.toSeq.map{ 31 | case (dim, values) => "%s -> %s".format(dim, values.mkString(", ")) 32 | }.sorted.mkString("; ") 33 | ) 34 | key -> "%.0f".format(m.value.doubleValue()) 35 | }.toMap 36 | } 37 | 38 | @Test def testLong() { 39 | val counters = new LongCounters 40 | 41 | counters.inc("a", Map("x" -> Seq("1"))) 42 | counters.add("a", Map("x" -> Seq("1")), 2) 43 | 44 | counters.add("b", Map("z" -> Seq("4")), 10000) 45 | counters.inc("b", Map("z" -> Seq("4"))) 46 | 47 | counters.add("a", Map("x" -> Seq("1", "2")), 10) 48 | 49 | counters.add("a", Map("x" -> Seq("1"), "y" -> Seq("3")), 100) 50 | 51 | counters.add("c", Map("x" -> Seq("1")), 10) 52 | counters.del("c", Map("x" -> Seq("1"))) 53 | 54 | makeMetrics(counters) must be (Map( 55 | "metrics: a & (x -> 1)" -> "3", 56 | "metrics: a & (x -> 1, 2)" -> "10", 57 | "metrics: a & (x -> 1; y -> 3)" -> "100", 58 | "metrics: b & (z -> 4)" -> "10001" 59 | )) 60 | 61 | counters.inc("a", Map("x" -> Seq("1"))) 62 | 63 | makeMetrics(counters) must be (Map( 64 | "metrics: a & (x -> 1)" -> "1" 65 | )) 66 | } 67 | 68 | @Test def testDouble() { 69 | implicit val counters = new DoubleCounters 70 | 71 | 72 | counters.inc("a", Map("x" -> Seq("1"))) 73 | counters.add("a", Map("x" -> Seq("1")), 2.0) 74 | 75 | counters.add("b", Map("z" -> Seq("4")), 10000.0) 76 | counters.inc("b", Map("z" -> Seq("4"))) 77 | 78 | counters.add("a", Map("x" -> Seq("1", "2")), 10.0) 79 | 80 | counters.add("a", Map("x" -> Seq("1"), "y" -> Seq("3")), 100.0) 81 | 82 | counters.add("c", Map("x" -> Seq("1")), 10.0) 83 | counters.del("c", Map("x" -> Seq("1"))) 84 | 85 | makeMetrics(counters) must be (Map( 86 | "metrics: a & (x -> 1)" -> "3", 87 | "metrics: a & (x -> 1, 2)" -> "10", 88 | "metrics: a & (x -> 1; y -> 3)" -> "100", 89 | "metrics: b & (z -> 4)" -> "10001" 90 | )) 91 | 92 | counters.inc("a", Map("x" -> Seq("1"))) 93 | 94 | makeMetrics(counters) must be (Map( 95 | "metrics: a & (x -> 1)" -> "1" 96 | )) 97 | } 98 | 99 | @Test def testLongWithCustomFeed(): Unit = { 100 | val counters = new LongCounters("test_feed") 101 | 102 | counters.inc("a", Map("x" -> Seq("1"))) 103 | counters.add("a", Map("x" -> Seq("1")), 2) 104 | counters.inc("b", Map("z" -> Seq("4"))) 105 | 106 | makeMetrics(counters) must be (Map( 107 | "test_feed: a & (x -> 1)" -> "3", 108 | "test_feed: b & (z -> 4)" -> "1" 109 | )) 110 | 111 | counters.inc("a", Map("x" -> Seq("1"))) 112 | 113 | makeMetrics(counters) must be (Map( 114 | "test_feed: a & (x -> 1)" -> "1" 115 | )) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/exception.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import scala.reflect.{ClassTag, classTag} 20 | import scala.util.control.Exception.{catching => _catching} 21 | 22 | // TODO Tests 23 | 24 | object exception { 25 | 26 | /** 27 | * Returns a lazily-computed stream of causes for a particular Throwable. The provided Throwable will be returned 28 | * first, followed by its cause chain, if any. 29 | */ 30 | def causes(e: Throwable): Stream[Throwable] = Stream.cons(e, Option(e.getCause) map causes getOrElse Stream.empty) 31 | 32 | /** 33 | * Checks if a Throwable, or any Throwable in its cause chain, matches a partial predicate. 34 | */ 35 | def causeMatches(e: Throwable)(f: PartialFunction[Throwable, Boolean]) = causes(e).collect(f).contains(true) 36 | 37 | /** 38 | * Checks if a Throwable, or any Throwable in its cause chain, is a particular type. 39 | */ 40 | def causedBy[E <: Throwable: ClassTag](e: Throwable) = causeMatches(e) { 41 | case x if classTag[E].runtimeClass.isAssignableFrom(x.getClass) => true 42 | } 43 | 44 | def raises[E <: Throwable] = new { 45 | def apply[X](x: => X)(implicit ct: ClassTag[E], ev: NotNothing[E]): Boolean = 46 | x.catchOption[E].isEmpty 47 | } 48 | 49 | def toOption[E <: Throwable] = new { 50 | def apply[X](x: => X)(implicit ct: ClassTag[E], ev: NotNothing[E]): Option[X] = 51 | x.catchOption[E] 52 | } 53 | 54 | def toEither[E <: Throwable] = new { 55 | def apply[X](x: => X)(implicit ct: ClassTag[E], ev: NotNothing[E]): Either[E,X] = 56 | x.catchEither[E] 57 | } 58 | 59 | class ExceptionOps[X](x: => X) { 60 | 61 | def mapException(f: PartialFunction[Throwable, Throwable]): X = 62 | try x catch { 63 | case e if f isDefinedAt e => throw f(e) 64 | } 65 | 66 | def catching[Y](f: PartialFunction[Throwable, Y]) = new { 67 | def orElse(y: => Y) : Y = orElse(_ => y) 68 | def orElse(g: X => Y) : Y = { 69 | (try Left(x) catch { 70 | case e if f isDefinedAt e => Right(f(e)) 71 | }) match { 72 | case Left(x) => g(x) 73 | case Right(y) => y 74 | } 75 | } 76 | } 77 | 78 | def swallow(f: PartialFunction[Throwable, Any]): Option[X] = 79 | try Some(x) catch { 80 | case e if f.isDefinedAt(e) => f(e); None 81 | } 82 | 83 | def catchOption[E <: Throwable : ClassTag : NotNothing]: Option[X] = 84 | _catching(classTag[E].runtimeClass) opt x 85 | 86 | def catchEither[E <: Throwable : ClassTag : NotNothing]: Either[E,X] = 87 | (_catching(classTag[E].runtimeClass) either x).left map (_.asInstanceOf[E]) 88 | 89 | } 90 | 91 | implicit def ExceptionOps[X](x: => X): ExceptionOps[X] = new ExceptionOps(x) 92 | // Bug: by-name implicit conversions don't work for X = Nothing 93 | // Workaround: manually upcast at the call site 94 | 95 | // Negative type bounds via ambiguous implicits [https://gist.github.com/206a147b7291fd5b2193] 96 | trait NotNothing[X] 97 | implicit def notNothing[X] : NotNothing[X] = null // Accept NotNothing[X] for all X 98 | implicit def isNothing0[X <: Nothing] : NotNothing[X] = null // Reject NotNothing[Nothing] 99 | implicit def isNothing1[X <: Nothing] : NotNothing[X] = null 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/RetryOnErrorTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala 2 | 3 | import com.github.nscala_time.time.Imports._ 4 | import com.metamx.common.scala.control._ 5 | import com.simple.simplespec.Matchers 6 | import org.junit.Test 7 | import scala.reflect.ClassTag 8 | 9 | class RetryOnErrorTest extends Matchers 10 | { 11 | 12 | def transientFailure[E <: Exception](period: Period)(implicit ct: ClassTag[E]) = { 13 | val end = DateTime.now + period 14 | () => if (DateTime.now >= end) "hello world" else throw ct.runtimeClass.newInstance().asInstanceOf[E] 15 | } 16 | 17 | @Test 18 | def testSimpleOneArg() 19 | { 20 | val f = transientFailure[IllegalStateException](1.second) 21 | retryOnError((e: IllegalStateException) => true) { f.apply() } must be("hello world") 22 | } 23 | 24 | @Test 25 | def testSimpleOneArgAltForm() 26 | { 27 | val f = transientFailure[IllegalStateException](1.second) 28 | retryOnError[IllegalStateException](_ => true) { f() } must be("hello world") 29 | } 30 | 31 | @Test 32 | def testWrongExceptionOneArg() 33 | { 34 | val f = transientFailure[IllegalStateException](1.second) 35 | evaluating { 36 | retryOnError((e: IllegalArgumentException) => true)(f()) 37 | } must throwAn[IllegalStateException] 38 | } 39 | 40 | @Test 41 | def testSimpleTwoArg() 42 | { 43 | val f = transientFailure[IllegalStateException](1.second) 44 | retryOnErrors( 45 | ifException[IllegalArgumentException], 46 | ifException[IllegalStateException] 47 | )(f()) must be("hello world") 48 | } 49 | 50 | @Test 51 | def testWrongExceptionTwoArg() 52 | { 53 | val f = transientFailure[IllegalStateException](1.second) 54 | evaluating { 55 | retryOnErrors( 56 | ifException[IllegalArgumentException], 57 | ifException[NumberFormatException] 58 | )(f()) 59 | } must throwAn[IllegalStateException] 60 | } 61 | 62 | @Test 63 | def testTimeoutNotReached() 64 | { 65 | val f = transientFailure[IllegalStateException](1.second) 66 | retryOnError( 67 | ifException[IllegalStateException] untilPeriod(3.seconds) 68 | )(f()) must be("hello world") 69 | } 70 | 71 | @Test 72 | def testTimeoutReached() 73 | { 74 | val f = transientFailure[IllegalStateException](10.seconds) 75 | evaluating { 76 | retryOnError( 77 | ifException[IllegalStateException] untilPeriod(500.millis) 78 | )(f()) 79 | } must throwAn[IllegalStateException] 80 | } 81 | 82 | @Test 83 | def testCountoutNotReached() 84 | { 85 | val f = transientFailure[IllegalStateException](1.second) 86 | retryOnError( 87 | ifException[IllegalStateException] untilCount(30) 88 | )(f()) must be("hello world") 89 | } 90 | 91 | @Test 92 | def testCountoutReached() 93 | { 94 | var count = 0 95 | val f = transientFailure[IllegalStateException](1.second) 96 | evaluating { 97 | retryOnError( 98 | ifException[IllegalStateException] untilCount(1) 99 | ) { 100 | count += 1 101 | f() 102 | } 103 | } must throwAn[IllegalStateException] 104 | count must be(2) 105 | } 106 | 107 | 108 | @Test 109 | def testCountoutZero() 110 | { 111 | var count = 0 112 | val f = transientFailure[IllegalStateException](1.second) 113 | evaluating { 114 | retryOnError( 115 | ifException[IllegalStateException] untilCount(0) 116 | ) { 117 | count += 1 118 | f() 119 | } 120 | } must throwAn[IllegalStateException] 121 | count must be(1) 122 | } 123 | 124 | @Test 125 | def testIfExceptionSatisfies() 126 | { 127 | val f = transientFailure[IllegalStateException](1.second) 128 | retryOnErrors(ifExceptionSatisfies[IllegalStateException](_.getCause == null)) { f.apply() } must be("hello world") 129 | } 130 | 131 | @Test 132 | def testIfExceptionDoesNotSatisfyClass() 133 | { 134 | val f = transientFailure[IllegalStateException](1.second) 135 | evaluating { 136 | retryOnErrors(ifExceptionSatisfies[IllegalArgumentException](_.getCause == null)) { f.apply() } 137 | } must throwAn[IllegalStateException] 138 | } 139 | 140 | @Test 141 | def testIfExceptionDoesNotSatisfyTest() 142 | { 143 | val f = transientFailure[IllegalStateException](1.second) 144 | evaluating { 145 | retryOnErrors(ifExceptionSatisfies[IllegalStateException](_.getCause != null)) { f.apply() } 146 | } must throwAn[IllegalStateException] 147 | } 148 | 149 | } 150 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/event/Metric.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.event 2 | 3 | import com.github.nscala_time.time.Imports._ 4 | import com.google.common.collect.ImmutableMap 5 | import com.metamx.common.scala.untyped._ 6 | import com.metamx.emitter.service.{ServiceEventBuilder, ServiceMetricEvent} 7 | 8 | // A partially constructed ServiceEventBuilder[ServiceMetricEvent]. Immutable, unlike ServiceMetricEvent.Builder. 9 | case class Metric( 10 | metric: String, 11 | value: Number, 12 | userDims: Map[String, Iterable[String]], 13 | created: DateTime, 14 | feed: String 15 | ) extends ServiceEventBuilder[ServiceMetricEvent] 16 | { 17 | 18 | // Join two partially constructed metrics; throw IllegalArgumentException if any field is defined on both 19 | def +(that: Metric): Metric = { 20 | def onlyOne[X >: Null](x: X, y: X, desc: String): X = (Option(x), Option(y)) match { 21 | case (None, None) => null 22 | case (Some(x), None) => x 23 | case (None, Some(y)) => y 24 | case (Some(x), Some(y)) => throw new IllegalArgumentException( 25 | "%s already defined as %s, refusing to shadow with %s" format(desc, x, y) 26 | ) 27 | } 28 | 29 | def onlyOneOrEqual[X >: Null](x: X, y: X, desc: String): X = (Option(x), Option(y)) match { 30 | case (None, None) => null 31 | case (Some(x), None) => x 32 | case (None, Some(y)) => y 33 | case (Some(x), Some(y)) if x == y => x 34 | case (Some(x), Some(y)) => throw new IllegalArgumentException( 35 | "%s already defined as %s, refusing to shadow with %s" format(desc, x, y) 36 | ) 37 | } 38 | 39 | def intersectMap(x: Map[String, Iterable[String]], y: Map[String, Iterable[String]]): Map[String, Iterable[String]] = 40 | { 41 | x.keySet intersect y.keySet match { 42 | case xy if xy.isEmpty => x ++ y 43 | case xy => throw new IllegalArgumentException( 44 | "userDims has common keys: %s" format xy.mkString(",") 45 | ) 46 | } 47 | } 48 | 49 | new Metric( 50 | metric = onlyOne(this.metric, that.metric, "metric"), 51 | value = onlyOne(this.value, that.value, "value"), 52 | userDims = intersectMap(this.userDims, that.userDims), 53 | created = onlyOne(this.created, that.created, "created"), 54 | feed = onlyOneOrEqual(this.feed, that.feed, "feed") 55 | ) 56 | } 57 | 58 | def +(name: String, value: String): Metric = { 59 | this + Metric(userDims = Map(name -> Seq(value))) 60 | } 61 | 62 | def +(name: String, value: Iterable[String]): Metric = { 63 | this + Metric(userDims = Map(name -> value)) 64 | } 65 | 66 | override def build(service: String, host: String): ServiceMetricEvent = { 67 | build(ImmutableMap.of("service", noNull(service), "host", noNull(host))) 68 | } 69 | 70 | // Build into a ServiceMetricEvent, throwing NullPointerException if any required field is null 71 | override def build(serviceDimensions: ImmutableMap[String, String]): ServiceMetricEvent = { 72 | val builder = ServiceMetricEvent.builder() 73 | Option(feed).foreach(builder.setFeed) 74 | userDims.foreach { case (k, v) => builder.setDimension(k, v.toArray) } 75 | 76 | builder.build(created, noNull(metric), noNull(value)).build(serviceDimensions) 77 | } 78 | } 79 | 80 | //This object for backward compatibility with old stuff 81 | object Metric 82 | { 83 | @deprecated("userX dimensions are deprecated", "1.13.2") 84 | def apply( 85 | metric: String = null, 86 | value: Number = null, 87 | user1: Iterable[String] = null, 88 | user2: Iterable[String] = null, 89 | user3: Iterable[String] = null, 90 | user4: Iterable[String] = null, 91 | user5: Iterable[String] = null, 92 | user6: Iterable[String] = null, 93 | user7: Iterable[String] = null, 94 | user8: Iterable[String] = null, 95 | user9: Iterable[String] = null, 96 | user10: Iterable[String] = null, 97 | created: DateTime = null, 98 | userDims: Map[String, Iterable[String]] = Map.empty, 99 | feed: String = null 100 | ) = 101 | { 102 | var result = new Metric(metric, value, userDims, created, feed) 103 | 104 | if (user1 != null) { result = result + ("user1", user1) } 105 | if (user2 != null) { result = result + ("user2", user2) } 106 | if (user3 != null) { result = result + ("user3", user3) } 107 | if (user4 != null) { result = result + ("user4", user4) } 108 | if (user5 != null) { result = result + ("user5", user5) } 109 | if (user6 != null) { result = result + ("user6", user6) } 110 | if (user7 != null) { result = result + ("user7", user7) } 111 | if (user8 != null) { result = result + ("user8", user8) } 112 | if (user9 != null) { result = result + ("user9", user9) } 113 | if (user10 != null) { result = result + ("user10", user10) } 114 | 115 | result 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/event/MetricTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.event 2 | 3 | import com.simple.simplespec.Matchers 4 | import org.junit.Test 5 | import com.github.nscala_time.time.Imports._ 6 | 7 | 8 | class MetricTest extends Matchers 9 | { 10 | @Test 11 | def testUnionMetrics() { 12 | val m1 = Metric(metric = "metric") 13 | val m2 = Metric(value = 1) 14 | 15 | val res = m1 + m2 16 | 17 | res.metric must be("metric") 18 | res.value must be(1) 19 | } 20 | 21 | @Test 22 | def testUnionWithException() { 23 | val m1 = Metric(metric = "metric1", value = 1, created = new DateTime()) 24 | val m2 = Metric(metric = "metric2", value = 2, created = new DateTime()) 25 | 26 | evaluating { 27 | m1 + m2 28 | } must throwAn[IllegalArgumentException]("metric already defined as metric1, refusing to shadow with metric2") 29 | } 30 | 31 | @Test 32 | def testUnionDimensions() { 33 | (Metric() + ("dim", "val")).userDims.get("dim").isDefined must be(true) 34 | (Metric() + ("dim", Seq("val"))).userDims.get("dim").isDefined must be(true) 35 | 36 | evaluating { 37 | Metric(userDims = Map("dim" -> Seq("value1"))) + Metric(userDims = Map("dim" -> Seq("value2"))) 38 | } must throwAn[IllegalArgumentException]("userDims has common keys: dim") 39 | 40 | val res = Metric(userDims = Map("dim1" -> Seq("value1"))) + 41 | Metric(userDims = Map("dim2" -> Seq("value2.1", "value2.2"))) 42 | 43 | res.userDims.get("dim1").isDefined must be(true) 44 | res.userDims.get("dim1") must be(Some(Seq("value1"))) 45 | 46 | res.userDims.get("dim2").isDefined must be(true) 47 | res.userDims.get("dim2") must be(Some(Seq("value2.1", "value2.2"))) 48 | } 49 | 50 | @Test 51 | def testUnionWithCustomFeed(): Unit = { 52 | val m1 = Metric(metric = "metric", feed = "test_feed") 53 | val m2 = Metric(value = 1) 54 | 55 | val res = m1 + m2 56 | 57 | res.metric must be ("metric") 58 | res.value must be (1) 59 | res.feed must be ("test_feed") 60 | } 61 | 62 | @Test 63 | def testUnionWithSameCustomFeeds(): Unit = { 64 | val m1 = Metric(metric = "metric", feed = "test_feed") 65 | val m2 = Metric(value = 1, feed = "test_feed") 66 | 67 | val res = m1 + m2 68 | 69 | res.metric must be ("metric") 70 | res.value must be (1) 71 | res.feed must be ("test_feed") 72 | } 73 | 74 | @Test 75 | def testUnionWithDifferentCustomFeeds(): Unit = { 76 | val m1 = Metric(metric = "metric", feed = "test_feed") 77 | val m2 = Metric(value = 1, feed = "another_test_feed") 78 | 79 | evaluating { 80 | m1 + m2 81 | } must throwAn[IllegalArgumentException]("feed already defined as test_feed, refusing to shadow with another_test_feed") 82 | } 83 | 84 | @Test 85 | def testBuildServiceMetric() { 86 | val event = Metric( 87 | metric = "metric", 88 | value = 1, 89 | userDims = Map("dim1" -> Seq("value1"), "dim2" -> Seq("value2.1", "value2.2")) 90 | ).build("test", "localhost") 91 | 92 | event.getService must be ("test") 93 | event.getHost must be ("localhost") 94 | event.getMetric must be ("metric") 95 | event.getValue must be (1) 96 | 97 | event.getUserDims.containsKey("dim1") must be(true) 98 | event.getUserDims.containsKey("dim2") must be(true) 99 | } 100 | 101 | @Test 102 | def testBuildCustomFeed(): Unit = { 103 | val event = Metric( 104 | metric = "metric", 105 | value = 1, 106 | feed = "test_feed" 107 | ).build("test", "localhost") 108 | 109 | event.getService must be ("test") 110 | event.getHost must be ("localhost") 111 | event.getMetric must be ("metric") 112 | event.getValue must be (1) 113 | event.getFeed must be ("test_feed") 114 | } 115 | 116 | @Test 117 | def testUserDimensions(): Unit = { 118 | val metric = Metric( 119 | user1 = Seq("user1"), 120 | user2 = Seq("user2"), 121 | user3 = Seq("user3"), 122 | user4 = Seq("user4"), 123 | user5 = Seq("user5"), 124 | user6 = Seq("user6"), 125 | user7 = Seq("user7"), 126 | user8 = Seq("user8"), 127 | user9 = Seq("user9"), 128 | user10 = Seq("user10") 129 | ) 130 | 131 | metric.userDims.get("user1") must be(Some(Seq("user1"))) 132 | metric.userDims.get("user2") must be(Some(Seq("user2"))) 133 | metric.userDims.get("user3") must be(Some(Seq("user3"))) 134 | metric.userDims.get("user4") must be(Some(Seq("user4"))) 135 | metric.userDims.get("user5") must be(Some(Seq("user5"))) 136 | metric.userDims.get("user6") must be(Some(Seq("user6"))) 137 | metric.userDims.get("user7") must be(Some(Seq("user7"))) 138 | metric.userDims.get("user8") must be(Some(Seq("user8"))) 139 | metric.userDims.get("user9") must be(Some(Seq("user9"))) 140 | metric.userDims.get("user10") must be(Some(Seq("user10"))) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/curator/Disco.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.net.curator 2 | 3 | import com.metamx.common.lifecycle.Lifecycle.Handler 4 | import com.metamx.common.lifecycle.{Lifecycle, LifecycleStart, LifecycleStop} 5 | import java.net.URI 6 | import org.apache.curator.framework.CuratorFramework 7 | import org.apache.curator.x.discovery.{ServiceInstance, _} 8 | import scala.collection.JavaConverters._ 9 | 10 | abstract class AbstractDisco[T >: Null](curator: CuratorFramework, config: DiscoConfig, payload: Option[T] = None)(clazz: Class[T]) 11 | { 12 | val me: Option[ServiceInstance[T]] = config.discoAnnounce map { 13 | service => 14 | val builder = ServiceInstance.builder[T]().name(service.name).payload(payload.orNull) 15 | if (service.ssl) { 16 | builder.sslPort(service.port) 17 | } else { 18 | builder.port(service.port) 19 | } 20 | 21 | builder.build() 22 | } 23 | 24 | val disco: ServiceDiscovery[T] = { 25 | val builder = ServiceDiscoveryBuilder.builder(clazz) 26 | .basePath(config.discoPath) 27 | .client(curator) 28 | 29 | if (me.isDefined) { 30 | builder.thisInstance(me.get) 31 | } 32 | 33 | builder.build() 34 | } 35 | 36 | def providerFor(service: String, lifecycle: Lifecycle) = { 37 | val provider = disco.serviceProviderBuilder().serviceName(service).build() 38 | 39 | lifecycle.addHandler( 40 | new Handler 41 | { 42 | 43 | def start() { 44 | provider.start() 45 | } 46 | 47 | def stop() { 48 | provider.close() 49 | } 50 | } 51 | ) 52 | 53 | provider 54 | } 55 | 56 | def cacheFor(service: String, lifecycle: Lifecycle) = { 57 | val cache = disco.serviceCacheBuilder().name(service).build() 58 | 59 | lifecycle.addHandler( 60 | new Handler 61 | { 62 | def start() { 63 | cache.start() 64 | } 65 | 66 | def stop() { 67 | cache.close() 68 | } 69 | } 70 | ) 71 | 72 | cache 73 | } 74 | 75 | /** 76 | * Discovers a URI once, without a provider. This should be avoided in high volume use cases. 77 | */ 78 | def instanceFor(service: String): Option[ServiceInstance[T]] = disco.queryForInstances(service).asScala.headOption 79 | 80 | @LifecycleStart 81 | def start() { 82 | disco.start() 83 | } 84 | 85 | @LifecycleStop 86 | def stop() { 87 | disco.close() 88 | } 89 | } 90 | 91 | class Disco(curator: CuratorFramework, config: DiscoConfig) extends AbstractDisco[Void](curator, config)(Void.TYPE) 92 | 93 | /** 94 | * Please note, that if you use PayloadDisco with not null payload you will get a ClassCastException when you try 95 | * to get this data using the Disco class. But you can use PayloadDisco to get data which was written by Disco class. 96 | * If you want to update your service's announce from Disco to PayloadDisco, first make sure that all clients 97 | * of this service are updated to PayloadDisco. 98 | */ 99 | class PayloadDisco( 100 | curator: CuratorFramework, config: DiscoConfig, payload: Option[Array[Byte]] = None 101 | ) extends AbstractDisco[Array[Byte]](curator, config, payload)(classOf[Array[Byte]]) 102 | { 103 | def this(curator: CuratorFramework, config: DiscoConfig, payload: Array[Byte]) = { 104 | this(curator, config, Some(payload)) 105 | } 106 | } 107 | 108 | class ServiceProviderOps[T](provider: ServiceProvider[T]) 109 | { 110 | def instance: Option[ServiceInstance[T]] = Option(provider.getInstance()) 111 | } 112 | 113 | class ServiceCacheOps[T](cache: ServiceCache[T]) 114 | { 115 | def instances: Seq[ServiceInstance[T]] = cache.getInstances.asScala 116 | } 117 | 118 | class ServiceInstanceOps[T](service: ServiceInstance[T]) 119 | { 120 | def name = service.getName 121 | 122 | def id = service.getId 123 | 124 | def port = service.getPort 125 | 126 | def sslPort = service.getSslPort 127 | 128 | def payload = Option(service.getPayload) 129 | 130 | def registrationTimeUTC = service.getRegistrationTimeUTC 131 | 132 | def serviceType = service.getServiceType 133 | 134 | def uriSpec = service.getUriSpec 135 | 136 | /** 137 | * Extract a usable URI from this ServiceInstance. Will use the uriSpec if present, or otherwise will 138 | * attempt some reasonable default. 139 | */ 140 | def uri = Option(uriSpec) map (uriSpec => new URI(uriSpec.build(service))) getOrElse { 141 | val (proto, port) = if (service.getSslPort != null && service.getSslPort > 0) { 142 | ("https", service.getSslPort) 143 | } else { 144 | ("http", service.getPort) 145 | } 146 | 147 | new URI(proto, "%s:%s" format(service.getAddress, port), "/", null, null) 148 | } 149 | } 150 | 151 | trait DiscoConfig 152 | { 153 | def discoPath: String 154 | 155 | def discoAnnounce: Option[DiscoAnnounceConfig] 156 | } 157 | 158 | case class DiscoAnnounceConfig(name: String, port: Int, ssl: Boolean) 159 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/net/uri.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.net 18 | 19 | import java.net.URLEncoder 20 | 21 | // Scala api for java.net.URI 22 | 23 | object uri { 24 | 25 | type URI = java.net.URI 26 | 27 | class URIOps(u: URI) 28 | { 29 | 30 | // URI terminology reference: http://docs.oracle.com/javase/6/docs/api/java/net/URI.html 31 | 32 | def authority : String = u.getAuthority 33 | def fragment : String = u.getFragment 34 | def host : String = u.getHost 35 | def path : String = u.getPath 36 | def port : Int = u.getPort 37 | def query : String = u.getQuery 38 | def rawAuthority : String = u.getRawAuthority 39 | def rawFragment : String = u.getRawFragment 40 | def rawPath : String = u.getRawPath 41 | def rawQuery : String = u.getRawQuery 42 | def rawSSP : String = u.getRawSchemeSpecificPart 43 | def rawUserInfo : String = u.getRawUserInfo 44 | def scheme : String = u.getScheme 45 | def ssp : String = u.getSchemeSpecificPart 46 | def userInfo : String = u.getUserInfo 47 | 48 | // Aliases 49 | def schemeSpecificPart : String = ssp 50 | def rawSchemeSpecificPart : String = rawSSP 51 | 52 | def withScheme (x: String) = new URI(x, ssp, fragment) 53 | def withSSP (x: String) = new URI(scheme, x, fragment) 54 | def withFragment (x: String) = new URI(scheme, ssp, x) 55 | // Hierarchical 56 | def withAuthority (x: String) = new URI(scheme, x, path, query, fragment) 57 | def withPath (x: String) = new URI(scheme, authority, x, query, fragment) 58 | def withQuery (x: String) = new URI(scheme, authority, path, x, fragment) 59 | // Hierarchical with server-based authority 60 | def withUserInfo (x: String) = new URI(scheme, x, host, port, path, query, fragment) 61 | def withHost (x: String) = new URI(scheme, userInfo, x, port, path, query, fragment) 62 | def withPort (x: Int) = new URI(scheme, userInfo, host, x, path, query, fragment) 63 | 64 | def withScheme (f: String => String) = new URI(f(scheme), ssp, fragment) 65 | def withSSP (f: String => String) = new URI(scheme, f(ssp), fragment) 66 | def withFragment (f: String => String) = new URI(scheme, ssp, f(fragment)) 67 | // Hierarchical 68 | def withAuthority (f: String => String) = new URI(scheme, f(authority), path, query, fragment) 69 | def withPath (f: String => String) = new URI(scheme, authority, f(path), query, fragment) 70 | def withQuery (f: String => String) = new URI(scheme, authority, path, f(query), fragment) 71 | // Hierarchical with server-based authority 72 | def withUserInfo (f: String => String) = new URI(scheme, f(userInfo), host, port, path, query, fragment) 73 | def withHost (f: String => String) = new URI(scheme, userInfo, f(host), port, path, query, fragment) 74 | def withPort (f: Int => Int) = new URI(scheme, userInfo, host, f(port), path, query, fragment) 75 | 76 | // Aliases 77 | def withSchemeSpecificPart (x: String) : URI = withSchemeSpecificPart(x) 78 | def withSchemeSpecificPart (f: String => String) : URI = withSchemeSpecificPart(f) 79 | 80 | } 81 | implicit def URIOps(u: URI) = new URIOps(u) 82 | 83 | implicit val uriOrdering: Ordering[URI] = Ordering.by(_.toString) 84 | 85 | class TraversableOnceQueryStringOps[X, F[Y] <: TraversableOnce[Y]](xs: F[X]) 86 | { 87 | def toQueryString[T](implicit ev: X <:< (String, T)): String = xs.toSeq map { 88 | case (k: String, v: Any) => 89 | "%s=%s" format(URLEncoder.encode(k, "UTF-8"), URLEncoder.encode(v.toString, "UTF-8")) 90 | } mkString "&" 91 | } 92 | implicit def TraversableOnceQueryStringOps[X, F[Y] <: TraversableOnce[Y]](xs: F[X]) = new 93 | TraversableOnceQueryStringOps[X, F](xs) 94 | 95 | // Not sure why this is needed, but the compiler can't find TOQSO when given a Map... 96 | class MapQueryStringOps[T](m: Map[String, T]) 97 | { 98 | def toQueryString = new TraversableOnceQueryStringOps(m).toQueryString 99 | } 100 | implicit def Map2TraversableOnceQueryStringOps[T](m: Map[String, T]) = new MapQueryStringOps(m) 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/untyped.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.Predef._ 20 | import java.{util => ju, lang => jl} 21 | import scala.collection.immutable 22 | import scala.collection.JavaConverters._ 23 | import scala.{collection => _collection} 24 | 25 | // Casts and conversions from "untyped" data, like what we get from parsing json, yaml, etc. 26 | // Shamelessly inspired by python. 27 | object untyped { 28 | 29 | // Casts: null input throws NPE 30 | 31 | def bool (x: Any, onNull: => Boolean = throwNPE): Boolean = (x mapNull onNull).asInstanceOf[jl.Boolean].booleanValue 32 | def int (x: Any, onNull: => Int = throwNPE): Int = (x mapNull onNull).asInstanceOf[jl.Number].intValue 33 | def long (x: Any, onNull: => Long = throwNPE): Long = (x mapNull onNull).asInstanceOf[jl.Number].longValue 34 | def float (x: Any, onNull: => Float = throwNPE): Float = (x mapNull onNull).asInstanceOf[jl.Number].floatValue 35 | def double (x: Any, onNull: => Double = throwNPE): Double = (x mapNull onNull).asInstanceOf[jl.Number].doubleValue 36 | 37 | def str(x: Any, onNull: => String = throwNPE): String = { 38 | tryCasts(x mapNull onNull)( 39 | _.asInstanceOf[String], 40 | x => new String(x.asInstanceOf[Array[Byte]]) 41 | ) 42 | } 43 | 44 | // TODO Tests 45 | def dict(x: Any, onNull: => Dict = throwNPE): Dict = { 46 | tryCasts(x mapNull onNull)( 47 | _.asInstanceOf[Dict], 48 | _.asInstanceOf[ju.Map[String,Any]].asScala.toMap, 49 | list(_).asInstanceOf[_collection.Seq[(String,Any)]].toMap 50 | ) 51 | } 52 | 53 | // TODO Tests 54 | def list(x: Any, onNull: => UList = throwNPE): UList = { 55 | tryCasts(x mapNull onNull)( 56 | _.asInstanceOf[UList], 57 | _.asInstanceOf[ju.List[Any]].asScala.toList, 58 | _.asInstanceOf[Array[Any]].toList, 59 | _.asInstanceOf[TraversableOnce[Any]].toList 60 | ) 61 | } 62 | 63 | type Dict = immutable.Map[String, Any] 64 | def Dict(elems: (String, Any)*) = immutable.Map.apply[String, Any](elems : _*) 65 | 66 | type UList = immutable.Seq[Any] 67 | def UList(elems: (String, Any)*) = immutable.Seq.apply[Any](elems : _*) 68 | 69 | // TODO Tests 70 | def tryCasts[X,Y](x: X)(f: X => Y, fs: (X => Y)*): Y = { 71 | for (g <- f +: fs) { 72 | try { 73 | return g(x) 74 | } catch { 75 | case e: ClassCastException => () 76 | } 77 | } 78 | throw new ClassCastException("No casts succeeded for: %s" format x.asInstanceOf[AnyRef].getClass) 79 | } 80 | 81 | // Nested collections 82 | 83 | // Recursively normalize dict-like objects into Maps and all other collections into Seqs 84 | def normalize(x: Any): Any = gfold(x)(Map.empty) 85 | 86 | // Recursively normalize dict-like objects into ju.Maps and all other collections into ju.Lists 87 | def normalizeJava(x: Any): Any = gfold(x) { 88 | case x: Map[_,_] => x.asJava : ju.Map[_,_] 89 | case x: Seq[_] => x.asJava : ju.List[_] 90 | } 91 | 92 | // Generically "fold" x as a tree of dict- and list-like objects, normalizing them to Maps and Seqs so that f is 93 | // guaranteed to see all dict-likes as Maps and list-likes as Seqs (e.g. no ju.Maps or Sets). Nodes on which f isn't 94 | // defined are mapped through identity. 95 | def gfold(x: Any)(_f: PartialFunction[Any, Any]): Any = { 96 | val f = _f orElse { case x => x } : PartialFunction[Any, Any] 97 | x match { 98 | 99 | case x: Map[_,_] => f(x.mapValues(gfold(_)(f))) 100 | case x: Seq[_] => f(x.map(gfold(_)(f))) 101 | 102 | case x: _collection.Map[_,_] => gfold(x.toMap: Map[_,_])(f) // -> Predef.Map (= immutable.Map) 103 | case x: _collection.TraversableOnce[_] => gfold(x.toSeq: Seq[_])(f) // -> Predef.Seq (= immutable.Seq) 104 | case x: Array[_] => gfold(x.toSeq)(f) // Arrays are ill behaved, make them Seqs 105 | case x: ju.Map[_,_] => gfold(x.asScala)(f) // Java -> scala 106 | case x: jl.Iterable[_] => gfold(x.asScala)(f) // Java -> scala 107 | 108 | case x => f(x) 109 | 110 | } 111 | } 112 | 113 | // Null 114 | 115 | def noNull[X](x: X): X = x mapNull throwNPE 116 | 117 | def throwNPE = throw new NullPointerException("Non-null input required") 118 | 119 | } 120 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/CollectionTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.Predef._ 20 | import com.metamx.common.scala.collection.implicits._ 21 | import com.metamx.common.scala.collection.untilEmpty 22 | import com.simple.simplespec.Matchers 23 | import org.junit.Test 24 | 25 | class CollectionTest extends Matchers { 26 | 27 | @Test def untilEmptyTest { 28 | var n = 5 29 | val xs = untilEmpty { 30 | 0 until n withEffect { _ => 31 | n -= 1 32 | } 33 | } 34 | n must be(4) 35 | xs.take(5).toList must be(List(0,1,2,3,4)) 36 | n must be(4) 37 | xs.take(6).toList must be(List(0,1,2,3,4, 0)) 38 | n must be(3) 39 | xs.take(10).toList must be(List(0,1,2,3,4, 0,1,2,3, 0)) 40 | n must be(2) 41 | xs.toList must be(List(0,1,2,3,4, 0,1,2,3, 0,1,2, 0,1, 0)) 42 | n must be(-1) 43 | } 44 | 45 | @Test def onCompleteBasicStream { 46 | var complete = false 47 | val xs: Stream[Int] = Stream(1,2,3) onComplete { _ => complete = true } 48 | complete must be(false) 49 | xs.toList must be(List(1,2,3)) 50 | complete must be(true) 51 | } 52 | 53 | @Test def onCompleteBasicIterator { 54 | var complete = false 55 | val xs: Iterator[Int] = Iterator(1,2,3) onComplete { _ => complete = true } 56 | complete must be(false) 57 | xs.toList must be(List(1,2,3)) 58 | complete must be(true) 59 | } 60 | 61 | @Test def onCompleteBasicList { 62 | var complete = false 63 | val xs: List[Int] = List(1,2,3) onComplete { _ => complete = true } 64 | complete must be(true) 65 | xs.toList must be(List(1,2,3)) 66 | complete must be(true) 67 | } 68 | 69 | @Test def takeUntilStream { 70 | { Stream.from(0) map { x => assert(x < 3); x } takeUntil (_ == 2) toList } must be(List(0,1,2)) 71 | evaluating { Stream.from(0) map { x => assert(x < 3); x } takeWhile (_ < 3) toList } must throwAn[Error] 72 | } 73 | 74 | @Test def takeUntilIterator { 75 | { Iterator.from(0) map { x => assert(x < 3); x } takeUntil (_ == 2) toList } must be(List(0,1,2)) 76 | evaluating { Iterator.from(0) map { x => assert(x < 3); x } takeWhile (_ < 3) toList } must throwAn[Error] 77 | } 78 | 79 | @Test def takeUntilList { 80 | evaluating { List(0,1,2,3) map { x => assert(x < 3); x } takeUntil (_ == 2) toList } must throwAn[Error] 81 | evaluating { List(0,1,2,3) map { x => assert(x < 3); x } takeWhile (_ < 3) toList } must throwAn[Error] 82 | } 83 | 84 | @Test def testToMapOfSeqs() { 85 | val tuples = Seq("x" -> 1, "y" -> 2, "x" -> 3, "y" -> 2) 86 | tuples.toMapOfSeqs must be(Map("x" -> Seq(1, 3), "y" -> Seq(2, 2))) 87 | } 88 | 89 | @Test def testToMapOfSets() { 90 | val tuples = Seq("x" -> 1, "y" -> 2, "x" -> 3, "y" -> 2) 91 | tuples.toMapOfSets must be(Map("x" -> Set(1, 3), "y" -> Set(2))) 92 | } 93 | 94 | @Test def testOnlyElement() { 95 | Seq("foo").onlyElement must be("foo") 96 | Seq("foo").iterator.onlyElement must be("foo") 97 | 98 | evaluating { 99 | Seq().onlyElement 100 | } must throwAn[IllegalArgumentException] 101 | 102 | evaluating { 103 | Seq("foo", "bar").onlyElement 104 | } must throwAn[IllegalArgumentException] 105 | 106 | evaluating { 107 | Seq("foo", "bar").iterator.onlyElement 108 | } must throwAn[IllegalArgumentException] 109 | } 110 | 111 | @Test def testStrictMapValues() { 112 | var n = 0 113 | val m = Map("foo" -> 3, "bar" -> 4) 114 | val m2 = m.strictMapValues(x => { n += 1; x.toString }) 115 | n must be(2) 116 | m2 must be(Map("foo" -> "3", "bar" -> "4")) 117 | } 118 | 119 | @Test def testStrictFilterKeys() { 120 | var n = 0 121 | val m = Map("foo" -> 3, "bar" -> 4) 122 | val m2 = m.strictFilterKeys(x => { n += 1; x == "bar" }) 123 | n must be(2) 124 | m2 must be(Map("bar" -> 4)) 125 | } 126 | 127 | @Test def testChunked() { 128 | val xs = Seq(1, 2, 3, 4, 5) 129 | val grouped = xs.grouped(2) 130 | val chunked = xs.chunked(0)((a, _) => a + 1)(_ <= 2) 131 | chunked.toSeq must be(grouped.toSeq) 132 | } 133 | 134 | @Test def testChunkedFailure() { 135 | val xs = Seq(1, 2, 3, 4, 5) 136 | evaluating { 137 | xs.chunked(0)((a, _) => a + 3)(_ <= 2) 138 | } must throwAn[IllegalArgumentException]("""single element refuses to chunk""") 139 | } 140 | 141 | } 142 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/Logger.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2012 Daniel Lundin 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import org.slf4j.{LoggerFactory, Logger => Slf4jLogger} 20 | import org.slf4j.spi.{LocationAwareLogger => Slf4jLocationAwareLogger} 21 | 22 | /** 23 | * Factory for concrete Logger instances 24 | */ 25 | object Logger { 26 | 27 | /** Returns a Logger for the given class */ 28 | def apply(clazz: Class[_]): Logger = { 29 | LoggerFactory.getLogger(clazz) match { 30 | case logger: Slf4jLocationAwareLogger => new LocationAwareLogger(logger) 31 | case logger @ _ => new BasicLogger(logger) 32 | } 33 | } 34 | } 35 | 36 | /** 37 | * Main Logger API. Base class for loggers 38 | * 39 | * Note: Don't use this class directly, use the [[com.metamx.common.scala.Logging]] trait instead. 40 | */ 41 | abstract class Logger { 42 | /** Name of the underlying logger */ 43 | lazy val name = logger.getName 44 | 45 | /** The underlying logger as provided by slf4j */ 46 | protected val logger: Slf4jLogger 47 | 48 | /** A function that logs a message with an exception */ 49 | protected type LogFunc = (String, Throwable) => Unit 50 | 51 | // Concrete implementations of logger functions 52 | protected val logTrace: LogFunc 53 | protected val logDebug: LogFunc 54 | protected val logInfo: LogFunc 55 | protected val logWarn: LogFunc 56 | protected val logError: LogFunc 57 | 58 | // Exposes some config of underlying logger 59 | def isTraceEnabled = logger.isTraceEnabled 60 | def isDebugEnabled = logger.isDebugEnabled 61 | def isInfoEnabled = logger.isInfoEnabled 62 | def isWarnEnabled = logger.isWarnEnabled 63 | def isErrorEnabled = logger.isErrorEnabled 64 | 65 | /** Format a string using params, if any, otherwise use the string as-is */ 66 | @inline protected final def format(fmt: String, params: Seq[Any]) = { 67 | if (params.nonEmpty) fmt.format(params:_*) else fmt 68 | } 69 | 70 | /** Log trace message */ 71 | def trace(message: String, params: Any*) { 72 | if (logger.isTraceEnabled) logTrace(format(message, params), null) 73 | } 74 | 75 | /** Log trace message with an exception */ 76 | def trace(thrown: Throwable, message: String, params: Any*) { 77 | if (logger.isTraceEnabled) logTrace(format(message, params), thrown) 78 | } 79 | 80 | /** Log debug message */ 81 | def debug(message: String, params: Any*) { 82 | if (logger.isDebugEnabled) logDebug(format(message, params), null) 83 | } 84 | 85 | /** Log debug message with an exception */ 86 | def debug(thrown: Throwable, message: String, params: Any*) { 87 | if (logger.isDebugEnabled) logDebug(format(message, params), thrown) 88 | } 89 | 90 | /** Log info message */ 91 | def info(message: String, params: Any*) { 92 | if (logger.isInfoEnabled) logInfo(format(message, params), null) 93 | } 94 | 95 | /** Log info message with an exception */ 96 | def info(thrown: Throwable, message: String, params: Any*) { 97 | if (logger.isInfoEnabled) logInfo(format(message, params), thrown) 98 | } 99 | 100 | /** Log warn message */ 101 | def warn(message: String, params: Any*) { 102 | if (logger.isWarnEnabled) logWarn(format(message, params), null) 103 | } 104 | 105 | /** Log warning message with an exception */ 106 | def warn(thrown: Throwable, message: String, params: Any*) { 107 | if (logger.isWarnEnabled) logWarn(format(message, params), thrown) 108 | } 109 | 110 | /** Log error message */ 111 | def error(message: String, params: Any*) { 112 | if (logger.isErrorEnabled) logError(format(message, params), null) 113 | } 114 | 115 | /** Log error message with an exception */ 116 | def error(thrown: Throwable, message: String, params: Any*) { 117 | if (logger.isErrorEnabled) logError(format(message, params), thrown) 118 | } 119 | } 120 | 121 | /** 122 | * Logger without location awareness 123 | */ 124 | protected final class BasicLogger(protected val logger: Slf4jLogger) extends Logger { 125 | protected val logTrace: LogFunc = logger.trace(_, _) 126 | protected val logDebug: LogFunc = logger.debug(_, _) 127 | protected val logInfo: LogFunc = logger.info(_, _) 128 | protected val logWarn: LogFunc = logger.warn(_, _) 129 | protected val logError: LogFunc = logger.error(_, _) 130 | } 131 | 132 | /** 133 | * Logger using slf4j's location aware logger. 134 | */ 135 | protected final class LocationAwareLogger(protected val logger: Slf4jLocationAwareLogger) extends Logger { 136 | import Slf4jLocationAwareLogger.{ERROR_INT, WARN_INT, INFO_INT, DEBUG_INT, TRACE_INT} 137 | private val fqcn = classOf[LocationAwareLogger].getCanonicalName 138 | protected val logTrace: LogFunc = logger.log(null, fqcn, TRACE_INT, _, null, _) 139 | protected val logDebug: LogFunc = logger.log(null, fqcn, DEBUG_INT, _, null, _) 140 | protected val logInfo: LogFunc = logger.log(null, fqcn, INFO_INT, _, null, _) 141 | protected val logWarn: LogFunc = logger.log(null, fqcn, WARN_INT, _, null, _) 142 | protected val logError: LogFunc = logger.log(null, fqcn, ERROR_INT, _, null, _) 143 | } 144 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/time/Intervals.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.time 18 | 19 | import com.github.nscala_time.time.Imports._ 20 | import com.metamx.common.scala.Predef._ 21 | import scala.collection.IndexedSeqLike 22 | import scala.collection.generic.CanBuildFrom 23 | import scala.collection.immutable.Vector 24 | import scala.collection.mutable.{Builder, ListBuffer} 25 | import scala.util.control.Breaks 26 | 27 | // Property: Intervals.intervals are separated (i.e. disjoint and non-contiguous), ascending, and all nonempty 28 | 29 | class Intervals private (self: Vector[Interval]) 30 | extends IndexedSeq[Interval] with IndexedSeqLike[Interval, Intervals] with Serializable { 31 | 32 | Intervals.validate(self) 33 | 34 | // IndexedSeqLike 35 | override def apply(i: Int) = self.apply(i) 36 | override def length = self.length 37 | override def newBuilder = Intervals.newBuilder 38 | 39 | def duration = new Duration(self.map(_.millis).sum) 40 | 41 | def overlaps(interval: Interval) = self.exists(_ overlaps interval) 42 | 43 | def overlaps(dt: DateTime) = self.exists(_ contains dt) 44 | 45 | def earliest(_duration: Duration) = new Intervals(Vector() ++ new ListBuffer[Interval].withEffect { results => 46 | val breaks = new Breaks; import breaks.{break, breakable} 47 | var duration = _duration 48 | breakable { 49 | for (i <- self.iterator) { 50 | if (duration.millis == 0) { 51 | break 52 | } else { 53 | results += new Interval(i.start, Seq(i.end, i.start + duration).min) withEffect { result => 54 | duration -= result.duration 55 | } 56 | } 57 | } 58 | } 59 | }) 60 | 61 | def latest(_duration: Duration) = new Intervals(Vector() ++ new ListBuffer[Interval].withEffect { results => 62 | val breaks = new Breaks; import breaks.{break, breakable} 63 | var duration = _duration 64 | breakable { 65 | for (i <- self.reverseIterator) { 66 | if (duration.millis == 0) { 67 | break 68 | } else { 69 | results.insert(0, new Interval(i.start max (i.end - duration), i.end) withEffect { result => 70 | duration -= result.duration 71 | }) 72 | } 73 | } 74 | } 75 | }) 76 | 77 | def -- (that: Iterable[Interval]) = new Intervals(Vector.newBuilder[Interval].withEffect { results => 78 | var is = self 79 | val js = that.iterator.buffered 80 | def i = is.head 81 | def j = js.head 82 | while (is.nonEmpty) { 83 | while (js.nonEmpty && (j isBefore i)) { 84 | js.next 85 | } 86 | if (js.isEmpty || (j isAfter i)) { 87 | results += i 88 | is = is.tail 89 | } else { 90 | val overlap = i overlap j ensuring (_ != null, "Expected overlap: %s, %s" format (i,j)) 91 | val (a, b) = (i.start to overlap.start, overlap.end to i.end) 92 | is = is.tail 93 | if (a.millis > 0) results += a 94 | if (b.millis > 0) is = b +: is 95 | } 96 | } 97 | }.result) 98 | 99 | } 100 | 101 | // Companion object in the style of BitSet, which we take as the canonical 0-type-arg collection type 102 | object Intervals { 103 | 104 | // Factory methods 105 | val empty : Intervals = apply() 106 | def apply(is: Interval*) : Intervals = apply(is) 107 | def apply(is: TraversableOnce[Interval]) : Intervals = newBuilder.withEffect { b => is foreach (b += _) }.result 108 | 109 | // An efficient builder in terms of union (loglinear time, linear space) 110 | implicit def canBuildFrom = new CanBuildFrom[Intervals, Interval, Intervals] { 111 | def apply(from: Intervals) = newBuilder 112 | def apply() = newBuilder 113 | } 114 | def newBuilder: Builder[Interval, Intervals] = Vector.newBuilder mapResult { xs => 115 | new Intervals(union(xs)) 116 | } 117 | 118 | // Union a sequence of intervals so that they satisfy: separated, ascending, all nonempty. 119 | // Loglinear time, linear space. 120 | def union(intervals: Iterable[Interval]) = Vector[Interval]() ++ { 121 | // Sort input intervals by start time and build up result incrementally 122 | val separated = new ListBuffer[Interval] 123 | var current = None : Option[Interval] 124 | val ascending = intervals.toIndexedSeq sortBy ((i: Interval) => i.start.getMillis) 125 | for (i <- ascending; if i.millis > 0) { 126 | current match { 127 | case None => current = Some(i) 128 | case Some(j) if (j gap i) == null => current = Some(j.start to (j.end max i.end)) 129 | case Some(j) => current = Some(i); separated += j 130 | } 131 | } 132 | current foreach { separated += _ } 133 | separated 134 | } 135 | 136 | // Ensure that intervals satisfy: separated, ascending, all nonempty 137 | def validate(intervals: Iterable[Interval]) = intervals withEffect { _ => 138 | var prev = None : Option[Interval] 139 | for (b <- intervals) { 140 | assert(b.millis > 0, "Intervals must be all nonempty: %s" format b) 141 | for (a <- prev) { 142 | assert((a gap b) != null, "Intervals must be separated: %s, %s" format (a,b)) 143 | assert(a isBefore b, "Intervals must be ascending: %s, %s" format (a,b)) 144 | } 145 | prev = Some(b) 146 | } 147 | } 148 | 149 | } 150 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/collection/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala 18 | 19 | import com.metamx.common.scala.Predef.EffectOps 20 | import scala.collection.generic.{CanBuildFrom => CBF} 21 | import scala.collection.MapLike 22 | import scala.collection.TraversableLike 23 | 24 | package object collection { 25 | 26 | /** 27 | * Build a stream by repeatedly evaluating xs until it's empty 28 | * Space safe: O(max xs.size) space 29 | */ 30 | def untilEmpty[X](xs: => Iterable[X]): Stream[X] = (new Iterator[Iterator[X]] { 31 | var _xs = None : Option[Iterable[X]] 32 | def hasNext = (_xs orElse { Some(xs) withEffect { _xs = _ } }).get.nonEmpty 33 | def next = (_xs orElse Some(xs) withEffect { _ => _xs = None }).get.iterator 34 | }: Iterator[Iterator[X]]).flatten.toStream 35 | 36 | class TraversableOnceOps[X, F[Y] <: TraversableOnce[Y]](xs: F[X]) { 37 | 38 | def ifEmpty (f: => Any): F[X] = { if (xs.isEmpty) f; xs } 39 | def ifNonEmpty (f: => Any): F[X] = { if (xs.nonEmpty) f; xs } 40 | 41 | def toMapOfSets[K, V](implicit ev: X <:< (K, V)): Map[K, Set[V]] = { 42 | for ((k, vs) <- xs.toSeq.groupBy(_._1)) yield { 43 | (k, vs.iterator.map(_._2).toSet) 44 | } 45 | } 46 | 47 | def toMapOfSeqs[K, V](implicit ev: X <:< (K, V)): Map[K, Seq[V]] = { 48 | for ((k, vs) <- xs.toSeq.groupBy(_._1)) yield { 49 | (k, vs.map(_._2)) 50 | } 51 | } 52 | 53 | def onlyElement: X = { 54 | val iter = xs.toIterator 55 | if (!iter.hasNext) { 56 | throw new IllegalArgumentException("expected single element") 57 | } 58 | val elt = iter.next() 59 | if (iter.hasNext) { 60 | throw new IllegalArgumentException("expected single element") 61 | } 62 | elt 63 | } 64 | 65 | def maxByOpt[Z](f: X => Z)(implicit cmp: Ordering[Z]): Option[X] = { 66 | if (xs.isEmpty) { 67 | None 68 | } else { 69 | Some(xs.maxBy(f)(cmp)) 70 | } 71 | } 72 | 73 | def minByOpt[Z](f: X => Z)(implicit cmp: Ordering[Z]): Option[X] = { 74 | if (xs.isEmpty) { 75 | None 76 | } else { 77 | Some(xs.minBy(f)(cmp)) 78 | } 79 | } 80 | } 81 | implicit def TraversableOnceOps[X, F[Y] <: TraversableOnce[Y]](xs: F[X]) = new TraversableOnceOps[X,F](xs) 82 | 83 | class TraversableLikeOps[X, F[Y] <: TraversableLike[Y, F[Y]]](xs: F[X]) { 84 | 85 | /** 86 | * For preserving laziness, e.g. 87 | * 88 | * Stream.fill(n) { ... } onComplete { ns => log.debug("Evaluation complete: %s", ns) } 89 | */ 90 | def onComplete(f: F[X] => Unit)(implicit bf: CBF[F[X], X, F[X]]): F[X] = { 91 | xs ++ new Iterator[X] { 92 | def hasNext = { f(xs); false } 93 | def next = throw new Exception("Unreachable") 94 | } 95 | } 96 | 97 | /** 98 | * Similar to takeWhile(!p), but include the last-tested element and don't evaluate beyond it: 99 | * 100 | * Stream.from(0) map { x => assert(x < 3); x } takeUntil (_ == 2) toList --> List(0,1,2) 101 | * Stream.from(0) map { x => assert(x < 3); x } takeWhile (_ < 3) toList --> assertion error: 3 < 3 102 | */ 103 | def takeUntil(p: X => Boolean)(implicit bf: CBF[F[X], X, F[X]]): F[X] = { 104 | object last { var x: X = _ } // (Can't use _ for local vars; stick it into a field of a local object instead) 105 | xs.takeWhile { x => last.x = x; !p(x) } ++ Iterator.fill(1) { last.x } // (Defer eval of last.x) 106 | } 107 | 108 | /** 109 | * Chunk (un-flatten) objects greedily using an accumulator. When the continue-chunk condition returns false, the 110 | * element for which it returned false will become the start of a new chunk. It will then be re-evaluated with a 111 | * new accumulator. If this re-evaluation fails to return true, an exception will be thrown. 112 | * 113 | * Example: xs.grouped(2) == xs.chunked(0)((a, _) => a + 1)(_ <= 2) 114 | * 115 | * @param z zero for the accumlator type 116 | * @param accumulate accumulation function 117 | * @param continueChunk continue-chunk condition 118 | * @tparam Acc accumulator type 119 | * @return chunked objects 120 | */ 121 | def chunked[Acc](z: Acc)(accumulate: (Acc, X) => Acc)(continueChunk: Acc => Boolean): Iterator[F[X]] = { 122 | var acc = z 123 | val (chunk, rest) = xs span { 124 | x => 125 | acc = accumulate(acc, x) 126 | continueChunk(acc) 127 | } 128 | if (chunk.isEmpty) { 129 | if (rest.nonEmpty) { 130 | throw new IllegalArgumentException("single element refuses to chunk") 131 | } else { 132 | Iterator.empty 133 | } 134 | } else { 135 | Iterator(chunk) ++ new TraversableLikeOps(rest).chunked(z)(accumulate)(continueChunk) 136 | } 137 | } 138 | } 139 | implicit def TraversableLikeOps[X, F[Y] <: TraversableLike[Y, F[Y]]](xs: F[X]) = new TraversableLikeOps[X,F](xs) 140 | 141 | class MapLikeOps[A, +B, +Repr <: MapLike[A, B, Repr] with scala.collection.Map[A, B]](m: MapLike[A, B, Repr]) { 142 | 143 | def strictMapValues[C, That](f: B => C)(implicit bf: CBF[Repr, (A, C), That]): That = { 144 | m.map(kv => (kv._1, f(kv._2))) 145 | } 146 | 147 | def strictFilterKeys(f: A => Boolean): Repr = { 148 | m.filter(kv => f(kv._1)) 149 | } 150 | 151 | def headOpt[V](key: A)(implicit ev: B <:< Iterable[V]) = m.get(key).flatMap(_.headOption) 152 | 153 | def extractPrefixed(prefix: String)(implicit ev: A =:= String): Map[String, B] = { 154 | val prefixDot = "%s.".format(prefix) 155 | m.collect { 156 | case (k: String, v) if k.startsWith(prefixDot) => 157 | k.stripPrefix(prefixDot) -> v 158 | }.toMap 159 | } 160 | 161 | } 162 | implicit def MapLikeOps[A, B, Repr <: MapLike[A, B, Repr] with scala.collection.Map[A, B]](m: MapLike[A, B, Repr]) = new MapLikeOps[A, B, Repr](m) 163 | 164 | // Mimic TravserableLikeOps for Iterator, which isn't TraversableLike 165 | class IteratorOps[X](xs: Iterator[X]) { 166 | 167 | def onComplete(f: Iterator[X] => Unit): Iterator[X] = xs ++ new Iterator[X] { 168 | def hasNext = { f(xs); false } 169 | def next = throw new Exception("Unreachable") 170 | } 171 | 172 | def takeUntil(p: X => Boolean): Iterator[X] = { 173 | object last { var x: X = _ } // (Can't use _ for local vars; stick it into a field of a local object instead) 174 | xs.takeWhile { x => last.x = x; !p(x) } ++ Iterator.fill(1) { last.x } 175 | } 176 | 177 | } 178 | implicit def IteratorOps[X](xs: Iterator[X]) = new IteratorOps[X](xs) 179 | 180 | } 181 | -------------------------------------------------------------------------------- /src/main/scala/com/metamx/common/scala/db/DB.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.db 18 | 19 | import com.github.nscala_time.time.Imports._ 20 | import com.mchange.v2.c3p0.DataSources 21 | import com.metamx.common.lifecycle.{LifecycleStart, LifecycleStop} 22 | import com.metamx.common.scala.LateVal.LateVal 23 | import com.metamx.common.scala.Logging 24 | import com.metamx.common.scala.Predef._ 25 | import com.metamx.common.scala.collection._ 26 | import com.metamx.common.scala.control.retryOnError 27 | import com.metamx.common.scala.exception.{raises, _} 28 | import com.metamx.common.scala.untyped.Dict 29 | import javax.sql.DataSource 30 | import org.skife.jdbi.v2._ 31 | import org.skife.jdbi.v2.exceptions.{CallbackFailedException, StatementException, TransactionFailedException} 32 | import org.skife.jdbi.v2.tweak.HandleCallback 33 | import scala.collection.JavaConverters._ 34 | import scala.collection.mutable.Buffer 35 | import scala.util.DynamicVariable 36 | 37 | // TODO Extract not-mysql-specific stuff from MySQLDB.createIsTransient, provide base implementation here 38 | 39 | abstract class DB(config: DBConfig) extends Logging { 40 | 41 | def createIsTransient: Throwable => Boolean 42 | 43 | def createTable(table: String, decls: Seq[String]) 44 | 45 | def select(sql: String, args: Any*): IndexedSeq[Dict] = { 46 | inTransaction { 47 | log.trace("select: %s, %s", oneLineSql(sql), args) 48 | val query = configured(h.createQuery(sql)) 49 | for (i <- args.indices) { 50 | query.bind(i, args(i).asInstanceOf[AnyRef]) 51 | } 52 | val results = query.list 53 | IndexedSeq[Dict]() ++ results.asScala.map(_.asScala.toMap) withEffect { rows => 54 | log.trace("%s rows <- %s, %s", rows.length, oneLineSql(sql), args) 55 | } 56 | } 57 | } 58 | 59 | // Streaming select, paginated by uniqueKey, which must be a selected name. Hacky, but useful. Example: 60 | // 61 | // stream(uniqueKey="id", select="id,uri,mtime", from="files", where="uri = ? and mtime > ?", uri, mtime) 62 | // 63 | // Atomic if all uses of result Stream are enclosed within inTransaction{...}. 64 | def stream(uniqueKey: String, select: String, from: String, where: String, args: Any*): Stream[Dict] = { 65 | log.trace("stream[%s]: select [%s] from [%s] where [%s], %s", uniqueKey, select, from, where, args) 66 | val _fetchSize = fetchSize.value // Snapshot fetchSize: our result stream can be evaluated in many dynamic scopes 67 | var last = None : Option[Any] 68 | Stream.from(0) map { i => 69 | this.select( 70 | "select %s from %s where (%s) and %s order by %s limit ?" format ( 71 | select, 72 | from, 73 | if (where.nonEmpty) where else "true", 74 | last map (_ => "%s > ?" format uniqueKey) getOrElse "true", 75 | uniqueKey 76 | ), 77 | args ++ last ++ Seq(_fetchSize) : _* 78 | ) withEffect { rows => 79 | last = rows.lastOption map (_(uniqueKey)) 80 | } 81 | } takeUntil (_.length < _fetchSize) flatten 82 | } 83 | 84 | val fetchSize = new DynamicVariable[Int](config.fetchSize) 85 | 86 | def execute(sql: String, args: Any*): Int = { 87 | inTransaction { 88 | log.trace("execute: %s, %s", oneLineSql(sql), args) 89 | val query = configured(h.createStatement(sql)) 90 | for (i <- args.indices) { 91 | query.bind(i, args(i).asInstanceOf[AnyRef]) 92 | } 93 | query.execute 94 | } 95 | } 96 | 97 | // Atomic 98 | def batch(sql: String, argss: Iterable[Seq[Any]]): Int = { 99 | inTransaction { 100 | argss.grouped(config.batchSize).map { argss => // (Is this necessary? Does something below already chunk?) 101 | log.trace("batch: %s, %s", oneLineSql(sql), argss) 102 | val query = configured(h.prepareBatch(sql)) 103 | for (args <- argss) { 104 | query.add(args.map(_.asInstanceOf[AnyRef]): _*) 105 | } 106 | query.execute.sum 107 | }.sum 108 | } 109 | } 110 | 111 | // Ensure a transaction. Reentrant. Outermost inTransaction retries on "transient" errors (using createIsTransient). 112 | def inTransaction[X](body: => X): X = currentTransationStatus.value match { 113 | case Some(_tx) => body 114 | case None => 115 | retryOnError(createIsTransient) { 116 | singleHandle { 117 | h.inTransaction(new TransactionCallback[X] { def inTransaction(_h: Handle, _tx: TransactionStatus) = { 118 | currentTransationStatus.withValue(Some(_tx)) { body } 119 | }}) mapException { 120 | case e: TransactionFailedException => e.getCause // Wrapped exceptions are anti-useful; unwrap them 121 | } 122 | } 123 | } 124 | } 125 | 126 | // Fix a single handle. Reentrant. Doesn't retry on transient errors. 127 | def singleHandle[X](body: => X): X = currentHandle.value match { 128 | case Some(_h) => body 129 | case None => 130 | dbi.withHandle(new HandleCallback[X] { def withHandle(_h: Handle) = { 131 | currentHandle.withValue(Some(_h)) { body } 132 | }}) mapException { 133 | case e: CallbackFailedException => e.getCause // Wrapped exceptions are anti-useful; unwrap them 134 | } 135 | } 136 | 137 | private[this] val currentHandle = new DynamicVariable[Option[Handle]](None) 138 | private[this] val currentTransationStatus = new DynamicVariable[Option[TransactionStatus]](None) 139 | 140 | // Convenient, idiomatic names for currentHandle, currentTransationStatus 141 | def h : Handle = currentHandle.value.get 142 | def tx : TransactionStatus = currentTransationStatus.value.get 143 | 144 | def configured[X <: SQLStatement[X]](x: X): X = { // TODO Figure out jdbi's statement customizers 145 | x.setQueryTimeout(config.queryTimeout.seconds.toInt) 146 | } 147 | 148 | def oneLineSql(sql: String) = """\s*\n\s*""".r.replaceAllIn(sql, " ").trim 149 | 150 | @LifecycleStart 151 | def start { 152 | log.info("Starting") 153 | dbds assign DataSources.pooledDataSource( 154 | DataSources.unpooledDataSource( 155 | config.uri, 156 | config.user, 157 | config.password 158 | ) 159 | ) 160 | log.info("Connecting to %s", config.uri) 161 | dbi assign new DBI(dbds) 162 | schema.create 163 | } 164 | 165 | @LifecycleStop 166 | def stop { 167 | DataSources.destroy(dbds) 168 | } 169 | 170 | val dbds = new LateVal[DataSource] 171 | val dbi = new LateVal[DBI] 172 | 173 | lazy val schema = new { 174 | 175 | lazy val tables = Buffer[(String, Seq[String])]() // (Maintain declaration order for creation) 176 | 177 | def create { 178 | for ((table, decls) <- tables) { 179 | if (exists(table)) { 180 | log.info("Table already exists: %s", table) 181 | } else { 182 | log.info("Creating table: %s", table) 183 | createTable(table, decls) 184 | } 185 | } 186 | } 187 | 188 | def exists(table: String) = !raises[StatementException] { select("select * from %s limit 0" format table) } 189 | 190 | } 191 | 192 | } 193 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/event/AlertAggregatorTest.scala: -------------------------------------------------------------------------------- 1 | package com.metamx.common.scala.event 2 | 3 | import com.metamx.common.scala.Logging 4 | import com.metamx.common.scala.Predef._ 5 | import com.metamx.common.scala.untyped._ 6 | import com.metamx.emitter.core.{Emitter, Event} 7 | import com.metamx.emitter.service.ServiceEmitter 8 | import com.simple.simplespec.Matchers 9 | import org.junit.Test 10 | import scala.collection.JavaConverters._ 11 | import scala.collection.mutable 12 | import scala.collection.mutable.ArrayBuffer 13 | import scala.util.Random 14 | import scala.util.control.NoStackTrace 15 | 16 | class AlertAggregatorTest extends Matchers with Logging 17 | { 18 | 19 | class TestException(message: String) extends Exception(message) with NoStackTrace 20 | { 21 | def this() = this("boo!") 22 | } 23 | 24 | def createEmitter(): (Seq[Event], ServiceEmitter) = { 25 | val buffer = new ArrayBuffer[Event] with mutable.SynchronizedBuffer[Event] 26 | val emitter = new ServiceEmitter( 27 | "service", "host", new Emitter 28 | { 29 | override def start() {} 30 | 31 | override def flush() {} 32 | 33 | override def emit(event: Event) { 34 | buffer += event 35 | } 36 | 37 | override def close() {} 38 | } 39 | ) 40 | (buffer, emitter) 41 | } 42 | 43 | @Test 44 | def testNothing() 45 | { 46 | val (events, emitter) = createEmitter() 47 | val alerts = new AlertAggregator(log) 48 | alerts.monitor.withEffect(_.start()).monitor(emitter) 49 | events must be(Nil) 50 | } 51 | 52 | @Test 53 | def testAggregationSimple() 54 | { 55 | val (events, emitter) = createEmitter() 56 | val alerts = new AlertAggregator(log) 57 | alerts.put("foo", Map("baz" -> 3)) 58 | alerts.put("foo", Map("baz" -> 3)) 59 | alerts.put("foo", Map("baz" -> 3)) 60 | alerts.put("bar", Map("baz" -> 4)) 61 | alerts.monitor.withEffect(_.start()).monitor(emitter) 62 | events.map(d => dict(normalize(d.toMap.asScala - "timestamp"))).sortBy(x => str(x("description"))) must be( 63 | Seq( 64 | Map( 65 | "feed" -> "alerts", 66 | "service" -> "service", 67 | "host" -> "host", 68 | "severity" -> "anomaly", 69 | "description" -> "bar", 70 | "data" -> Map( 71 | "sampleDetails" -> Map("baz" -> 4), 72 | "alertCount" -> 1 73 | ) 74 | ), 75 | Map( 76 | "feed" -> "alerts", 77 | "service" -> "service", 78 | "host" -> "host", 79 | "severity" -> "anomaly", 80 | "description" -> "foo", 81 | "data" -> Map( 82 | "sampleDetails" -> Map("baz" -> 3), 83 | "alertCount" -> 3 84 | ) 85 | ) 86 | ) 87 | ) 88 | } 89 | 90 | @Test 91 | def testAggregationWithExceptions() 92 | { 93 | val (events, emitter) = createEmitter() 94 | val alerts = new AlertAggregator(log) 95 | alerts.put(new TestException, "foo", Map("baz" -> 3)) 96 | alerts.put(new TestException, "foo", Map("baz" -> 3)) 97 | alerts.put("foo", Map("baz" -> 3)) 98 | alerts.put("bar", Map("baz" -> 4)) 99 | alerts.monitor.withEffect(_.start()).monitor(emitter) 100 | val eventDicts = events 101 | .map(d => dict(normalize(d.toMap.asScala - "timestamp"))) 102 | .sortBy(x => str(x("description")) + str(dict(x("data")).getOrElse("exceptionType", ""))) 103 | eventDicts must be( 104 | Seq( 105 | Map( 106 | "feed" -> "alerts", 107 | "service" -> "service", 108 | "host" -> "host", 109 | "severity" -> "anomaly", 110 | "description" -> "bar", 111 | "data" -> Map( 112 | "sampleDetails" -> Map("baz" -> 4), 113 | "alertCount" -> 1 114 | ) 115 | ), 116 | Map( 117 | "feed" -> "alerts", 118 | "service" -> "service", 119 | "host" -> "host", 120 | "severity" -> "anomaly", 121 | "description" -> "foo", 122 | "data" -> Map( 123 | "sampleDetails" -> Map("baz" -> 3), 124 | "alertCount" -> 1 125 | ) 126 | ), 127 | Map( 128 | "feed" -> "alerts", 129 | "service" -> "service", 130 | "host" -> "host", 131 | "severity" -> "anomaly", 132 | "description" -> "foo", 133 | "data" -> Map( 134 | "sampleDetails" -> Map("baz" -> 3), 135 | "exceptionType" -> (new TestException).getClass.getName, 136 | "exceptionMessage" -> "boo!", 137 | "exceptionStackTrace" -> 138 | Seq( 139 | "com.metamx.common.scala.event.AlertAggregatorTest$TestException: boo!\n" 140 | ).mkString, 141 | "alertCount" -> 2 142 | ) 143 | ) 144 | ) 145 | ) 146 | } 147 | 148 | @Test 149 | def testAggregationUponAggregation() 150 | { 151 | val (events, emitter) = createEmitter() 152 | val alerts = new AlertAggregator(log) 153 | alerts.put("foo", Map("baz" -> 3)) 154 | alerts.put("foo", Map("baz" -> 3)) 155 | alerts.monitor.withEffect(_.start()).monitor(emitter) 156 | alerts.put("foo", Map("baz" -> 3)) 157 | alerts.put("bar", Map("baz" -> 4)) 158 | alerts.monitor.withEffect(_.start()).monitor(emitter) 159 | events.map(d => dict(normalize(d.toMap.asScala - "timestamp"))).sortBy(x => str(x("description"))) must be( 160 | Seq( 161 | Map( 162 | "feed" -> "alerts", 163 | "service" -> "service", 164 | "host" -> "host", 165 | "severity" -> "anomaly", 166 | "description" -> "bar", 167 | "data" -> Map( 168 | "sampleDetails" -> Map("baz" -> 4), 169 | "alertCount" -> 1 170 | ) 171 | ), 172 | Map( 173 | "feed" -> "alerts", 174 | "service" -> "service", 175 | "host" -> "host", 176 | "severity" -> "anomaly", 177 | "description" -> "foo", 178 | "data" -> Map( 179 | "sampleDetails" -> Map("baz" -> 3), 180 | "alertCount" -> 2 181 | ) 182 | ), 183 | Map( 184 | "feed" -> "alerts", 185 | "service" -> "service", 186 | "host" -> "host", 187 | "severity" -> "anomaly", 188 | "description" -> "foo", 189 | "data" -> Map( 190 | "sampleDetails" -> Map("baz" -> 3), 191 | "alertCount" -> 1 192 | ) 193 | ) 194 | ) 195 | ) 196 | } 197 | 198 | @Test 199 | def testExceptionBindWithData() 200 | { 201 | val preferLastRandom = new Random(){ 202 | override def nextInt(count:Int): Int = 0 203 | } 204 | val (events, emitter) = createEmitter() 205 | val alerts = new AlertAggregator(log, preferLastRandom) 206 | alerts.put(new TestException("baz1"), "foo", Map("baz" -> 1)) 207 | alerts.put(new TestException("baz2"), "foo", Map("baz" -> 2)) 208 | alerts.monitor.withEffect(_.start()).monitor(emitter) 209 | val eventDicts = events 210 | .map(d => dict(normalize(d.toMap.asScala - "timestamp"))) 211 | .sortBy(x => str(x("description")) + str(dict(x("data")).getOrElse("exceptionType", ""))) 212 | eventDicts must be( 213 | Seq( 214 | Map( 215 | "feed" -> "alerts", 216 | "service" -> "service", 217 | "host" -> "host", 218 | "severity" -> "anomaly", 219 | "description" -> "foo", 220 | "data" -> Map( 221 | "sampleDetails" -> Map("baz" -> 2), 222 | "exceptionType" -> (new TestException).getClass.getName, 223 | "exceptionMessage" -> "baz2", 224 | "exceptionStackTrace" -> 225 | Seq( 226 | "com.metamx.common.scala.event.AlertAggregatorTest$TestException: baz2\n" 227 | ).mkString, 228 | "alertCount" -> 2 229 | ) 230 | ) 231 | ) 232 | ) 233 | } 234 | 235 | } 236 | -------------------------------------------------------------------------------- /src/test/scala/com/metamx/common/scala/collection/concurrent/BlockingQueueTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012 Metamarkets Group Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.metamx.common.scala.collection.concurrent 18 | 19 | import com.metamx.common.scala.concurrent._ 20 | import com.simple.simplespec.Matchers 21 | import java.nio.ByteBuffer 22 | import java.util.concurrent.atomic.AtomicLong 23 | import java.{util => ju} 24 | import org.junit.Test 25 | import scala.util.Random 26 | 27 | class BlockingQueueTest extends Matchers 28 | { 29 | 30 | @Test def testST() { 31 | testMT(size => new ByteBufferQueue(ByteBuffer.allocate(size))) 32 | testMT(size => new SizeBoundedQueue[Array[Byte]](arr => arr.length + 4, size, new ju.LinkedList[Array[Byte]])) 33 | } 34 | 35 | @Test def testMT() { 36 | testMT(size => new ByteBufferQueue(ByteBuffer.allocate(size))) 37 | testMT(size => new SizeBoundedQueue[Array[Byte]](arr => arr.length + 4, size, new ju.LinkedList[Array[Byte]])) 38 | } 39 | 40 | def testST(qF: (Int) => BlockingQueue[Array[Byte]]) { 41 | 42 | { 43 | val q = qF(10) 44 | 45 | q.poll().map(_.toList) must be(None) 46 | 47 | q.offer(Array(111.toByte, -37.toByte)) must be(true) 48 | q.offer(Array(14.toByte, 14.toByte)) must be(false) 49 | q.offer(Array(14.toByte, 14.toByte)) must be(false) 50 | q.poll().map(_.toList) must be(Some(List(111.toByte, -37.toByte))) 51 | q.poll().map(_.toList) must be(None) 52 | 53 | q.offer(Array(11.toByte, -3.toByte)) must be(true) 54 | q.poll().map(_.toList) must be(Some(List(11.toByte, -3.toByte))) 55 | 56 | q.offer(Array(7.toByte, -100.toByte)) must be(true) 57 | q.poll().map(_.toList) must be(Some(List(7.toByte, -100.toByte))) 58 | q.poll().map(_.toList) must be(None) 59 | 60 | q.offer(Array(11.toByte, 15.toByte, -20.toByte)) must be(true) 61 | q.poll().map(_.toList) must be(Some(List(11.toByte, 15.toByte, -20.toByte))) 62 | q.poll().map(_.toList) must be(None) 63 | 64 | q.offer(Array(11.toByte, 15.toByte, -20.toByte)) must be(true) 65 | q.offer(Array(14.toByte, 14.toByte)) must be(false) 66 | q.offer(Array(14.toByte, 14.toByte)) must be(false) 67 | q.poll().map(_.toList) must be(Some(List(11.toByte, 15.toByte, -20.toByte))) 68 | 69 | q.offer(Array(11.toByte, 15.toByte, -20.toByte)) must be(true) 70 | q.poll().map(_.toList) must be(Some(List(11.toByte, 15.toByte, -20.toByte))) 71 | q.drain(1).map(_.toList) must be(List()) 72 | q.poll().map(_.toList) must be(None) 73 | q.drain().map(_.toList) must be(List()) 74 | 75 | q.offer(Array(15.toByte, 20.toByte)) must be(true) 76 | q.offer(Array()) must be(true) 77 | q.poll().map(_.toList) must be(Some(List(15.toByte, 20.toByte))) 78 | q.poll().map(_.toList) must be(Some(List())) 79 | 80 | q.offer(Array(11.toByte)) must be(true) 81 | q.offer(Array(102.toByte)) must be(true) 82 | q.offer(Array(14.toByte)) must be(false) 83 | q.offer(Array(14.toByte, 14.toByte)) must be(false) 84 | q.offer(Array(14.toByte, 14.toByte)) must be(false) 85 | q.drain().map(_.toList) must be( 86 | List( 87 | List(11.toByte), 88 | List(102.toByte) 89 | ) 90 | ) 91 | } 92 | 93 | { 94 | val q = qF(23) 95 | 96 | q.offer(Array(111.toByte, -37.toByte, 11.toByte)) must be(true) 97 | q.offer(Array(127.toByte, -128.toByte)) must be(true) 98 | q.offer(Array(0.toByte)) must be(true) 99 | q.offer(Array(0.toByte, 10.toByte)) must be(false) 100 | q.offer(Array(3.toByte)) must be(true) 101 | q.drain(1).map(_.toList) must be(List(List(111.toByte, -37.toByte, 11.toByte))) 102 | q.offer(Array(0.toByte, 10.toByte, 12.toByte, 11.toByte)) must be(false) 103 | q.offer(Array(0.toByte, 10.toByte)) must be(true) 104 | q.poll().map(_.toList) must be(Some(List(127.toByte, -128.toByte))) 105 | q.poll().map(_.toList) must be(Some(List(0.toByte))) 106 | q.poll().map(_.toList) must be(Some(List(3.toByte))) 107 | q.poll().map(_.toList) must be(Some(List(0.toByte, 10.toByte))) 108 | q.poll().map(_.toList) must be(None) 109 | } 110 | } 111 | 112 | def testMT(qF: (Int) => BlockingQueue[Array[Byte]]) { 113 | 114 | for (queueSize <- Seq(12, 100, 10000, 1000000)) { 115 | val q = qF(queueSize) 116 | 117 | val putData = List( 118 | Array(0.toByte, 10.toByte, 111.toByte, -50.toByte, 10.toByte, 31.toByte, 65.toByte, -128.toByte), 119 | Array(0.toByte, 10.toByte, 111.toByte, -50.toByte), 120 | Array(22.toByte, 73.toByte), 121 | Array(14.toByte), 122 | Array(2.toByte, 43.toByte) 123 | ) 124 | 125 | // We have a separate take data to be able to check if the tests are catching errors at all 126 | val takeData = List( 127 | Array(0.toByte, 10.toByte, 111.toByte, -50.toByte, 10.toByte, 31.toByte, 65.toByte, -128.toByte), 128 | Array(0.toByte, 10.toByte, 111.toByte, -50.toByte), 129 | Array(22.toByte, 73.toByte), 130 | Array(14.toByte), 131 | Array(2.toByte, 43.toByte) 132 | ) 133 | 134 | // -------------------------------- 135 | 136 | val putTakeRepetitions = 3127 137 | 138 | val putThread = thread( 139 | { 140 | val r = new Random() 141 | for (i <- 0 until putTakeRepetitions) { 142 | for (elem <- putData) { 143 | q.put(elem) 144 | } 145 | if (i % 97 == 0) { 146 | Thread.sleep(r.nextInt(111)) 147 | } 148 | } 149 | } 150 | ) 151 | 152 | val takeThread = thread( 153 | { 154 | val r = new Random() 155 | for (i <- 0 until putTakeRepetitions) { 156 | for (elem <- takeData) { 157 | q.take().toList must be(elem.toList) 158 | } 159 | if (i % 121 == 0) { 160 | Thread.sleep(r.nextInt(99)) 161 | } 162 | } 163 | } 164 | ) 165 | 166 | // -------------------------------- 167 | 168 | val offerPollRepetitions = 2043 169 | val offerFailed = new AtomicLong 170 | val pollFailed = new AtomicLong 171 | 172 | val offerThread = thread( 173 | { 174 | val r = new Random() 175 | for (i <- 0 until offerPollRepetitions) { 176 | for (elem <- putData) { 177 | while (!q.offer(elem)) { 178 | offerFailed.incrementAndGet() 179 | } 180 | } 181 | if (i % 45 == 0) { 182 | Thread.sleep(r.nextInt(33)) 183 | } 184 | } 185 | } 186 | ) 187 | 188 | val pollThread = thread( 189 | { 190 | val r = new Random() 191 | for (i <- 0 until offerPollRepetitions) { 192 | for (elem <- takeData) { 193 | var result = q.poll() 194 | while (result.isEmpty) { 195 | result = q.poll() 196 | pollFailed.incrementAndGet() 197 | } 198 | result.get.toList must be(elem.toList) 199 | } 200 | if (i % 217 == 0) { 201 | Thread.sleep(r.nextInt(193)) 202 | } 203 | } 204 | } 205 | ) 206 | 207 | // -------------------------------- 208 | 209 | val offerPollPutTakeRepetitions = 1249 210 | val offerPutFailed = new AtomicLong 211 | val pollTakeFailed = new AtomicLong 212 | 213 | val offerPutThread = thread( 214 | { 215 | val r = new Random() 216 | for (i <- 0 until offerPollPutTakeRepetitions) { 217 | for (elem <- putData) { 218 | if (r.nextInt(4) == 0) { 219 | while (!q.offer(elem)) { 220 | offerPutFailed.incrementAndGet() 221 | } 222 | } else { 223 | q.put(elem) 224 | } 225 | } 226 | if (i % 31 == 0) { 227 | Thread.sleep(r.nextInt(30)) 228 | } 229 | } 230 | } 231 | ) 232 | 233 | val pollTakeThread = thread( 234 | { 235 | val r = new Random() 236 | for (i <- 0 until offerPollPutTakeRepetitions) { 237 | for (elem <- takeData) { 238 | val res = if (r.nextInt(7) == 0) { 239 | var result = q.poll() 240 | while (result.isEmpty) { 241 | result = q.poll() 242 | pollTakeFailed.incrementAndGet() 243 | } 244 | result.get 245 | } else { 246 | q.take() 247 | } 248 | res.toList must be(elem.toList) 249 | } 250 | if (i % 49 == 0) { 251 | Thread.sleep(r.nextInt(88)) 252 | } 253 | } 254 | } 255 | ) 256 | 257 | // -------------------------------- 258 | 259 | putThread.start() 260 | takeThread.start() 261 | putThread.join() 262 | takeThread.join() 263 | println( 264 | "<%s> Put-Take completed.". 265 | format(queueSize) 266 | ) 267 | 268 | offerThread.start() 269 | pollThread.start() 270 | offerThread.join() 271 | pollThread.join() 272 | println( 273 | "<%s> Offer-Poll completed. Offer failed: %s; Poll failed: %s.". 274 | format(queueSize, offerFailed.longValue(), pollFailed.longValue()) 275 | ) 276 | 277 | offerPutThread.start() 278 | pollTakeThread.start() 279 | offerPutThread.join() 280 | pollTakeThread.join() 281 | println( 282 | "<%s> OfferPut-PollTake completed. OfferPut failed: %s; PollTake failed: %s.". 283 | format(queueSize, offerPutFailed.longValue(), pollTakeFailed.longValue()) 284 | ) 285 | } 286 | } 287 | 288 | private def thread(f: => Any) = new Thread(abortingRunnable(f)) 289 | 290 | } 291 | --------------------------------------------------------------------------------