├── project ├── build.properties └── plugins.sbt ├── version.sbt ├── .gitignore ├── streams └── src │ ├── test │ └── scala │ │ └── play │ │ └── api │ │ └── libs │ │ └── iteratee │ │ └── streams │ │ ├── StreamsSpec.scala │ │ └── impl │ │ ├── SubscriberEvents.scala │ │ ├── EventRecorder.scala │ │ ├── PublisherEvents.scala │ │ ├── PromiseSubscriberSpec.scala │ │ ├── FuturePublisherSpec.scala │ │ ├── SubscriberIterateeSpec.scala │ │ ├── IterateeSubscriberSpec.scala │ │ └── EnumeratorPublisherSpec.scala │ └── main │ └── scala │ └── play │ └── api │ └── libs │ └── iteratee │ └── streams │ ├── impl │ ├── PublisherEnumerator.scala │ ├── SubscriberPublisherProcessor.scala │ ├── RelaxedPublisher.scala │ ├── CheckingPublisher.scala │ ├── SubscriptionFactory.scala │ ├── PromiseSubscriber.scala │ ├── SubscriberIteratee.scala │ ├── FuturePublisher.scala │ ├── EnumeratorPublisher.scala │ └── IterateeSubscriber.scala │ └── IterateeStreams.scala ├── iteratees └── src │ ├── test │ └── scala │ │ └── play │ │ └── api │ │ └── libs │ │ └── iteratee │ │ ├── TraversableIterateesSpec.scala │ │ ├── ExecutionSpecification.scala │ │ ├── TestExecutionContext.scala │ │ ├── ParsingSpec.scala │ │ ├── ExecutionSpec.scala │ │ ├── IterateeSpecification.scala │ │ ├── concurrent │ │ └── NonBlockingMutexSpec.scala │ │ ├── RunQueueSpec.scala │ │ ├── CharEncodingSpec.scala │ │ ├── EnumerateesSpec.scala │ │ ├── ConcurrentSpec.scala │ │ ├── EnumeratorsSpec.scala │ │ └── IterateesSpec.scala │ └── main │ └── scala │ └── play │ └── api │ └── libs │ └── iteratee │ ├── concurrent │ ├── StateMachine.scala │ └── NonBlockingMutex.scala │ ├── package.scala │ ├── Parsing.scala │ ├── Execution.scala │ ├── RunQueue.scala │ ├── TraversableIteratee.scala │ └── CharEncoding.scala ├── .github ├── PULL_REQUEST_TEMPLATE.md └── ISSUE_TEMPLATE.md ├── .travis.yml └── README.md /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.2.8 -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | version in ThisBuild := "2.6.2-SNAPSHOT" -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | resolvers ++= DefaultOptions.resolvers(snapshot = true) 2 | 3 | addSbtPlugin("com.typesafe.play" % "interplay" % "2.0.5") 4 | addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.3.0") 5 | addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.8.2") 6 | 7 | 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | target 3 | .idea 4 | .idea_modules 5 | .classpath 6 | .project 7 | .settings 8 | RUNNING_PID 9 | generated.keystore 10 | generated.truststore 11 | *.log 12 | 13 | # Scala-IDE specific 14 | .scala_dependencies 15 | .project 16 | .settings 17 | .cache-main 18 | .cache-tests 19 | 20 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/StreamsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams 5 | 6 | import org.specs2.mutable.Specification 7 | import play.api.libs.iteratee.streams.impl.FuturePublisher 8 | import scala.concurrent.Future 9 | 10 | class StreamsSpec extends Specification { 11 | 12 | "Streams helper interface" should { 13 | // TODO: Better tests needed, these are only here to ensure Streams compiles 14 | "create a Publisher from a Future" in { 15 | val pubr = IterateeStreams.futureToPublisher(Future.successful(1)) 16 | pubr must haveClass[FuturePublisher[Int]] 17 | } 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/TraversableIterateesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import org.specs2.mutable._ 7 | 8 | object TraversableIterateesSpec extends Specification 9 | with IterateeSpecification with ExecutionSpecification { 10 | 11 | "Traversable.splitOnceAt" should { 12 | 13 | "yield input while predicate is satisfied" in { 14 | mustExecute(1) { splitEC => 15 | val e = Traversable.splitOnceAt[String, Char] { c => c != 'e' }( 16 | implicitly[String => scala.collection.TraversableLike[Char, String]], 17 | splitEC 18 | ) 19 | mustTransformTo("hello", "there")("h")(e) 20 | } 21 | } 22 | 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/PublisherEnumerator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import play.api.libs.iteratee._ 8 | 9 | import scala.concurrent.Future 10 | 11 | /** 12 | * Adapts a Publisher to an Enumerator. 13 | * 14 | * When an Iteratee is applied to the Enumerator, we adapt the Iteratee into 15 | * a Subscriber, then subscribe it to the Publisher. 16 | */ 17 | private[streams] final class PublisherEnumerator[T](pubr: Publisher[T]) extends Enumerator[T] { 18 | def apply[A](i: Iteratee[T, A]): Future[Iteratee[T, A]] = { 19 | val subr = new IterateeSubscriber(i) 20 | pubr.subscribe(subr) 21 | Future.successful(subr.result) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/SubscriberEvents.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams.{ Subscriber, Subscription } 7 | 8 | object SubscriberEvents { 9 | case object OnComplete 10 | case class OnError(t: Throwable) 11 | case class OnSubscribe(s: Subscription) 12 | case class OnNext(t: Int) 13 | } 14 | 15 | trait SubscriberEvents { 16 | self: EventRecorder => 17 | 18 | import SubscriberEvents._ 19 | 20 | object subscriber extends Subscriber[Int] { 21 | def onError(t: Throwable) = record(OnError(t)) 22 | def onSubscribe(s: Subscription) = record(OnSubscribe(s)) 23 | def onComplete() = record(OnComplete) 24 | def onNext(t: Int) = record(OnNext(t)) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/SubscriberPublisherProcessor.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | 8 | /** 9 | * Very simple Processor that delegates to its Subscriber and Publisher arguments. 10 | */ 11 | private[streams] final class SubscriberPublisherProcessor[T, U](subr: Subscriber[T], pubr: Publisher[U]) extends Processor[T, U] { 12 | override def subscribe(subscriber: Subscriber[_ >: U]): Unit = pubr.subscribe(subscriber) 13 | override def onSubscribe(subscription: Subscription): Unit = subr.onSubscribe(subscription) 14 | override def onError(cause: Throwable): Unit = subr.onError(cause) 15 | override def onComplete(): Unit = subr.onComplete() 16 | override def onNext(element: T): Unit = subr.onNext(element) 17 | } 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Pull Request Checklist 2 | 3 | * [ ] Have you read through the [contributor guidelines](https://www.playframework.com/contributing)? 4 | * [ ] Have you signed the [Typesafe CLA](https://www.typesafe.com/contribute/cla)? 5 | * [ ] Have you [squashed your commits](https://www.playframework.com/documentation/latest/WorkingWithGit#Squashing-commits)? 6 | * [ ] Have you added copyright headers to new files? 7 | * [ ] Have you checked that both Scala and Java APIs are updated? 8 | * [ ] Have you updated the documentation for both Scala and Java sections? 9 | * [ ] Have you added tests for any changed functionality? 10 | 11 | ## Fixes 12 | 13 | Fixes #xxxx 14 | 15 | ## Purpose 16 | 17 | What does this PR do? 18 | 19 | ## Background Context 20 | 21 | Why did you take this approach? 22 | 23 | ## References 24 | 25 | Are there any relevant issues / PRs / mailing lists discussions? 26 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/RelaxedPublisher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | 8 | /** 9 | * A Publisher which subscribes Subscribers without performing any 10 | * checking about whether he Suscriber is already subscribed. This 11 | * makes RelaxedPublisher a bit faster, but possibly a bit less safe, 12 | * than a CheckingPublisher. 13 | */ 14 | private[streams] abstract class RelaxedPublisher[T] extends Publisher[T] { 15 | self: SubscriptionFactory[T] => 16 | 17 | // Streams method 18 | final override def subscribe(subr: Subscriber[_ >: T]): Unit = { 19 | val handle: SubscriptionHandle[_] = createSubscription(subr, RelaxedPublisher.onSubscriptionEndedNop) 20 | handle.start() 21 | } 22 | 23 | } 24 | 25 | private[streams] object RelaxedPublisher { 26 | val onSubscriptionEndedNop: Any => Unit = _ => () 27 | } 28 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/concurrent/StateMachine.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.concurrent 5 | 6 | /** 7 | * A state machine with a non-blocking mutex protecting its state. 8 | */ 9 | private[play] class StateMachine[S](initialState: S) { 10 | 11 | /** 12 | * The current state. Modifications to the state should be performed 13 | * inside the body of a call to `exclusive`. To read the state, it is 14 | * usually OK to read this field directly, even though its not volatile 15 | * or atomic, so long as you're happy about happens-before relationships. 16 | */ 17 | var state: S = initialState 18 | 19 | val mutex = new NonBlockingMutex() 20 | 21 | /** 22 | * Exclusive access to the state. The state is read and passed to 23 | * f. Inside f it is safe to modify the state, if desired. 24 | */ 25 | def exclusive(f: S => Unit) = mutex.exclusive { f(state) } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | dist: trusty 3 | sudo: false 4 | group: beta 5 | scala: 6 | - 2.12.1 7 | - 2.11.8 8 | jdk: 9 | - oraclejdk8 10 | cache: 11 | directories: 12 | - "$HOME/.ivy2/cache" 13 | before_cache: 14 | - rm -rf $HOME/.ivy2/cache/com.typesafe.play/* 15 | - rm -rf $HOME/.ivy2/cache/scala_*/sbt_*/com.typesafe.play/* 16 | - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print0 | xargs -n10 -0 rm 17 | notifications: 18 | slack: 19 | secure: o9pCAb6jSqmVtn+YJs46AEB5zDl8Ez6nFYg+ylMJ6bg5xUfEo00Qfb/oVfzwO88UvszCfL54OEOXGE7fLSC8UkmCjhHu19oWEp8ph45bXm5bIUfcUmwJ9Bcl+OzOgdx4kT6dHa30jYEU/uohTQbobY0ciDEUp1UUyHRy+29CnTuo6kCYf4i0VIZaEPPiZmZN6Ubhww1+eWt+zFgS1/LVloyJXl4fxIho0YPxgOuSK8rAHv+P1Q5W+sPxc2jA8l+F82etcggajCJMnAlLITvlpGEsOW3oL6/Bq2AnaPnkVZGeKj3wMGBfRHVuM9OLNVlRvZygbTPcYCg0cojPDeEtEaLfD+M2rgXGwpLB25NDhx/Y8d4noVVyfnQ83GO6hVxH83hFvhdzU9mTJmBvkLOkTCxLbfMrp6CndL5evGCLJ3PEQEytXBtJsepB6zTHIclD/KI8QZjsTwvLkI+6CIeQs/oCyZUMugHMZmcP6CayqySPTQvvw8OF1s9EFLk1+009Qrk9Kb9DB5O1NnG6CCNG1QvuVD1kOen+MuCw4/t4OgEKlI7Lk2L/XZqIoqu+8uxWBTUGMU9IE81r28RpM0WyGn9BTzL4+CW7sYlZvQh1/WU7BHxpxceYswIGQ8bOqB0DUTYKoCpZL6kCK2jpwqeF+IXB5i8DcmqZ2DQKjJk2kmI= 20 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/EventRecorder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import java.util.concurrent.LinkedBlockingQueue 7 | import scala.concurrent.duration.{ FiniteDuration, SECONDS, MILLISECONDS } 8 | 9 | /** 10 | * Utility for recording events in a queue. Useful for 11 | * checking the ordering properties of asynchronous code. Code that does 12 | * stuff records an event, then the test can check that events occur in 13 | * the right order and at the right time. 14 | */ 15 | class EventRecorder( 16 | nextTimeout: FiniteDuration = FiniteDuration(20, SECONDS), 17 | isEmptyDelay: FiniteDuration = FiniteDuration(200, MILLISECONDS) 18 | ) { 19 | 20 | private val events = new LinkedBlockingQueue[AnyRef] 21 | 22 | /** Record an event. */ 23 | def record(e: AnyRef) = events.add(e) 24 | 25 | /** Pull the next event, waiting up to `nextTimeout`. */ 26 | def next(): AnyRef = { 27 | events.poll(nextTimeout.length, nextTimeout.unit) 28 | } 29 | 30 | /** Wait for `isEmptyDelay` then check if the event queue is empty. */ 31 | def isEmptyAfterDelay(waitMillis: Long = 50): Boolean = { 32 | Thread.sleep(isEmptyDelay.toMillis) 33 | events.isEmpty 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # End of Life 2 | 3 | The active Playframework contributors consider this repository has reached End of Life and archived it. 4 | 5 | This repository is not being used anymore and won't get any further updates. 6 | 7 | Thank you to all contributors that worked on this repository! 8 | 9 | # Play Iteratees Library 10 | 11 | This is the Play iteratees library. It has two modules, `play-iteratees`, which provides the iteratees themselves, as well as `play-iteratees-reactive-streams`, which provides a reactive streams implementation on top of iteratees. 12 | 13 | ## Library Dependencies 14 | 15 | To add play-iteratees to your project: 16 | 17 | ```scala 18 | libraryDependencies += "com.typesafe.play" %% "play-iteratees" % "2.6.1" 19 | ``` 20 | 21 | To add play-iteratees reactive streams to your project: 22 | 23 | ```scala 24 | libraryDependencies += "com.typesafe.play" %% "play-iteratees-reactive-streams" % "2.6.1" 25 | ``` 26 | 27 | ## Documentation 28 | 29 | The Play Iteratees implementation is described in the Play 2.5.x documentation on Iteratees. If you are looking to migrate from iteratees to streams, the migration guide has a section: 30 | 31 | * https://www.playframework.com/documentation/2.4.x/Iteratees 32 | * https://www.playframework.com/documentation/2.5.x/StreamsMigration25 33 | 34 | If you are using Play 2.6.x or higher and are looking for `Streams` class, you can find it under: 35 | 36 | * https://github.com/playframework/play-iteratees/blob/master/streams/src/main/scala/play/api/libs/iteratee/streams/IterateeStreams.scala 37 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/ExecutionSpecification.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import scala.concurrent.ExecutionContext 7 | import org.specs2.mutable.SpecificationLike 8 | 9 | /** 10 | * Common functionality for iteratee tests. 11 | */ 12 | trait ExecutionSpecification { 13 | self: SpecificationLike => 14 | 15 | def testExecution[A](f: TestExecutionContext => A): A = { 16 | val ec = TestExecutionContext() 17 | val result = ec.preparable(f(ec)) 18 | result 19 | } 20 | 21 | def testExecution[A](f: (TestExecutionContext, TestExecutionContext) => A): A = { 22 | testExecution(ec1 => testExecution(ec2 => f(ec1, ec2))) 23 | } 24 | 25 | def testExecution[A](f: (TestExecutionContext, TestExecutionContext, TestExecutionContext) => A): A = { 26 | testExecution(ec1 => testExecution(ec2 => testExecution(ec3 => f(ec1, ec2, ec3)))) 27 | } 28 | 29 | def mustExecute[A](expectedCount: => Int)(f: ExecutionContext => A): A = { 30 | testExecution { tec => 31 | val result = f(tec) 32 | tec.executionCount must equalTo(expectedCount) 33 | result 34 | } 35 | } 36 | 37 | def mustExecute[A](expectedCount1: Int, expectedCount2: Int)(f: (ExecutionContext, ExecutionContext) => A): A = { 38 | mustExecute(expectedCount1)(ec1 => mustExecute(expectedCount2)(ec2 => f(ec1, ec2))) 39 | } 40 | 41 | def mustExecute[A](expectedCount1: Int, expectedCount2: Int, expectedCount3: Int)(f: (ExecutionContext, ExecutionContext, ExecutionContext) => A): A = { 42 | mustExecute(expectedCount1)(ec1 => mustExecute(expectedCount2)(ec2 => mustExecute(expectedCount3)(ec3 => f(ec1, ec2, ec3)))) 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/TestExecutionContext.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import scala.concurrent.ExecutionContext 7 | 8 | object TestExecutionContext { 9 | 10 | /** 11 | * Create a `TestExecutionContext` that delegates to the iteratee package's default `ExecutionContext`. 12 | */ 13 | def apply(): TestExecutionContext = new TestExecutionContext(Execution.trampoline) 14 | 15 | } 16 | 17 | /** 18 | * An `ExecutionContext` that counts its executions. 19 | * 20 | * @param delegate The underlying `ExecutionContext` to delegate execution to. 21 | */ 22 | class TestExecutionContext(delegate: ExecutionContext) extends ExecutionContext { 23 | top => 24 | 25 | val count = new java.util.concurrent.atomic.AtomicInteger() 26 | 27 | val local = new ThreadLocal[java.lang.Boolean] 28 | 29 | def preparable[A](body: => A): A = { 30 | local.set(true) 31 | try body finally local.set(null) 32 | } 33 | 34 | def execute(runnable: Runnable): Unit = { 35 | throw new RuntimeException("Cannot execute unprepared TestExecutionContext") 36 | } 37 | 38 | def reportFailure(t: Throwable): Unit = { 39 | println(t) 40 | } 41 | 42 | override def prepare(): ExecutionContext = { 43 | val isLocal = Option(local.get()).getOrElse(false: java.lang.Boolean) 44 | if (!isLocal) throw new RuntimeException("Can only prepare TestExecutionContext within 'preparable' scope") 45 | val preparedDelegate = delegate.prepare() 46 | return new ExecutionContext { 47 | 48 | def execute(runnable: Runnable): Unit = { 49 | count.getAndIncrement() 50 | preparedDelegate.execute(runnable) 51 | } 52 | 53 | def reportFailure(t: Throwable): Unit = { 54 | println(t) 55 | } 56 | 57 | } 58 | } 59 | 60 | def executionCount: Int = count.get() 61 | 62 | } 63 | 64 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/PublisherEvents.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import scala.concurrent._ 8 | 9 | object PublisherEvents { 10 | case class RequestMore(elementCount: Long) 11 | case object Cancel 12 | } 13 | 14 | trait PublisherEvents[T] { 15 | self: EventRecorder => 16 | 17 | import PublisherEvents._ 18 | 19 | case class BoundedSubscription[T, U >: T](pr: Publisher[T], sr: Subscriber[U]) extends Subscription { 20 | def cancel(): Unit = { 21 | record(Cancel) 22 | } 23 | def request(elements: Long): Unit = { 24 | record(RequestMore(elements)) 25 | } 26 | def onNext(element: T): Unit = sr.onNext(element) 27 | } 28 | 29 | object publisher extends Publisher[T] { 30 | val subscription = Promise[BoundedSubscription[T, _]]() 31 | override def subscribe(sr: Subscriber[_ >: T]) = { 32 | subscription.success(BoundedSubscription(this, sr)) 33 | } 34 | } 35 | 36 | private def forSubscription(f: BoundedSubscription[T, _] => Any)(implicit ec: ExecutionContext): Future[Unit] = { 37 | publisher.subscription.future.map { sn => 38 | f(sn) 39 | () 40 | } 41 | } 42 | 43 | def onSubscribe()(implicit ec: ExecutionContext): Unit = { 44 | forSubscription { sn => 45 | sn.sr.onSubscribe(sn) 46 | } 47 | } 48 | 49 | def onNext(element: T)(implicit ec: ExecutionContext): Unit = { 50 | forSubscription { sn => 51 | sn.onNext(element) 52 | } 53 | } 54 | 55 | def onError(t: Throwable)(implicit ec: ExecutionContext): Unit = { 56 | forSubscription { sn => 57 | sn.sr.onError(t) 58 | } 59 | } 60 | 61 | def onComplete()(implicit ec: ExecutionContext): Unit = { 62 | forSubscription { sn => 63 | sn.sr.onComplete() 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/CheckingPublisher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import java.util.concurrent.atomic.AtomicReference 7 | 8 | import org.reactivestreams._ 9 | 10 | import scala.annotation.tailrec 11 | 12 | /** 13 | * A Publisher which checks that it never starts more than one active 14 | * Subscription for the same Subscriber. This extra safety comes at a 15 | * little bit of extra cost. If you want a cheaper Publisher, use 16 | * RelaxedPublisher. 17 | */ 18 | private[streams] abstract class CheckingPublisher[T] extends Publisher[T] { 19 | self: SubscriptionFactory[T] => 20 | 21 | /** 22 | * The list of handles to currently active Subscriptions. 23 | */ 24 | private val subscriptions = new AtomicReference[List[SubscriptionHandle[_]]](Nil) 25 | 26 | // Streams method 27 | final override def subscribe(subr: Subscriber[_ >: T]): Unit = { 28 | val handle: SubscriptionHandle[_] = createSubscription(subr, removeSubscription) 29 | 30 | @tailrec 31 | def addSubscription(): Unit = { 32 | val oldSubscriptions = subscriptions.get 33 | if (oldSubscriptions.exists(s => (s.subscriber eq subr) && s.isActive)) { 34 | subr.onError(new IllegalStateException("Subscriber is already subscribed to this Publisher")) 35 | } else { 36 | val newSubscriptions: List[SubscriptionHandle[_]] = handle :: oldSubscriptions 37 | if (subscriptions.compareAndSet(oldSubscriptions, newSubscriptions)) { 38 | handle.start() 39 | } else addSubscription() 40 | } 41 | } 42 | addSubscription() 43 | } 44 | 45 | @tailrec 46 | private def removeSubscription(subscription: SubscriptionHandle[_]): Unit = { 47 | val oldSubscriptions = subscriptions.get 48 | val newSubscriptions = oldSubscriptions.filterNot(_.subscriber eq subscription.subscriber) 49 | if (subscriptions.compareAndSet(oldSubscriptions, newSubscriptions)) () else removeSubscription(subscription) 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/SubscriptionFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | 8 | /** 9 | * A SubscriptionFactory is an object that knows how to create Subscriptions. 10 | * It can be used as a building block for creating Publishers, allowing the 11 | * Subscription creation logic to be factored out. 12 | */ 13 | private[streams] trait SubscriptionFactory[T] { 14 | 15 | /** 16 | * Create a Subscription object and return a handle to it. 17 | * 18 | * After calling this method the Subscription may be discarded, so the Subscription 19 | * shouldn't perform any actions or call any methods on the Subscriber 20 | * until `start` is called. The purpose of the `start` method is to give 21 | * the caller an opportunity to optimistically create Subscription objects 22 | * but then discard them if they can't be used for some reason. For 23 | * example, if two `Subscriptions` are concurrently created for the same 24 | * `Subscriber` then some implementations will only call `start` on 25 | * one of the `SubscriptionHandle`s. 26 | */ 27 | def createSubscription[U >: T]( 28 | subr: Subscriber[U], 29 | onSubscriptionEnded: SubscriptionHandle[U] => Unit 30 | ): SubscriptionHandle[U] 31 | 32 | } 33 | 34 | /** 35 | * Wraps a Subscription created by a SubscriptionFactory, allowing the 36 | * Subscription to be started and queried. 37 | */ 38 | trait SubscriptionHandle[U] { 39 | 40 | /** 41 | * Called to start the Subscription. This will typically call the 42 | * onSubscribe method on the Suscription's Subscriber. In the event 43 | * that this method is never called the Subscription should not 44 | * leak resources. 45 | */ 46 | def start(): Unit 47 | 48 | /** 49 | * The Subscriber for this Subscription. 50 | */ 51 | def subscriber: Subscriber[U] 52 | 53 | /** 54 | * Whether or not this Subscription is active. It won't be active if it has 55 | * been cancelled, completed or had an error. 56 | */ 57 | def isActive: Boolean 58 | } 59 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs { 5 | 6 | /** 7 | * The Iteratee monad provides strict, safe, and functional I/O. 8 | */ 9 | package object iteratee { 10 | type K[E, A] = Input[E] => Iteratee[E, A] 11 | } 12 | } 13 | 14 | package play.api.libs.iteratee { 15 | 16 | private[iteratee] object internal { 17 | import play.api.libs.iteratee.Iteratee 18 | import scala.concurrent.{ ExecutionContext, Future } 19 | import scala.util.control.NonFatal 20 | 21 | /** 22 | * Executes code immediately on the current thread, 23 | * returning a successful or failed Future depending on the result. 24 | * 25 | * TODO: Rename to `tryFuture`. 26 | */ 27 | def eagerFuture[A](body: => A): Future[A] = 28 | try Future.successful(body) catch { case NonFatal(e) => Future.failed(e) } 29 | 30 | /** 31 | * Executes code in the given ExecutionContext, 32 | * flattening the resulting Future. 33 | */ 34 | def executeFuture[A](body: => Future[A])(implicit ec: ExecutionContext): Future[A] = { 35 | Future { 36 | body 37 | }(ec /* Future.apply will prepare */ ).flatMap(identityFunc.asInstanceOf[Future[A] => Future[A]])(Execution.trampoline) 38 | } 39 | 40 | /** 41 | * Executes code in the given ExecutionContext, flattening the resulting Iteratee. 42 | */ 43 | def executeIteratee[A, E](body: => Iteratee[A, E])(implicit ec: ExecutionContext): Iteratee[A, E] = Iteratee.flatten(Future(body)(ec)) 44 | 45 | /** 46 | * Prepare an ExecutionContext and pass it to the given function, returning the result of 47 | * the function. 48 | * 49 | * Makes it easy to write single line functions with a prepared ExecutionContext, eg: 50 | * {{{ 51 | * def myFunc(implicit ec: ExecutionContext) = prepared(ec)(pec => ...) 52 | * }}} 53 | */ 54 | def prepared[A](ec: ExecutionContext)(f: ExecutionContext => A): A = { 55 | val pec = ec.prepare() 56 | f(pec) 57 | } 58 | 59 | val identityFunc: (Any => Any) = (x: Any) => x 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/ParsingSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import Parsing._ 7 | 8 | import org.specs2.mutable._ 9 | import scala.concurrent.duration.Duration 10 | import scala.concurrent.Await 11 | 12 | object ParsingSpec extends Specification 13 | with IterateeSpecification with ExecutionSpecification { 14 | 15 | "Parsing" should { 16 | 17 | "split case 1" in { 18 | mustExecute(10) { foldEC => 19 | val data = Enumerator(List("xx", "kxckikixckikio", "cockik", "isdodskikisd", "ksdloii").map(_.getBytes): _*) 20 | val parsed = data |>>> Parsing.search("kiki".getBytes).transform(Iteratee.fold(List.empty[MatchInfo[Array[Byte]]]) { (s, c: MatchInfo[Array[Byte]]) => s :+ c }(foldEC)) 21 | 22 | val result = Await.result(parsed, Duration.Inf).map { 23 | case Matched(kiki) => "Matched(" + new String(kiki) + ")" 24 | case Unmatched(data) => "Unmatched(" + new String(data) + ")" 25 | }.mkString(", ") 26 | 27 | result must equalTo( 28 | "Unmatched(xxkxc), Matched(kiki), Unmatched(xc), Matched(kiki), Unmatched(ococ), Matched(kiki), Unmatched(sdods), Matched(kiki), Unmatched(sdks), Unmatched(dloii)" 29 | ) 30 | } 31 | } 32 | 33 | "split case 1" in { 34 | mustExecute(11) { foldEC => 35 | val data = Enumerator(List("xx", "kxckikixcki", "k", "kicockik", "isdkikodskikisd", "ksdlokiikik", "i").map(_.getBytes): _*) 36 | val parsed = data |>>> Parsing.search("kiki".getBytes).transform(Iteratee.fold(List.empty[MatchInfo[Array[Byte]]]) { (s, c: MatchInfo[Array[Byte]]) => s :+ c }(foldEC)) 37 | 38 | val result = Await.result(parsed, Duration.Inf).map { 39 | case Matched(kiki) => "Matched(" + new String(kiki) + ")" 40 | case Unmatched(data) => "Unmatched(" + new String(data) + ")" 41 | }.mkString(", ") 42 | 43 | result must equalTo( 44 | "Unmatched(xxkxc), Matched(kiki), Unmatched(xckikkico), Unmatched(c), Matched(kiki), Unmatched(sdkikods), Matched(kiki), Unmatched(sdksdlok), Unmatched(ii), Matched(kiki), Unmatched()" 45 | ) 46 | } 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Are you looking for help? 2 | 3 | This is an issue tracker, used to manage and track the development of Play Iteratees. It is not a support system and so it is not a place to ask questions or get help. If you're not sure if you have found a bug, the best place to start is with either the [users mailing list](https://groups.google.com/forum/#!forum/play-framework) or [Stack Overflow](http://stackoverflow.com/questions/ask?tags=playframework). If you have a feature request, the [developer mailing list](https://groups.google.com/forum/#!forum/play-framework-dev) is a better forum than an issue tracker to discuss it. 4 | 5 | If you want to discuss a feature request, the same tools are available. 6 | 7 | ### Play Iteratees Version (2.5.x / etc) 8 | 9 | 10 | 11 | ### API (Scala / Java / Neither / Both) 12 | 13 | 14 | 15 | ### Operating System (Ubuntu 15.10 / MacOS 10.10 / Windows 10) 16 | 17 | Use `uname -a` if on Linux. 18 | 19 | ### JDK (Oracle 1.8.0_72, OpenJDK 1.8.x, Azul Zing) 20 | 21 | Paste the output from `java -version` at the command line. 22 | 23 | ### Library Dependencies 24 | 25 | If this is an issue that involves integration with another system, include the exact version and OS of the other system, including any intermediate drivers or APIs i.e. if you connect to a PostgreSQL database, include both the version / OS of PostgreSQL and the JDBC driver version used to connect to the database. 26 | 27 | ### Expected Behavior 28 | 29 | Please describe the expected behavior of the issue, starting from the first action. 30 | 31 | 1. 32 | 2. 33 | 3. 34 | 35 | ### Actual Behavior 36 | 37 | Please provide a description of what actually happens, working from the same starting point. 38 | 39 | Be descriptive: "it doesn't work" does not describe what the behavior actually is -- instead, say "the page renders a 500 error code with no body content." Copy and paste logs, and include any URLs. Turn on internal Play Iteratees logging with `` if there is no log output. 40 | 41 | 1. 42 | 2. 43 | 3. 44 | 45 | ### Reproducible Test Case 46 | 47 | Please provide a PR with a failing test. 48 | 49 | If the issue is more complex or requires configuration, please provide a link to a project on Github that reproduces the issue. 50 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/PromiseSubscriberSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.specs2.mutable.Specification 7 | import scala.concurrent.duration.{ FiniteDuration => ScalaFiniteDuration, SECONDS } 8 | import scala.concurrent.Promise 9 | import scala.util.{ Failure, Success, Try } 10 | 11 | import scala.concurrent.ExecutionContext.Implicits.global 12 | 13 | class PromiseSubscriberSpec extends Specification { 14 | 15 | import PublisherEvents._ 16 | case class OnComplete(result: Try[Any]) 17 | 18 | class TestEnv[T] extends EventRecorder(ScalaFiniteDuration(2, SECONDS)) with PublisherEvents[T] { 19 | 20 | val prom = Promise[T]() 21 | val subr = new PromiseSubscriber(prom) 22 | prom.future.onComplete { result => record(OnComplete(result)) } 23 | 24 | } 25 | 26 | "PromiseSubscriber" should { 27 | "consume 1 item" in { 28 | val testEnv = new TestEnv[Int] 29 | import testEnv._ 30 | isEmptyAfterDelay() must beTrue 31 | 32 | publisher.subscribe(subr) 33 | onSubscribe() 34 | next must_== RequestMore(1) 35 | isEmptyAfterDelay() must beTrue 36 | 37 | onNext(3) 38 | next must_== OnComplete(Success(3)) 39 | isEmptyAfterDelay() must beTrue 40 | } 41 | "consume an error" in { 42 | val testEnv = new TestEnv[Int] 43 | import testEnv._ 44 | isEmptyAfterDelay() must beTrue 45 | 46 | publisher.subscribe(subr) 47 | onSubscribe() 48 | next must_== RequestMore(1) 49 | isEmptyAfterDelay() must beTrue 50 | 51 | val e = new Exception("!!!") 52 | onError(e) 53 | next must_== OnComplete(Failure(e)) 54 | isEmptyAfterDelay() must beTrue 55 | } 56 | "fail when completed too early" in { 57 | val testEnv = new TestEnv[Int] 58 | import testEnv._ 59 | isEmptyAfterDelay() must beTrue 60 | 61 | publisher.subscribe(subr) 62 | onSubscribe() 63 | next must_== RequestMore(1) 64 | isEmptyAfterDelay() must beTrue 65 | 66 | onComplete() 67 | next must beLike { case OnComplete(Failure(_: IllegalStateException)) => ok } 68 | isEmptyAfterDelay() must beTrue 69 | } 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/ExecutionSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import scala.language.reflectiveCalls 7 | 8 | import org.specs2.mutable._ 9 | import scala.concurrent.{ ExecutionContext, Future, Await } 10 | import scala.concurrent.duration.{ Duration, SECONDS } 11 | import scala.util.Try 12 | 13 | object ExecutionSpec extends Specification { 14 | import Execution.trampoline 15 | 16 | val waitTime = Duration(5, SECONDS) 17 | 18 | "trampoline" should { 19 | 20 | "execute code in the same thread" in { 21 | val f = Future(Thread.currentThread())(trampoline) 22 | Await.result(f, waitTime) must equalTo(Thread.currentThread()) 23 | } 24 | 25 | "not overflow the stack" in { 26 | def executeRecursively(ec: ExecutionContext, times: Int): Unit = { 27 | if (times > 0) { 28 | ec.execute(new Runnable { 29 | def run() = executeRecursively(ec, times - 1) 30 | }) 31 | } 32 | } 33 | 34 | // Work out how deep to go to cause an overflow 35 | val overflowingExecutionContext = new ExecutionContext { 36 | def execute(runnable: Runnable): Unit = { 37 | runnable.run() 38 | } 39 | def reportFailure(t: Throwable): Unit = t.printStackTrace() 40 | } 41 | 42 | var overflowTimes = 1 << 10 43 | try { 44 | while (overflowTimes > 0) { 45 | executeRecursively(overflowingExecutionContext, overflowTimes) 46 | overflowTimes = overflowTimes << 1 47 | } 48 | sys.error("Can't get the stack to overflow") 49 | } catch { 50 | case _: StackOverflowError => () 51 | } 52 | 53 | // Now verify that we don't overflow 54 | Try(executeRecursively(trampoline, overflowTimes)) must beSuccessfulTry[Unit] 55 | } 56 | 57 | "execute code in the order it was submitted" in { 58 | val runRecord = scala.collection.mutable.Buffer.empty[Int] 59 | case class TestRunnable(id: Int, children: Runnable*) extends Runnable { 60 | def run() = { 61 | runRecord += id 62 | for (c <- children) trampoline.execute(c) 63 | } 64 | } 65 | 66 | trampoline.execute( 67 | TestRunnable( 68 | 0, 69 | TestRunnable(1), 70 | TestRunnable( 71 | 2, 72 | TestRunnable( 73 | 4, 74 | TestRunnable(6), 75 | TestRunnable(7) 76 | ), 77 | TestRunnable( 78 | 5, 79 | TestRunnable(8) 80 | ) 81 | ), 82 | TestRunnable(3) 83 | ) 84 | ) 85 | 86 | runRecord must equalTo(0 to 8) 87 | } 88 | 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/PromiseSubscriber.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import play.api.libs.iteratee.concurrent.StateMachine 8 | 9 | import scala.concurrent.Promise 10 | 11 | private[streams] object PromiseSubscriber { 12 | /** 13 | * Internal state of the Subscriber. 14 | */ 15 | sealed trait State 16 | /** 17 | * A Subscriber that hasn't had onSubscribe called on it yet, and whose 18 | * Promise is not complete. 19 | */ 20 | final case object AwaitingSubscription extends State 21 | /** 22 | * A Subscriber that has had onSubscribe called on and whose 23 | * Promise is not complete. 24 | */ 25 | final case object Subscribed extends State 26 | /** 27 | * A Subscriber that is complete, either because onComplete or onError was 28 | * called or because its Promise is complete. 29 | */ 30 | final case object Completed extends State 31 | } 32 | 33 | import PromiseSubscriber._ 34 | 35 | // Assume that promise's onComplete handler runs asynchronously 36 | private[streams] class PromiseSubscriber[T](prom: Promise[T]) 37 | extends StateMachine[State](initialState = AwaitingSubscription) with Subscriber[T] { 38 | 39 | // Streams methods 40 | 41 | override def onSubscribe(subscription: Subscription): Unit = exclusive { 42 | case Subscribed => 43 | throw new IllegalStateException("Can't call onSubscribe twice") 44 | case AwaitingSubscription => 45 | // Check if promise is completed. Even if we request elements, we 46 | // still need to handle the Promise completing in some other way. 47 | if (prom.isCompleted) { 48 | state = Completed 49 | subscription.cancel() 50 | } else { 51 | state = Subscribed 52 | subscription.request(1) 53 | } 54 | case Completed => 55 | subscription.cancel() 56 | } 57 | 58 | override def onError(cause: Throwable): Unit = exclusive { 59 | case AwaitingSubscription | Subscribed => 60 | state = Completed 61 | prom.failure(cause) // we assume any Future.onComplete handlers run asynchronously 62 | case Completed => 63 | () 64 | } 65 | 66 | override def onComplete(): Unit = exclusive { 67 | case AwaitingSubscription | Subscribed => 68 | prom.failure(new IllegalStateException("Can't handle onComplete until an element has been received")) 69 | state = Completed 70 | case Completed => 71 | () 72 | } 73 | 74 | override def onNext(element: T): Unit = exclusive { 75 | case AwaitingSubscription => 76 | state = Completed 77 | throw new IllegalStateException("Can't handle onNext until at least one subscription has occurred") 78 | case Subscribed => 79 | state = Completed 80 | prom.success(element) // we assume any Future.onComplete handlers run asynchronously 81 | case Completed => 82 | () 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/IterateeSpecification.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import play.api.libs.iteratee.internal.executeFuture 7 | import scala.concurrent.{ Await, ExecutionContext, Future, Promise } 8 | import scala.concurrent.duration.{ Duration, SECONDS, MILLISECONDS } 9 | import scala.util.Try 10 | 11 | /** 12 | * Common functionality for iteratee tests. 13 | */ 14 | trait IterateeSpecification { 15 | self: org.specs2.mutable.SpecificationLike => 16 | 17 | val waitTime = Duration(30, SECONDS) 18 | def await[A](f: Future[A]): A = Await.result(f, waitTime) 19 | def ready[A](f: Future[A]): Future[A] = Await.ready(f, waitTime) 20 | 21 | def mustTransformTo[E, A](in: E*)(out: A*)(e: Enumeratee[E, A]) = { 22 | val f = Future(Enumerator(in: _*) |>>> e &>> Iteratee.getChunks[A])(Execution.defaultExecutionContext).flatMap[List[A]](x => x)(Execution.defaultExecutionContext) 23 | Await.result(f, Duration.Inf) must equalTo(List(out: _*)) 24 | } 25 | 26 | def enumeratorChunks[E](e: Enumerator[E]): Future[List[E]] = { 27 | executeFuture(e |>>> Iteratee.getChunks[E])(Execution.defaultExecutionContext) 28 | } 29 | 30 | def mustEnumerateTo[E, A](out: A*)(e: Enumerator[E]) = { 31 | Await.result(enumeratorChunks(e), Duration.Inf) must equalTo(List(out: _*)) 32 | } 33 | 34 | def mustPropagateFailure[E](e: Enumerator[E]) = { 35 | Try(Await.result( 36 | e(Cont { case _ => throw new RuntimeException() }), 37 | Duration.Inf 38 | )) must beAFailedTry 39 | } 40 | 41 | /** 42 | * Convenience function for creating a Done Iteratee that returns the given value 43 | */ 44 | def done(value: String): Iteratee[String, String] = Done[String, String](value) 45 | 46 | /** 47 | * Convenience function for an Error Iteratee that contains the given error message 48 | */ 49 | def error(msg: String): Iteratee[String, String] = Error[String](msg, Input.Empty) 50 | 51 | /** 52 | * Convenience function for creating a Cont Iteratee that feeds its input to the given function 53 | */ 54 | def cont(f: String => Iteratee[String, String]): Iteratee[String, String] = { 55 | Cont[String, String]({ 56 | case Input.El(input: String) => f(input) 57 | case unrecognized => throw new IllegalArgumentException(s"Unexpected input for Cont iteratee: $unrecognized") 58 | }) 59 | } 60 | 61 | /** 62 | * Convenience function for creating the given Iteratee after the given delay 63 | */ 64 | def delayed(it: => Iteratee[String, String], delay: Duration = Duration(5, MILLISECONDS))(implicit ec: ExecutionContext): Iteratee[String, String] = { 65 | Iteratee.flatten(timeout(it, delay)) 66 | } 67 | 68 | val timer = new java.util.Timer(true) 69 | def timeout[A](a: => A, d: Duration)(implicit e: ExecutionContext): Future[A] = { 70 | val p = Promise[A]() 71 | timer.schedule(new java.util.TimerTask { 72 | def run(): Unit = { 73 | p.complete(Try(a)) 74 | } 75 | }, d.toMillis) 76 | p.future 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/concurrent/NonBlockingMutex.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.concurrent 5 | 6 | import scala.annotation.tailrec 7 | import java.util.concurrent.atomic.AtomicReference 8 | 9 | /** 10 | * Provides mutual exclusion without blocking. 11 | * 12 | * {{{ 13 | * // The following two tasks will run one at a time, but in any order. 14 | * val mutex = new NonBlockingMutex() 15 | * Future { 16 | * mutex.exclusive { task1() } 17 | * } 18 | * Future { 19 | * mutex.exclusive { task2() } 20 | * } 21 | * }}} 22 | * 23 | * A queue of operations is maintained internally and updated atomically. 24 | * If `exclusive` is called while no other operations are running then 25 | * that operation will run immediately. If another operation is running 26 | * then the new operation will be added to the end of the queue, scheduling 27 | * the operation to run later without blocking the current thread. When an 28 | * operation finishes running it looks at the queue and runs any tasks 29 | * that are enqueued. 30 | * 31 | * Because operations can run on other threads it is important that they 32 | * run very quickly. If any expensive work needs to be done then it the operation 33 | * should schedule the work to run asynchronously. 34 | */ 35 | private[play] final class NonBlockingMutex { 36 | 37 | /** 38 | * Schedule an operation to run with exclusive access to the mutex. If the 39 | * mutex is uncontended then the operation will run immediately. If not, it 40 | * will be enqueued and the method will yield immediately. The operation will 41 | * run later on another thread. 42 | * 43 | * Because operations can run on other threads it is important that they 44 | * run very quickly. If any expensive work needs to be done then it the operation 45 | * should schedule the work to run asynchronously. 46 | * 47 | * @param body The body of the operation to run. 48 | */ 49 | def exclusive(body: => Unit): Unit = { 50 | schedule(() => body) 51 | } 52 | 53 | private type Op = () => Unit 54 | 55 | private val state = new AtomicReference[Vector[Op]](null) 56 | 57 | @tailrec 58 | private def schedule(op: Op): Unit = { 59 | val prevState = state.get 60 | val newState = prevState match { 61 | case null => Vector.empty // This is very cheap because Vector.empty is only allocated once 62 | case pending => pending :+ op 63 | } 64 | if (state.compareAndSet(prevState, newState)) { 65 | prevState match { 66 | case null => 67 | // We've update the state to say that we're running an op, 68 | // so we need to actually start it running. 69 | executeAll(op) 70 | case _ => 71 | } 72 | } else schedule(op) // Try again 73 | } 74 | 75 | @tailrec 76 | private def executeAll(op: Op): Unit = { 77 | op.apply() 78 | val nextOp = dequeueNextOpToExecute() 79 | nextOp match { 80 | case None => () 81 | case Some(op) => executeAll(op) 82 | } 83 | } 84 | 85 | @tailrec 86 | private def dequeueNextOpToExecute(): Option[Op] = { 87 | val prevState = state.get 88 | val (newState, nextOp) = prevState match { 89 | case null => throw new IllegalStateException("When executing, must have a queue of pending elements") 90 | case pending if pending.isEmpty => (null, None) 91 | case pending => (pending.tail, Some(pending.head)) 92 | } 93 | if (state.compareAndSet(prevState, newState)) nextOp else dequeueNextOpToExecute() 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/FuturePublisherSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import org.specs2.mutable.Specification 8 | import scala.concurrent.{ Future, Promise } 9 | 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | 12 | class FuturePublisherSpec extends Specification { 13 | 14 | case object OnSubscribe 15 | case class OnError(t: Throwable) 16 | case class OnNext(element: Any) 17 | case object OnComplete 18 | case class RequestMore(elementCount: Int) 19 | case object Cancel 20 | case object GetSubscription 21 | 22 | class TestEnv[T] extends EventRecorder() { 23 | 24 | object subscriber extends Subscriber[T] { 25 | val subscription = Promise[Subscription]() 26 | override def onSubscribe(s: Subscription) = { 27 | record(OnSubscribe) 28 | subscription.success(s) 29 | } 30 | override def onError(t: Throwable) = record(OnError(t)) 31 | override def onNext(element: T) = record(OnNext(element)) 32 | override def onComplete() = record(OnComplete) 33 | } 34 | 35 | def forSubscription(f: Subscription => Any): Future[Unit] = { 36 | subscriber.subscription.future.map(f).map(_ => ()) 37 | } 38 | def request(elementCount: Int): Future[Unit] = { 39 | forSubscription { s => 40 | record(RequestMore(elementCount)) 41 | s.request(elementCount) 42 | } 43 | } 44 | def cancel(): Future[Unit] = { 45 | forSubscription { s => 46 | record(Cancel) 47 | s.cancel() 48 | } 49 | } 50 | 51 | } 52 | 53 | "FuturePublisher" should { 54 | "produce immediate success results" in { 55 | val testEnv = new TestEnv[Int] 56 | val fut = Future.successful(1) 57 | val pubr = new FuturePublisher(fut) 58 | pubr.subscribe(testEnv.subscriber) 59 | testEnv.next must_== OnSubscribe 60 | testEnv.request(1) 61 | testEnv.next must_== RequestMore(1) 62 | testEnv.next must_== OnNext(1) 63 | } 64 | "produce immediate failure results" in { 65 | val testEnv = new TestEnv[Int] 66 | val e = new Exception("test failure") 67 | val fut: Future[Int] = Future.failed(e) 68 | val pubr = new FuturePublisher(fut) 69 | pubr.subscribe(testEnv.subscriber) 70 | testEnv.next must_== OnError(e) 71 | testEnv.isEmptyAfterDelay() must beTrue 72 | } 73 | "produce delayed success results" in { 74 | val testEnv = new TestEnv[Int] 75 | val prom = Promise[Int]() 76 | val pubr = new FuturePublisher(prom.future) 77 | pubr.subscribe(testEnv.subscriber) 78 | testEnv.next must_== OnSubscribe 79 | testEnv.request(1) 80 | testEnv.next must_== RequestMore(1) 81 | testEnv.isEmptyAfterDelay() must beTrue 82 | prom.success(3) 83 | testEnv.next must_== OnNext(3) 84 | testEnv.next must_== OnComplete 85 | testEnv.isEmptyAfterDelay() must beTrue 86 | } 87 | "produce delayed failure results" in { 88 | val testEnv = new TestEnv[Int] 89 | val prom = Promise[Int]() 90 | val pubr = new FuturePublisher(prom.future) 91 | pubr.subscribe(testEnv.subscriber) 92 | testEnv.next must_== OnSubscribe 93 | testEnv.request(1) 94 | testEnv.next must_== RequestMore(1) 95 | testEnv.isEmptyAfterDelay() must beTrue 96 | val e = new Exception("test failure") 97 | prom.failure(e) 98 | testEnv.next must_== OnError(e) 99 | testEnv.isEmptyAfterDelay() must beTrue 100 | } 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/concurrent/NonBlockingMutexSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.concurrent 5 | 6 | import scala.language.reflectiveCalls 7 | 8 | import org.specs2.mutable._ 9 | import java.util.concurrent.atomic.AtomicInteger 10 | import scala.concurrent.{ ExecutionContext, Promise, Future, Await } 11 | import scala.concurrent.duration.{ Duration, SECONDS } 12 | 13 | object NonBlockingMutexSpec extends Specification { 14 | 15 | val waitTime = Duration(2, SECONDS) 16 | 17 | trait Tester { 18 | def run(body: => Unit): Unit 19 | } 20 | 21 | class MutexTester extends Tester { 22 | val mutex = new NonBlockingMutex() 23 | def run(body: => Unit) = mutex.exclusive(body) 24 | } 25 | 26 | class NaiveTester extends Tester { 27 | def run(body: => Unit) = body 28 | } 29 | 30 | def countOrderingErrors(runs: Int, tester: Tester)(implicit ec: ExecutionContext): Future[Int] = { 31 | val result = Promise[Int]() 32 | val runCount = new AtomicInteger(0) 33 | val orderingErrors = new AtomicInteger(0) 34 | 35 | for (i <- 0 until runs) { 36 | tester.run { 37 | val observedRunCount = runCount.getAndIncrement() 38 | 39 | // We see observedRunCount != i then this task was run out of order 40 | if (observedRunCount != i) { 41 | orderingErrors.incrementAndGet() // Record the error 42 | } 43 | // If this is the last task, complete our result promise 44 | if ((observedRunCount + 1) >= runs) { 45 | result.success(orderingErrors.get) 46 | } 47 | } 48 | } 49 | result.future 50 | } 51 | 52 | "NonBlockingMutex" should { 53 | 54 | "run a single operation" in { 55 | val p = Promise[Int]() 56 | val mutex = new NonBlockingMutex() 57 | mutex.exclusive { p.success(1) } 58 | Await.result(p.future, waitTime) must_== (1) 59 | } 60 | 61 | "run two operations" in { 62 | val p1 = Promise[Unit]() 63 | val p2 = Promise[Unit]() 64 | val mutex = new NonBlockingMutex() 65 | mutex.exclusive { p1.success(()) } 66 | mutex.exclusive { p2.success(()) } 67 | Await.result(p1.future, waitTime) must_== (()) 68 | Await.result(p2.future, waitTime) must_== (()) 69 | } 70 | 71 | "run code in order" in { 72 | import ExecutionContext.Implicits.global 73 | 74 | def percentageOfRunsWithOrderingErrors(runSize: Int, tester: Tester): Int = { 75 | val results: Seq[Future[Int]] = for (i <- 0 until 9) yield { 76 | countOrderingErrors(runSize, tester) 77 | } 78 | Await.result(Future.sequence(results), waitTime).filter(_ > 0).size * 10 79 | } 80 | 81 | // Iteratively increase the run size until we get observable errors 90% of the time 82 | // We want a high error rate because we want to then use the MutexTester 83 | // on the same run size and know that it is fixing up some problems. If the run size 84 | // is too small then the MutexTester probably isn't doing anything. We use 85 | // dynamic run sizing because the actual size that produces errors will vary 86 | // depending on the environment in which this test is run. 87 | var runSize = 8 // This usually reaches 8192 on my dev machine with 10 simultaneous queues 88 | var errorPercentage = 0 89 | while (errorPercentage < 90 && runSize < 1000000) { 90 | runSize = runSize << 1 91 | errorPercentage = percentageOfRunsWithOrderingErrors(runSize, new NaiveTester()) 92 | } 93 | //println(s"Got $errorPercentage% ordering errors on run size of $runSize") 94 | 95 | // Now show that this run length works fine with the MutexTester 96 | percentageOfRunsWithOrderingErrors(runSize, new MutexTester()) must_== 0 97 | } 98 | 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/RunQueueSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import scala.language.reflectiveCalls 7 | 8 | import org.specs2.mutable._ 9 | import java.util.concurrent.atomic.AtomicInteger 10 | import scala.concurrent.{ ExecutionContext, Promise, Future, Await } 11 | import scala.concurrent.duration.{ Duration, SECONDS } 12 | 13 | object RunQueueSpec extends Specification with ExecutionSpecification { 14 | 15 | val waitTime = Duration(20, SECONDS) 16 | 17 | trait QueueTester { 18 | def schedule(body: => Future[Unit])(implicit ec: ExecutionContext): Unit 19 | } 20 | 21 | class RunQueueTester extends QueueTester { 22 | val rq = new RunQueue() 23 | def schedule(body: => Future[Unit])(implicit ec: ExecutionContext) = rq.schedule(body) 24 | } 25 | 26 | class NaiveQueueTester extends QueueTester { 27 | def schedule(body: => Future[Unit])(implicit ec: ExecutionContext) = Future(body) 28 | } 29 | 30 | def countOrderingErrors(runs: Int, queueTester: QueueTester)(implicit ec: ExecutionContext): Future[Int] = { 31 | val result = Promise[Int]() 32 | val runCount = new AtomicInteger(0) 33 | val orderingErrors = new AtomicInteger(0) 34 | 35 | for (i <- 0 until runs) { 36 | queueTester.schedule { 37 | val observedRunCount = runCount.getAndIncrement() 38 | 39 | // Introduce another Future just to make things complicated :) 40 | Future { 41 | // We see observedRunCount != i then this task was run out of order 42 | if (observedRunCount != i) { 43 | orderingErrors.incrementAndGet() // Record the error 44 | } 45 | // If this is the last task, complete our result promise 46 | if ((observedRunCount + 1) >= runs) { 47 | result.success(orderingErrors.get) 48 | } 49 | } 50 | } 51 | } 52 | result.future 53 | } 54 | 55 | "RunQueue" should { 56 | 57 | "run code in order" in { 58 | import ExecutionContext.Implicits.global 59 | 60 | def percentageOfRunsWithOrderingErrors(runSize: Int, queueTester: QueueTester): Int = { 61 | val results: Seq[Future[Int]] = for (i <- 0 until 9) yield { 62 | countOrderingErrors(runSize, queueTester) 63 | } 64 | Await.result(Future.sequence(results), waitTime).filter(_ > 0).size * 10 65 | } 66 | 67 | // Iteratively increase the run size until we get observable errors 90% of the time 68 | // We want a high error rate because we want to then use the RunQueueTester 69 | // on the same run size and know that it is fixing up some problems. If the run size 70 | // is too small then the RunQueueTester probably isn't doing anything. We use 71 | // dynamic run sizing because the actual size that produces errors will vary 72 | // depending on the environment in which this test is run. 73 | var runSize = 8 // This usually reaches 8192 on my dev machine with 10 simultaneous queues 74 | var errorPercentage = 0 75 | while (errorPercentage < 90 && runSize < 1000000) { 76 | runSize = runSize << 1 77 | errorPercentage = percentageOfRunsWithOrderingErrors(runSize, new NaiveQueueTester()) 78 | } 79 | //println(s"Got $errorPercentage% ordering errors on run size of $runSize") 80 | 81 | // Now show that this run length works fine with the RunQueueTester 82 | percentageOfRunsWithOrderingErrors(runSize, new RunQueueTester()) must_== 0 83 | } 84 | 85 | "use the ExecutionContext exactly once per scheduled item" in { 86 | val rq = new RunQueue() 87 | mustExecute(1) { implicit runEC => 88 | val runFinished = Promise[Unit]() 89 | rq.schedule { 90 | runFinished.success(()) 91 | Future.successful(()) 92 | } 93 | Await.result(runFinished.future, waitTime) must_== (()) 94 | } 95 | } 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/SubscriberIterateeSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.specs2.mutable.Specification 7 | import org.specs2.specification.Scope 8 | import play.api.libs.iteratee.{ Step, Iteratee, Enumerator } 9 | import scala.concurrent.ExecutionContext.Implicits.global 10 | import scala.concurrent._ 11 | import scala.concurrent.duration._ 12 | 13 | object SubscriberIterateeSpec extends Specification { 14 | 15 | import SubscriberEvents._ 16 | 17 | def await[T](f: Future[T]) = Await.result(f, 5.seconds) 18 | 19 | trait TestEnv extends EventRecorder with SubscriberEvents with Scope { 20 | val iteratee = new SubscriberIteratee(subscriber) 21 | } 22 | 23 | "a subscriber iteratee" should { 24 | "not subscribe until fold is invoked" in new TestEnv { 25 | isEmptyAfterDelay() must beTrue 26 | } 27 | 28 | "not consume anything from the enumerator while there is no demand" in new TestEnv { 29 | iteratee.fold { folder => 30 | // Record if the folder was invoked 31 | record(folder) 32 | Future.successful(()) 33 | } 34 | next() must beAnInstanceOf[OnSubscribe] 35 | isEmptyAfterDelay() must beTrue 36 | } 37 | 38 | "not enter cont state until demand is requested" in new TestEnv { 39 | val step = iteratee.unflatten 40 | next() must beLike { 41 | case OnSubscribe(sub) => 42 | isEmptyAfterDelay() must beTrue 43 | step.isCompleted must beFalse 44 | sub.request(1) 45 | await(step) must beAnInstanceOf[Step.Cont[_, _]] 46 | } 47 | } 48 | 49 | "publish events one at a time in response to demand" in new TestEnv { 50 | val result = Enumerator(10, 20, 30) |>>> iteratee 51 | next() must beLike { 52 | case OnSubscribe(sub) => 53 | isEmptyAfterDelay() must beTrue 54 | sub.request(1) 55 | next() must_== OnNext(10) 56 | isEmptyAfterDelay() must beTrue 57 | sub.request(1) 58 | next() must_== OnNext(20) 59 | isEmptyAfterDelay() must beTrue 60 | sub.request(1) 61 | next() must_== OnNext(30) 62 | isEmptyAfterDelay() must beTrue 63 | sub.request(1) 64 | next() must_== OnComplete 65 | } 66 | await(result) must_== ((): Unit) 67 | } 68 | 69 | "publish events in batches in response to demand" in new TestEnv { 70 | val result = Enumerator(10, 20, 30) |>>> iteratee 71 | next() must beLike { 72 | case OnSubscribe(sub) => 73 | isEmptyAfterDelay() must beTrue 74 | sub.request(2) 75 | next() must_== OnNext(10) 76 | next() must_== OnNext(20) 77 | isEmptyAfterDelay() must beTrue 78 | sub.request(2) 79 | next() must_== OnNext(30) 80 | next() must_== OnComplete 81 | } 82 | await(result) must_== ((): Unit) 83 | } 84 | 85 | "publish events all at once in response to demand" in new TestEnv { 86 | val result = Enumerator(10, 20, 30) |>>> iteratee 87 | next() must beLike { 88 | case OnSubscribe(sub) => 89 | isEmptyAfterDelay() must beTrue 90 | sub.request(10) 91 | next() must_== OnNext(10) 92 | next() must_== OnNext(20) 93 | next() must_== OnNext(30) 94 | next() must_== OnComplete 95 | } 96 | await(result) must_== ((): Unit) 97 | } 98 | 99 | "become done when the stream is cancelled" in new TestEnv { 100 | val result = Enumerator(10, 20, 30) |>>> iteratee.flatMap(_ => Iteratee.getChunks[Int]) 101 | next() must beLike { 102 | case OnSubscribe(sub) => 103 | sub.request(1) 104 | next() must_== OnNext(10) 105 | sub.cancel() 106 | isEmptyAfterDelay() must beTrue 107 | } 108 | await(result) must_== Seq(20, 30) 109 | } 110 | 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/SubscriberIteratee.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams.{ Subscriber, Subscription } 7 | import play.api.libs.iteratee.concurrent.StateMachine 8 | import play.api.libs.iteratee._ 9 | 10 | import scala.concurrent.{ ExecutionContext, Future, Promise } 11 | 12 | private[streams] object SubscriberIteratee { 13 | 14 | sealed trait State 15 | 16 | /** 17 | * The iteratees fold method has not been called, and so it hasn't 18 | * subscribed to the subscriber yet. 19 | */ 20 | case object NotSubscribed extends State 21 | 22 | /** 23 | * There is currently no demand, and the iteratee isn't interested in any 24 | * demand. The stream is effectively idle. 25 | */ 26 | case object NoDemand extends State 27 | 28 | /** 29 | * The iteratee fold method has been invoked, so the iteratee is able to 30 | * consume elements, but there is no demand. 31 | * 32 | * @param demand A callback to be executed when there is demand 33 | * @param cancelled A callback to be executed if the subscription is cancelled 34 | */ 35 | case class AwaitingDemand(demand: () => Unit, cancelled: () => Unit) extends State 36 | 37 | /** 38 | * There is demand, but the iteratee is no elements to supply. 39 | * 40 | * @param n The number of elements demanded. 41 | */ 42 | case class Demand(n: Long) extends State 43 | 44 | /** 45 | * The subscription has been cancelled, the iteratee is done. 46 | */ 47 | case object Cancelled extends State 48 | 49 | } 50 | 51 | import streams.impl.SubscriberIteratee._ 52 | 53 | private[streams] class SubscriberIteratee[T](subscriber: Subscriber[T]) extends StateMachine[State](NotSubscribed) 54 | with Subscription with Iteratee[T, Unit] { self => 55 | 56 | def fold[B](folder: (Step[T, Unit]) => Future[B])(implicit ec: ExecutionContext): Future[B] = { 57 | val promise = Promise[B]() 58 | val pec = ec.prepare() 59 | exclusive { 60 | case NotSubscribed => 61 | state = awaitDemand(promise, folder, pec) 62 | subscriber.onSubscribe(this) 63 | case NoDemand => 64 | state = awaitDemand(promise, folder, pec) 65 | case AwaitingDemand(_, _) => 66 | throw new IllegalStateException("fold invoked while already waiting for demand") 67 | case Demand(n) => 68 | if (n == 1) { 69 | state = NoDemand 70 | } else { 71 | state = Demand(n - 1) 72 | } 73 | demand(promise, folder, pec) 74 | case Cancelled => 75 | cancelled(promise, folder, pec) 76 | } 77 | 78 | promise.future 79 | } 80 | 81 | private def awaitDemand[B](promise: Promise[B], folder: (Step[T, Unit]) => Future[B], ec: ExecutionContext): AwaitingDemand = { 82 | AwaitingDemand(() => demand(promise, folder, ec), () => cancelled(promise, folder, ec)) 83 | } 84 | 85 | private def demand[B](promise: Promise[B], folder: (Step[T, Unit]) => Future[B], ec: ExecutionContext): Unit = { 86 | Future { 87 | promise.completeWith(folder(Step.Cont[T, Unit] { 88 | case Input.EOF => 89 | subscriber.onComplete() 90 | Done(()) 91 | case Input.El(t) => 92 | subscriber.onNext(t) 93 | self 94 | case Input.Empty => 95 | self 96 | })) 97 | }(ec) 98 | } 99 | 100 | private def cancelled[B](promise: Promise[B], folder: (Step[T, Unit]) => Future[B], ec: ExecutionContext): Unit = { 101 | Future { 102 | promise.completeWith(folder(Step.Done((), Input.Empty))) 103 | }(ec) 104 | } 105 | 106 | def cancel() = exclusive { 107 | case AwaitingDemand(_, cancelled) => 108 | cancelled() 109 | state = Cancelled 110 | case _ => 111 | state = Cancelled 112 | } 113 | 114 | def request(n: Long) = exclusive { 115 | case NoDemand => 116 | state = Demand(n) 117 | case AwaitingDemand(demand, _) => 118 | demand() 119 | if (n == 1) { 120 | state = NoDemand 121 | } else { 122 | state = Demand(n - 1) 123 | } 124 | case Demand(old) => 125 | state = Demand(old + n) 126 | case Cancelled => 127 | // nop, 3.6 of reactive streams spec 128 | case NotSubscribed => 129 | throw new IllegalStateException("Demand requested before subscription made") 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/Parsing.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import scala.concurrent.Future 7 | 8 | import play.api.libs.iteratee.Execution.Implicits.{ defaultExecutionContext => dec } 9 | 10 | object Parsing { 11 | 12 | sealed trait MatchInfo[A] { 13 | def content: A 14 | def isMatch = this match { 15 | case Matched(_) => true 16 | case Unmatched(_) => false 17 | } 18 | } 19 | case class Matched[A](val content: A) extends MatchInfo[A] 20 | case class Unmatched[A](val content: A) extends MatchInfo[A] 21 | 22 | def search(needle: Array[Byte]): Enumeratee[Array[Byte], MatchInfo[Array[Byte]]] = new Enumeratee[Array[Byte], MatchInfo[Array[Byte]]] { 23 | val needleSize = needle.size 24 | val fullJump = needleSize 25 | val jumpBadCharecter: (Byte => Int) = { 26 | val map = Map(needle.dropRight(1).reverse.zipWithIndex.reverse: _*) //remove the last 27 | byte => map.get(byte).map(_ + 1).getOrElse(fullJump) 28 | } 29 | 30 | def applyOn[A](inner: Iteratee[MatchInfo[Array[Byte]], A]): Iteratee[Array[Byte], Iteratee[MatchInfo[Array[Byte]], A]] = { 31 | 32 | Iteratee.flatten(inner.fold1( 33 | (a, e) => Future.successful(Done(Done(a, e), Input.Empty: Input[Array[Byte]])), 34 | k => Future.successful(Cont(step(Array[Byte](), Cont(k)))), 35 | (err, r) => throw new Exception() 36 | )(dec)) 37 | 38 | } 39 | def scan(previousMatches: List[MatchInfo[Array[Byte]]], piece: Array[Byte], startScan: Int): (List[MatchInfo[Array[Byte]]], Array[Byte]) = { 40 | if (piece.length < needleSize) { 41 | (previousMatches, piece) 42 | } else { 43 | val fullMatch = Range(needleSize - 1, -1, -1).forall(scan => needle(scan) == piece(scan + startScan)) 44 | if (fullMatch) { 45 | val (prefix, suffix) = piece.splitAt(startScan) 46 | val (matched, left) = suffix.splitAt(needleSize) 47 | val newResults = previousMatches ++ List(Unmatched(prefix), Matched(matched)) filter (!_.content.isEmpty) 48 | 49 | if (left.length < needleSize) (newResults, left) else scan(newResults, left, 0) 50 | 51 | } else { 52 | val jump = jumpBadCharecter(piece(startScan + needleSize - 1)) 53 | val isFullJump = jump == fullJump 54 | val newScan = startScan + jump 55 | if (newScan + needleSize > piece.length) { 56 | val (prefix, suffix) = (piece.splitAt(startScan)) 57 | (previousMatches ++ List(Unmatched(prefix)), suffix) 58 | } else scan(previousMatches, piece, newScan) 59 | } 60 | } 61 | } 62 | 63 | def step[A](rest: Array[Byte], inner: Iteratee[MatchInfo[Array[Byte]], A])(in: Input[Array[Byte]]): Iteratee[Array[Byte], Iteratee[MatchInfo[Array[Byte]], A]] = { 64 | 65 | in match { 66 | case Input.Empty => Cont(step(rest, inner)) //here should rather pass Input.Empty along 67 | 68 | case Input.EOF => Done(inner, Input.El(rest)) 69 | 70 | case Input.El(chunk) => 71 | val all = rest ++ chunk 72 | def inputOrEmpty(a: Array[Byte]) = if (a.isEmpty) Input.Empty else Input.El(a) 73 | 74 | Iteratee.flatten(inner.fold1( 75 | (a, e) => Future.successful(Done(Done(a, e), inputOrEmpty(rest))), 76 | k => { 77 | val (result, suffix) = scan(Nil, all, 0) 78 | val fed = result.filter(!_.content.isEmpty).foldLeft(Future.successful(Array[Byte]() -> Cont(k))) { (p, m) => 79 | p.flatMap(i => i._2.fold1( 80 | (a, e) => Future.successful((i._1 ++ m.content, Done(a, e))), 81 | k => Future.successful((i._1, k(Input.El(m)))), 82 | (err, e) => throw new Exception() 83 | )(dec))(dec) 84 | } 85 | fed.flatMap { 86 | case (ss, i) => i.fold1( 87 | (a, e) => Future.successful(Done(Done(a, e), inputOrEmpty(ss ++ suffix))), 88 | k => Future.successful(Cont[Array[Byte], Iteratee[MatchInfo[Array[Byte]], A]]((in: Input[Array[Byte]]) => in match { 89 | case Input.EOF => Done(k(Input.El(Unmatched(suffix))), Input.EOF) //suffix maybe empty 90 | case other => step(ss ++ suffix, Cont(k))(other) 91 | })), 92 | (err, e) => throw new Exception() 93 | )(dec) 94 | }(dec) 95 | }, 96 | (err, e) => throw new Exception() 97 | )(dec)) 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/FuturePublisher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import play.api.libs.iteratee.concurrent.StateMachine 8 | import play.api.libs.iteratee.Execution 9 | 10 | import scala.concurrent.Future 11 | import scala.util.{ Failure, Success, Try } 12 | 13 | /* 14 | * Creates Subscriptions that link Subscribers to a Future. 15 | */ 16 | private[streams] trait FutureSubscriptionFactory[T] extends SubscriptionFactory[T] { 17 | 18 | def fut: Future[T] 19 | 20 | override def createSubscription[U >: T]( 21 | subr: Subscriber[U], 22 | onSubscriptionEnded: SubscriptionHandle[U] => Unit 23 | ) = { 24 | new FutureSubscription[T, U](fut, subr, onSubscriptionEnded) 25 | } 26 | 27 | } 28 | 29 | /** 30 | * Adapts an Future to a Publisher. 31 | */ 32 | private[streams] final class FuturePublisher[T]( 33 | val fut: Future[T] 34 | ) extends RelaxedPublisher[T] with FutureSubscriptionFactory[T] 35 | 36 | private[streams] object FutureSubscription { 37 | /** 38 | * Possible states of the Subscription. 39 | */ 40 | sealed trait State 41 | /** 42 | * A Subscription which hasn't had any elements requested. 43 | */ 44 | final case object AwaitingRequest extends State 45 | /** 46 | * A Subscription that has had at least one element requested. We only care 47 | * that the value requested is >1 because a Future will only emit a single 48 | * value. 49 | */ 50 | final case object Requested extends State 51 | /** 52 | * A Subscription completed by the Publisher. 53 | */ 54 | final case object Completed extends State 55 | /** 56 | * A Subscription cancelled by the Subscriber. 57 | */ 58 | final case object Cancelled extends State 59 | } 60 | 61 | import FutureSubscription._ 62 | 63 | private[streams] class FutureSubscription[T, U >: T]( 64 | fut: Future[T], 65 | subr: Subscriber[U], 66 | onSubscriptionEnded: SubscriptionHandle[U] => Unit 67 | ) 68 | extends StateMachine[State](initialState = AwaitingRequest) 69 | with Subscription with SubscriptionHandle[U] { 70 | 71 | // SubscriptionHandle methods 72 | 73 | override def start(): Unit = { 74 | fut.value match { 75 | case Some(Failure(t)) => 76 | subscriber.onError(t) 77 | onSubscriptionEnded(this) 78 | case _ => 79 | subscriber.onSubscribe(this) 80 | } 81 | } 82 | 83 | override def isActive: Boolean = state match { 84 | case AwaitingRequest | Requested => true 85 | case Cancelled | Completed => false 86 | } 87 | 88 | override def subscriber: Subscriber[U] = subr 89 | 90 | // Streams methods 91 | 92 | override def request(elements: Long): Unit = { 93 | if (elements <= 0) throw new IllegalArgumentException(s"The number of requested elements must be > 0: requested $elements elements") 94 | exclusive { 95 | case AwaitingRequest => 96 | state = Requested 97 | // When we receive a request for an element, we trigger a call to 98 | // onFutureCompleted. We call it immediately if we can, otherwise we 99 | // schedule the call for when the Future is completed. 100 | fut.value match { 101 | case Some(result) => 102 | onFutureCompleted(result) 103 | case None => 104 | // Safe to use trampoline because onFutureCompleted only schedules async operations 105 | fut.onComplete(onFutureCompleted)(Execution.trampoline) 106 | } 107 | case _ => 108 | () 109 | } 110 | } 111 | 112 | override def cancel(): Unit = exclusive { 113 | case AwaitingRequest => 114 | state = Cancelled 115 | case Requested => 116 | state = Cancelled 117 | case _ => 118 | () 119 | } 120 | 121 | /** 122 | * Called when both an element has been requested and the Future is 123 | * completed. Calls onNext/onComplete or onError on the Subscriber. 124 | */ 125 | private def onFutureCompleted(result: Try[T]): Unit = exclusive { 126 | case AwaitingRequest => 127 | throw new IllegalStateException("onFutureCompleted shouldn't be called when in state AwaitingRequest") 128 | case Requested => 129 | state = Completed 130 | result match { 131 | case Success(null) => 132 | subr.onError(new NullPointerException("Future completed with a null value that cannot be sent by a Publisher")) 133 | case Success(value) => 134 | subr.onNext(value) 135 | subr.onComplete() 136 | case Failure(t) => 137 | subr.onError(t) 138 | } 139 | onSubscriptionEnded(this) 140 | case Cancelled => 141 | () 142 | case Completed => 143 | throw new IllegalStateException("onFutureCompleted shouldn't be called when already in state Completed") 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/Execution.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import java.util.ArrayDeque 7 | import scala.annotation.tailrec 8 | import scala.concurrent.{ ExecutionContextExecutor, ExecutionContext } 9 | 10 | /** 11 | * Contains the default ExecutionContext used by Iteratees. 12 | */ 13 | object Execution { 14 | 15 | def defaultExecutionContext: ExecutionContext = Implicits.defaultExecutionContext 16 | 17 | object Implicits { 18 | implicit def defaultExecutionContext: ExecutionContext = Execution.trampoline 19 | implicit def trampoline: ExecutionContextExecutor = Execution.trampoline 20 | } 21 | 22 | /** 23 | * Executes in the current thread. Uses a thread local trampoline to make sure the stack 24 | * doesn't overflow. Since this ExecutionContext executes on the current thread, it should 25 | * only be used to run small bits of fast-running code. We use it here to run the internal 26 | * iteratee code. 27 | * 28 | * Blocking should be strictly avoided as it could hog the current thread. 29 | * Also, since we're running on a single thread, blocking code risks deadlock. 30 | */ 31 | object trampoline extends ExecutionContextExecutor { 32 | 33 | /* 34 | * A ThreadLocal value is used to track the state of the trampoline in the current 35 | * thread. When a Runnable is added to the trampoline it uses the ThreadLocal to 36 | * see if the trampoline is already running in the thread. If so, it starts the 37 | * trampoline. When it finishes, it checks the ThreadLocal to see if any Runnables 38 | * have subsequently been scheduled for execution. It runs all the Runnables until 39 | * there are no more to exit, then it clears the ThreadLocal and stops running. 40 | * 41 | * ThreadLocal states: 42 | * - null => 43 | * - no Runnable running: trampoline is inactive in the current thread 44 | * - Empty => 45 | * - a Runnable is running and trampoline is active 46 | * - no more Runnables are enqueued for execution after the current Runnable 47 | * completes 48 | * - next: Runnable => 49 | * - a Runnable is running and trampoline is active 50 | * - one Runnable is scheduled for execution after the current Runnable 51 | * completes 52 | * - queue: ArrayDeque[Runnable] => 53 | * - a Runnable is running and trampoline is active 54 | * - two or more Runnables are scheduled for execution after the current 55 | * Runnable completes 56 | */ 57 | private val local = new ThreadLocal[AnyRef] 58 | 59 | /** Marks an empty queue (see docs for `local`). */ 60 | private object Empty 61 | 62 | def execute(runnable: Runnable): Unit = { 63 | local.get match { 64 | case null => try { 65 | // Trampoline is inactive in this thread so start it up! 66 | // The queue of Runnables to run after this one 67 | // is initially empty. 68 | local.set(Empty) 69 | runnable.run() 70 | executeScheduled() 71 | } finally { 72 | // We've run all the Runnables, so show that the 73 | // trampoline has been shut down. 74 | local.set(null) 75 | } 76 | 77 | case Empty => 78 | // Add this Runnable to our empty queue 79 | local.set(runnable) 80 | 81 | case next: Runnable => 82 | // Convert the single queued Runnable into an ArrayDeque 83 | // so we can schedule 2+ Runnables 84 | val runnables = new ArrayDeque[Runnable](4) 85 | runnables.addLast(next) 86 | runnables.addLast(runnable) 87 | local.set(runnables) 88 | 89 | case arrayDeque: ArrayDeque[_] => 90 | // Add this Runnable to the end of the existing ArrayDeque 91 | val runnables = arrayDeque.asInstanceOf[ArrayDeque[Runnable]] 92 | runnables.addLast(runnable) 93 | 94 | case illegal => throw new IllegalStateException( 95 | s"Unsupported trampoline ThreadLocal value: $illegal" 96 | ) 97 | } 98 | } 99 | 100 | /** 101 | * Run all tasks that have been scheduled in the ThreadLocal. 102 | */ 103 | @tailrec 104 | private def executeScheduled(): Unit = local.get match { 105 | case Empty => ( /* Nothing to run */ ) 106 | 107 | case next: Runnable => 108 | // Mark the queue of Runnables after this one as empty 109 | local.set(Empty) 110 | // Run the only scheduled Runnable 111 | next.run() 112 | // Recurse in case more Runnables were added 113 | executeScheduled() 114 | 115 | case arrayDeque: ArrayDeque[_] => 116 | val runnables = arrayDeque.asInstanceOf[ArrayDeque[Runnable]] 117 | // Rather than recursing, we can use a more efficient 118 | // while loop. The value of the ThreadLocal will stay as 119 | // an ArrayDeque until all the scheduled Runnables have been 120 | // run. 121 | while (!runnables.isEmpty) { 122 | val runnable = runnables.removeFirst() 123 | runnable.run() 124 | } 125 | 126 | case illegal => throw new IllegalStateException( 127 | s"Unsupported trampoline ThreadLocal value: $illegal" 128 | ) 129 | } 130 | 131 | def reportFailure(t: Throwable): Unit = t.printStackTrace() 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/RunQueue.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import scala.annotation.tailrec 7 | import scala.concurrent.{ ExecutionContext, Future } 8 | import java.util.concurrent.atomic.AtomicReference 9 | 10 | /** 11 | * Runs asynchronous operations in order. Operations are queued until 12 | * they can be run. Each item is added to a schedule, then each item 13 | * in the schedule is executed in order. 14 | * 15 | * {{{ 16 | * val runQueue = new RunQueue() 17 | * 18 | * // This operation will run first. It completes when 19 | * // the future it returns is completed. 20 | * runQueue.schedule { 21 | * Future { ... do some stuff ... } 22 | * } 23 | * 24 | * // This operation will run second. It will start running 25 | * // when the previous operation's futures complete. 26 | * runQueue.schedule { 27 | * future1.flatMap(x => future2.map(y => x + y)) 28 | * } 29 | * 30 | * // This operation will run when the second operation's 31 | * // future finishes. It's a simple synchronous operation. 32 | * runQueue.scheduleSimple { 33 | * 25 34 | * } 35 | * }}} 36 | * 37 | * Unlike solutions built around a standard concurrent queue, there is no 38 | * need to use a separate thread to read from the queue and execute each 39 | * operation. The RunQueue runner performs both scheduling and 40 | * executions of operations internally without the need for a separate 41 | * thread. This means the RunQueue doesn't consume any resources 42 | * when it isn't being used. 43 | * 44 | * No locks are held by this class, only atomic operations are used. 45 | */ 46 | private[play] final class RunQueue { 47 | 48 | import RunQueue._ 49 | 50 | /** 51 | * The state of the RunQueue, either Inactive or Runnning. 52 | */ 53 | private val state = new AtomicReference[Vector[Op]](null) 54 | 55 | /** 56 | * Schedule an operation to be run. The operation is considered 57 | * complete when the Future that it returns is completed. In other words, 58 | * the next operation will not be started until the future is completed. 59 | * 60 | * Successive calls to the `run` and `runSynchronous` methods use an 61 | * atomic value to guarantee ordering (a *happens-before* relationship). 62 | * 63 | * The operation will execute in the given ExecutionContext. 64 | */ 65 | def schedule[A](body: => Future[A])(implicit ec: ExecutionContext): Unit = { 66 | schedule(Op(() => body.asInstanceOf[Future[Unit]], ec.prepare)) 67 | } 68 | 69 | /** 70 | * Schedule a simple synchronous operation to be run. The operation is considered 71 | * complete when it finishes executing. In other words, the next operation will begin 72 | * execution immediately when this operation finishes execution. 73 | * 74 | * This method is equivalent to 75 | * {{{ 76 | * schedule { 77 | * body 78 | * Future.successful(()) 79 | * } 80 | * }}} 81 | * 82 | * Successive calls to the `run` and `runSynchronous` methods use an 83 | * atomic value to guarantee ordering (a *happens-before* relationship). 84 | * 85 | * The operation will execute in the given ExecutionContext. 86 | */ 87 | def scheduleSimple(body: => Unit)(implicit ec: ExecutionContext): Unit = { 88 | schedule { 89 | body 90 | Future.successful(()) 91 | } 92 | } 93 | 94 | /** 95 | * Schedule a reified operation for execution. If no other operations 96 | * are currently executing then this operation will be started immediately. 97 | * But if there are other operations currently running then this operation 98 | * be added to the pending queue of operations awaiting execution. 99 | * 100 | * This method encapsulates an atomic compare-and-set operation, therefore 101 | * it may be retried. 102 | */ 103 | @tailrec 104 | private def schedule(op: Op): Unit = { 105 | val prevState = state.get 106 | val newState = prevState match { 107 | case null => Vector.empty 108 | case pending => pending :+ op 109 | } 110 | if (state.compareAndSet(prevState, newState)) { 111 | prevState match { 112 | case null => 113 | // We've update the state to say that we're running an op, 114 | // so we need to actually start it running. 115 | execute(op) 116 | case _ => 117 | } 118 | } else schedule(op) // Try again 119 | } 120 | 121 | private def execute(op: Op): Unit = { 122 | val f1: Future[Future[Unit]] = Future(op.thunk())(op.ec) 123 | val f2: Future[Unit] = f1.flatMap(identity)(Execution.trampoline) 124 | f2.onComplete(_ => opExecutionComplete())(Execution.trampoline) 125 | } 126 | 127 | /** 128 | * *De*schedule a reified operation for execution. If no other operations 129 | * are pending, then the RunQueue will enter an inactive state. 130 | * Otherwise, the first pending item will be scheduled for execution. 131 | */ 132 | @tailrec 133 | private def opExecutionComplete(): Unit = { 134 | val prevState = state.get 135 | val newState = prevState match { 136 | case null => throw new IllegalStateException("Can't be inactive, must have a queue of pending elements") 137 | case pending if pending.isEmpty => null 138 | case pending => pending.tail 139 | } 140 | if (state.compareAndSet(prevState, newState)) { 141 | prevState match { 142 | // We have a pending operation to execute 143 | case pending if !pending.isEmpty => execute(pending.head) 144 | case _ => 145 | } 146 | } else opExecutionComplete() // Try again 147 | } 148 | 149 | } 150 | 151 | private object RunQueue { 152 | 153 | /** 154 | * A reified operation to be executed. 155 | * 156 | * @param thunk The logic to execute. 157 | * @param ec The ExecutionContext to use for execution. Already prepared. 158 | */ 159 | final case class Op(thunk: () => Future[Unit], ec: ExecutionContext) 160 | 161 | } 162 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/TraversableIteratee.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import play.api.libs.iteratee.Execution.Implicits.{ defaultExecutionContext => dec } 7 | import scala.concurrent.{ ExecutionContext, Future } 8 | 9 | /** 10 | * @define paramEcSingle @param ec The context to execute the supplied function with. The context is prepared on the calling thread before being used. 11 | */ 12 | object Traversable { 13 | 14 | /** 15 | * A partially-applied function returned by the `head` method. 16 | */ 17 | trait Head[E] { 18 | def apply[A](implicit p: E => scala.collection.TraversableLike[A, E]): Iteratee[E, Option[A]] 19 | } 20 | 21 | def head[E] = new Head[E] { 22 | def apply[A](implicit p: E => scala.collection.TraversableLike[A, E]): Iteratee[E, Option[A]] = { 23 | 24 | def step: K[E, Option[A]] = { 25 | case Input.Empty => Cont(step) 26 | case Input.EOF => Done(None, Input.EOF) 27 | case Input.El(xs) if !xs.isEmpty => Done(Some(xs.head), Input.El(xs.tail)) 28 | case Input.El(empty) => Cont(step) 29 | } 30 | Cont(step) 31 | } 32 | } 33 | 34 | def takeUpTo[M](count: Long)(implicit p: M => scala.collection.TraversableLike[_, M]): Enumeratee[M, M] = new Enumeratee[M, M] { 35 | 36 | def applyOn[A](it: Iteratee[M, A]): Iteratee[M, Iteratee[M, A]] = { 37 | 38 | def step(inner: Iteratee[M, A], leftToTake: Long)(in: Input[M]): Iteratee[M, Iteratee[M, A]] = { 39 | in match { 40 | case in @ Input.El(e) => 41 | inner.pureFlatFold { 42 | case Step.Cont(k) if leftToTake < Int.MaxValue => e.splitAt(leftToTake.toInt) match { 43 | case (all, x) if x.isEmpty => Cont(step(k(Input.El(all)), leftToTake - all.size)) 44 | case (x, left) if x.isEmpty => Done(inner, Input.El(left)) 45 | case (toPush, left) => Done(k(Input.El(toPush)), Input.El(left)) 46 | } 47 | case Step.Cont(k) => Cont(step(k(Input.El(e)), leftToTake - e.size)) 48 | case _ => Done(inner, in) 49 | } 50 | 51 | case Input.EOF => Done(inner, Input.EOF) 52 | 53 | case Input.Empty => Cont(step(inner, leftToTake)) 54 | } 55 | 56 | } 57 | Cont(step(it, count)) 58 | } 59 | } 60 | 61 | def take[M](count: Int)(implicit p: M => scala.collection.TraversableLike[_, M]): Enumeratee[M, M] = new Enumeratee[M, M] { 62 | 63 | def applyOn[A](it: Iteratee[M, A]): Iteratee[M, Iteratee[M, A]] = { 64 | 65 | def step(inner: Iteratee[M, A], leftToTake: Int)(in: Input[M]): Iteratee[M, Iteratee[M, A]] = { 66 | in match { 67 | case in @ Input.El(e) => 68 | e.splitAt(leftToTake) match { 69 | case (all, x) if x.isEmpty => inner.pureFlatFold { 70 | case Step.Done(_, _) => Cont(step(inner, (leftToTake - all.size))) 71 | case Step.Cont(k) => Cont(step(k(Input.El(all)), (leftToTake - all.size))) 72 | case Step.Error(_, _) => Cont(step(inner, (leftToTake - all.size))) 73 | } 74 | case (x, left) if x.isEmpty => Done(inner, Input.El(left)) 75 | case (toPush, left) => Done(inner.pureFlatFold { case Step.Cont(k) => k(Input.El(toPush)); case _ => inner }, Input.El(left)) 76 | } 77 | 78 | case Input.EOF => Done(inner, Input.EOF) 79 | 80 | case Input.Empty => Cont(step(inner, leftToTake)) 81 | } 82 | 83 | } 84 | Cont(step(it, count)) 85 | 86 | } 87 | } 88 | 89 | import Enumeratee.CheckDone 90 | 91 | /** 92 | * Splits a stream of traversable input elements on a predicate. Yields all input up until 93 | * the predicate matches within an input element. The input following a match is unprocessed. 94 | * 95 | * @param p The predicate to split input on. 96 | * $paramEcSingle 97 | */ 98 | def splitOnceAt[M, E](p: E => Boolean)(implicit traversableLike: M => scala.collection.TraversableLike[E, M], ec: ExecutionContext): Enumeratee[M, M] = new CheckDone[M, M] { 99 | val pec = ec.prepare() 100 | 101 | def step[A](k: K[M, A]): K[M, Iteratee[M, A]] = { 102 | 103 | case in @ Input.El(e) => 104 | Iteratee.flatten(Future(e.span(p))(pec).map { 105 | case (prefix, suffix) if suffix.isEmpty => new CheckDone[M, M] { def continue[A](k: K[M, A]) = Cont(step(k)) } &> k(Input.El(prefix)) 106 | case (prefix, suffix) => Done(if (prefix.isEmpty) Cont(k) else k(Input.El(prefix)), Input.El(suffix.drop(1))) 107 | }(dec)) 108 | 109 | case Input.Empty => 110 | new CheckDone[M, M] { def continue[A](k: K[M, A]) = Cont(step(k)) } &> k(Input.Empty) 111 | 112 | case Input.EOF => Done(Cont(k), Input.EOF) 113 | 114 | } 115 | 116 | def continue[A](k: K[M, A]) = Cont(step(k)) 117 | 118 | } 119 | 120 | def drop[M](count: Int)(implicit p: M => scala.collection.TraversableLike[_, M]): Enumeratee[M, M] = new Enumeratee[M, M] { 121 | 122 | def applyOn[A](inner: Iteratee[M, A]): Iteratee[M, Iteratee[M, A]] = { 123 | 124 | def step(it: Iteratee[M, A], leftToDrop: Int)(in: Input[M]): Iteratee[M, Iteratee[M, A]] = { 125 | in match { 126 | case in @ Input.El(e) => 127 | val left = leftToDrop - e.size 128 | left match { 129 | case i if i > 0 => Cont(step(it, left)) 130 | case i => 131 | val toPass = if (i < 0) Input.El(e.drop(leftToDrop)) else Input.Empty 132 | it.pureFlatFold { 133 | case Step.Cont(k) => Enumeratee.passAlong.applyOn(k(toPass)) 134 | case _ => Done(it, toPass) 135 | } 136 | } 137 | case Input.Empty => Cont(step(it, leftToDrop)) 138 | 139 | case Input.EOF => Done(it, Input.EOF) 140 | } 141 | } 142 | 143 | Cont(step(inner, count)) 144 | 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /iteratees/src/main/scala/play/api/libs/iteratee/CharEncoding.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import java.io.{ ByteArrayOutputStream, StringWriter } 7 | import java.nio.{ ByteBuffer, CharBuffer } 8 | import java.nio.charset._ 9 | import play.api.libs.iteratee.Execution.defaultExecutionContext 10 | import scala.annotation.tailrec 11 | 12 | /** 13 | * [[Enumeratee]]s for converting chunks of bytes to chunks of chars, and vice-versa. 14 | * 15 | * These methods can handle cases where characters straddle a chunk boundary, and redistribute the data. 16 | * An erroneous encoding or an incompatible decoding causes a [[Step.Error]]. 17 | * 18 | * @define javadoc http://docs.oracle.com/javase/8/docs/api 19 | */ 20 | object CharEncoding { 21 | 22 | private trait Coder[From, To] extends Enumeratee[From, To] { 23 | private type Inner[A] = Iteratee[To, A] 24 | 25 | protected def empty: From 26 | 27 | protected def code(data: From, last: Boolean): Either[CoderResult, (To, From)] 28 | 29 | protected def concat(a: From, b: From): From 30 | 31 | private def step[A](initial: From = empty)(it: Inner[A]): K[From, Inner[A]] = { 32 | case in @ Input.El(chars) => 33 | it.pureFlatFold[From, Inner[A]] { 34 | case Step.Cont(k) => 35 | code(concat(initial, chars), false).fold({ result => 36 | Error(s"coding error: $result", in) 37 | }, { 38 | case (bytes, remaining) => 39 | val newIt = Iteratee.flatten(it.feed(Input.El(bytes))) 40 | Cont(step(remaining)(newIt)) 41 | }) 42 | case _ => Done(it) 43 | }(defaultExecutionContext) 44 | case in @ Input.Empty => 45 | val newIt = Iteratee.flatten(it.feed(in)) 46 | Cont(step(initial)(newIt)) 47 | case in @ Input.EOF => 48 | code(initial, true).fold({ result => 49 | Error(s"coding error: $result", in) 50 | }, { 51 | case (string, remaining) => 52 | val newIt = Iteratee.flatten(it.feed(Input.El(string)).flatMap(_.feed(in))(defaultExecutionContext)) 53 | Done(newIt) 54 | }) 55 | } 56 | 57 | def applyOn[A](inner: Inner[A]) = Cont(step()(inner)) 58 | } 59 | 60 | def decode(charset: Charset): Enumeratee[Array[Byte], String] = new Coder[Array[Byte], String] { 61 | protected val empty = Array[Byte]() 62 | 63 | protected def concat(a: Array[Byte], b: Array[Byte]) = a ++ b 64 | 65 | protected def code(bytes: Array[Byte], last: Boolean) = { 66 | val decoder = charset.newDecoder 67 | 68 | val byteBuffer = ByteBuffer.wrap(bytes) 69 | // at least 2, for UTF-32 70 | val charBuffer = CharBuffer.allocate(2 max math.ceil(bytes.length * decoder.averageCharsPerByte).toInt) 71 | val out = new StringWriter 72 | 73 | @tailrec 74 | def process(charBuffer: CharBuffer): CoderResult = { 75 | val result = decoder.decode(byteBuffer, charBuffer, true) 76 | out.write(charBuffer.array, 0, charBuffer.position) 77 | if (result.isOverflow) { 78 | if (charBuffer.position == 0) { 79 | // shouldn't happen for most encodings 80 | process(CharBuffer.allocate(2 * charBuffer.capacity)) 81 | } else { 82 | charBuffer.clear() 83 | process(charBuffer) 84 | } 85 | } else { 86 | result 87 | } 88 | } 89 | val result = process(charBuffer) 90 | 91 | if (result.isUnmappable || last && result.isMalformed) { 92 | Left(result) 93 | } else { 94 | val remaining = if (result.isError) bytes.drop(byteBuffer.position) else empty 95 | Right((out.toString, remaining)) 96 | } 97 | } 98 | } 99 | 100 | /** 101 | * @throws scala.Exception [[$javadoc/java/nio/charset/UnsupportedCharsetException.html UnsupportedCharsetException]] if no 102 | * charset could be found with the provided name 103 | */ 104 | def decode(charset: String): Enumeratee[Array[Byte], String] = decode(Charset.forName(charset)) 105 | 106 | def encode(charset: Charset): Enumeratee[String, Array[Byte]] = new Coder[String, Array[Byte]] { 107 | protected def empty = "" 108 | 109 | protected def concat(a: String, b: String) = a + b 110 | 111 | protected def code(chars: String, last: Boolean) = { 112 | val encoder = charset.newEncoder 113 | 114 | val charBuffer = CharBuffer.wrap(chars) 115 | // at least 6, for UTF-8 116 | val byteBuffer = ByteBuffer.allocate(6 max math.ceil(chars.length * encoder.averageBytesPerChar).toInt) 117 | val out = new ByteArrayOutputStream 118 | @tailrec 119 | def process(byteBuffer: ByteBuffer): CoderResult = { 120 | val result = encoder.encode(charBuffer, byteBuffer, true) 121 | out.write(byteBuffer.array, 0, byteBuffer.position) 122 | if (result.isOverflow) { 123 | if (byteBuffer.position == 0) { 124 | // shouldn't happen for most encodings 125 | process(ByteBuffer.allocate(2 * byteBuffer.capacity)) 126 | } else { 127 | byteBuffer.clear() 128 | process(byteBuffer) 129 | } 130 | } else { 131 | result 132 | } 133 | } 134 | val result = process(byteBuffer) 135 | if (result.isUnmappable || last && result.isMalformed) { 136 | Left(result) 137 | } else { 138 | val remaining = if (result.isError) chars.drop(charBuffer.position) else "" 139 | val bytes = out.toByteArray 140 | val bytesWithoutBom = if (charset.name.startsWith("UTF-") && bytes.length >= 2 && bytes(0) == 0xfe.toByte && bytes(1) == 0xff.toByte) { 141 | bytes.drop(2) 142 | } else { 143 | bytes 144 | } 145 | Right((bytesWithoutBom, remaining)) 146 | } 147 | } 148 | 149 | } 150 | 151 | /** 152 | * @throws scala.Exception [[$javadoc/java/nio/charset/UnsupportedCharsetException.html UnsupportedCharsetException]] if no 153 | * charset could be found with the provided name 154 | */ 155 | def encode(charset: String): Enumeratee[String, Array[Byte]] = encode(Charset.forName(charset)) 156 | 157 | } 158 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/EnumeratorPublisher.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import play.api.libs.iteratee.concurrent.StateMachine 8 | import play.api.libs.iteratee._ 9 | 10 | import scala.concurrent.Promise 11 | import scala.language.higherKinds 12 | import scala.util.{ Failure, Success, Try } 13 | 14 | /** 15 | * Creates Subscriptions that link Subscribers to an Enumerator. 16 | */ 17 | private[streams] trait EnumeratorSubscriptionFactory[T] extends SubscriptionFactory[T] { 18 | 19 | def enum: Enumerator[T] 20 | def emptyElement: Option[T] 21 | 22 | override def createSubscription[U >: T]( 23 | subr: Subscriber[U], 24 | onSubscriptionEnded: SubscriptionHandle[U] => Unit 25 | ) = { 26 | new EnumeratorSubscription[T, U](enum, emptyElement, subr, onSubscriptionEnded) 27 | } 28 | 29 | } 30 | 31 | /** 32 | * Adapts an Enumerator to a Publisher. 33 | */ 34 | private[streams] final class EnumeratorPublisher[T]( 35 | val enum: Enumerator[T], 36 | val emptyElement: Option[T] = None 37 | ) extends RelaxedPublisher[T] with EnumeratorSubscriptionFactory[T] 38 | 39 | private[streams] object EnumeratorSubscription { 40 | 41 | /** 42 | * Internal state of the Publisher. 43 | */ 44 | sealed trait State[+T] 45 | /** 46 | * An active Subscription with n outstanding requested elements. 47 | * @param n Elements that have been requested by the Subscriber. May be 0. 48 | * @param attached The attached Iteratee we're using to read from the 49 | * Enumerator. Will be Unattached until the first element is requested. 50 | */ 51 | final case class Requested[T](n: Long, attached: IterateeState[T]) extends State[T] 52 | /** 53 | * A Subscription completed by the Publisher. 54 | */ 55 | final case object Completed extends State[Nothing] 56 | /** 57 | * A Subscription cancelled by the Subscriber. 58 | */ 59 | final case object Cancelled extends State[Nothing] 60 | 61 | /** 62 | * We use an Iteratee to read from the Enumerator. Controlled by the 63 | * extendIteratee method. 64 | */ 65 | sealed trait IterateeState[+T] 66 | /** 67 | * The Iteratee state before any elements have been requested, before 68 | * we've attached an Iteratee to the Enumerator. 69 | */ 70 | final case object Unattached extends IterateeState[Nothing] 71 | /** 72 | * The Iteratee state when we're reading from the Enumerator. 73 | */ 74 | final case class Attached[T](link: Promise[Iteratee[T, Unit]]) extends IterateeState[T] 75 | 76 | } 77 | 78 | import streams.impl.EnumeratorSubscription._ 79 | 80 | /** 81 | * Adapts an Enumerator to a Publisher. 82 | */ 83 | private[streams] class EnumeratorSubscription[T, U >: T]( 84 | enum: Enumerator[T], 85 | emptyElement: Option[T], 86 | subr: Subscriber[U], 87 | onSubscriptionEnded: SubscriptionHandle[U] => Unit 88 | ) 89 | extends StateMachine[State[T]](initialState = Requested[T](0, Unattached)) 90 | with Subscription with SubscriptionHandle[U] { 91 | 92 | // SubscriptionHandle methods 93 | 94 | override def start(): Unit = { 95 | subr.onSubscribe(this) 96 | } 97 | 98 | override def subscriber: Subscriber[U] = subr 99 | 100 | override def isActive: Boolean = { 101 | // run immediately, don't need to wait for exclusive access 102 | state match { 103 | case Requested(_, _) => true 104 | case Completed | Cancelled => false 105 | } 106 | } 107 | 108 | // Streams methods 109 | 110 | override def request(elements: Long): Unit = { 111 | if (elements <= 0) throw new IllegalArgumentException(s"The number of requested elements must be > 0: requested $elements elements") 112 | exclusive { 113 | case Requested(0, its) => 114 | state = Requested(elements, extendIteratee(its)) 115 | case Requested(n, its) => 116 | state = Requested(n + elements, its) 117 | case Completed | Cancelled => 118 | () // FIXME: Check rules 119 | } 120 | } 121 | 122 | override def cancel(): Unit = exclusive { 123 | case Requested(_, its) => 124 | val cancelLink: Iteratee[T, Unit] = Done(()) 125 | its match { 126 | case Unattached => 127 | enum(cancelLink) 128 | case Attached(link0) => 129 | link0.success(cancelLink) 130 | } 131 | state = Cancelled 132 | case Cancelled | Completed => 133 | () 134 | } 135 | 136 | // Methods called by the iteratee when it receives input 137 | 138 | /** 139 | * Called when the Iteratee received Input.El, or when it recived 140 | * Input.Empty and the Publisher's `emptyElement` is Some(el). 141 | */ 142 | private def elementEnumerated(el: T): Unit = exclusive { 143 | case Requested(1, its) => 144 | subr.onNext(el) 145 | state = Requested(0, its) 146 | case Requested(n, its) => 147 | subr.onNext(el) 148 | state = Requested(n - 1, extendIteratee(its)) 149 | case Cancelled => 150 | () 151 | case Completed => 152 | throw new IllegalStateException("Shouldn't receive another element once completed") 153 | } 154 | 155 | /** 156 | * Called when the Iteratee received Input.Empty and the Publisher's 157 | * `emptyElement` value is `None` 158 | */ 159 | private def emptyEnumerated(): Unit = exclusive { 160 | case Requested(n, its) => 161 | state = Requested(n, extendIteratee(its)) 162 | case Cancelled => 163 | () 164 | case Completed => 165 | throw new IllegalStateException("Shouldn't receive an empty input once completed") 166 | } 167 | 168 | /** 169 | * Called when the Iteratee received Input.EOF 170 | */ 171 | private def eofEnumerated(): Unit = exclusive { 172 | case Requested(_, _) => 173 | subr.onComplete() 174 | state = Completed 175 | case Cancelled => 176 | () 177 | case Completed => 178 | throw new IllegalStateException("Shouldn't receive EOF once completed") 179 | } 180 | 181 | /** 182 | * Called when the enumerator is complete. If the enumerator didn't feed 183 | * EOF into the iteratee, then this is where the subscriber will be 184 | * completed. If the enumerator encountered an error, this error will be 185 | * sent to the subscriber. 186 | */ 187 | private def enumeratorApplicationComplete(result: Try[_]): Unit = exclusive { 188 | case Requested(_, _) => 189 | state = Completed 190 | result match { 191 | case Failure(error) => 192 | subr.onError(error) 193 | case Success(_) => 194 | subr.onComplete() 195 | } 196 | case Cancelled => 197 | () 198 | case Completed => 199 | () // Subscriber was already completed when the enumerator produced EOF 200 | } 201 | 202 | /** 203 | * Called when we want to read an input element from the Enumerator. This 204 | * method attaches an Iteratee to the end of the Iteratee chain. The 205 | * Iteratee it attaches will call one of the `*Enumerated` methods when 206 | * it recesives input. 207 | */ 208 | private def extendIteratee(its: IterateeState[T]): IterateeState[T] = { 209 | val link = Promise[Iteratee[T, Unit]]() 210 | val linkIteratee: Iteratee[T, Unit] = Iteratee.flatten(link.future) 211 | val iteratee: Iteratee[T, Unit] = Cont { input => 212 | input match { 213 | case Input.El(el) => 214 | elementEnumerated(el) 215 | case Input.Empty => 216 | emptyElement match { 217 | case None => emptyEnumerated() 218 | case Some(el) => elementEnumerated(el) 219 | } 220 | case Input.EOF => 221 | eofEnumerated() 222 | } 223 | linkIteratee 224 | } 225 | its match { 226 | case Unattached => 227 | enum(iteratee).onComplete(enumeratorApplicationComplete)(Execution.trampoline) 228 | case Attached(link0) => 229 | link0.success(iteratee) 230 | } 231 | Attached(link) 232 | } 233 | 234 | } 235 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/IterateeStreams.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams 5 | 6 | import org.reactivestreams._ 7 | import play.api.libs.iteratee._ 8 | 9 | import scala.concurrent.{ ExecutionContext, Future, Promise } 10 | 11 | /** 12 | * Methods to adapt Futures, Promises, Iteratees and Enumerators to 13 | * and from Reactive Streams' Publishers and Subscribers. 14 | */ 15 | object IterateeStreams { 16 | 17 | /** 18 | * Adapt a Future into a Publisher. For a successful Future the 19 | * Publisher will emit a single element with the value. For a failed 20 | * Future the Publisher will emit an onError event. 21 | */ 22 | def futureToPublisher[T](fut: Future[T]): Publisher[T] = new impl.FuturePublisher(fut) 23 | 24 | /** 25 | * Adapt a Promise into a Subscriber. The Subscriber accepts a 26 | * single value which is used to complete the Promise. If the 27 | * Subscriber's onError method is called then the Promise is 28 | * completed with a failure. 29 | * 30 | * A call to onError after onNext is called will be ignored. 31 | */ 32 | def promiseToSubscriber[T](prom: Promise[T]): Subscriber[T] = new impl.PromiseSubscriber(prom) 33 | 34 | /** 35 | * Adapt a Promise into a Processor, creating an Processor that 36 | * consumes a single element and publishes it. The Subscriber end 37 | * of the the Processor is created with `promiseToSubscriber`. The 38 | * Publisher end of the Processor is created with `futureToPublisher`. 39 | */ 40 | def promiseToProcessor[T](prom: Promise[T]): Processor[T, T] = { 41 | val subr = promiseToSubscriber(prom) 42 | val pubr = futureToPublisher(prom.future) 43 | val proc = join(subr, pubr) 44 | proc 45 | } 46 | 47 | /** 48 | * Adapt an Iteratee to a Subscriber and a result Iteratee. 49 | * 50 | * The Subscriber requests 51 | * elements one-by-one and feeds them to each Iteratee Cont step. When the 52 | * Iteratee step is Done or Error, the Subscription is cancelled. 53 | * 54 | * The result iteratee will be fulfilled when either the Iteratee 55 | * or the Subscriber subscription completes. The result Iteratee 56 | * may be in a Cont, Done or Error state, or a fourth state that 57 | * will yield errors or failed Futures when its methods are called. 58 | * 59 | * Calls to onNext send an Input.El to the iteratee and calls to 60 | * onComplete send an Input.EOF. If onError is called then the 61 | * iteratee enters an invalid state. 62 | * 63 | * If the Iteratee is in a Done or Error state then it will cancel 64 | * the Subscription as soon as possible, but it may still receive 65 | * calls to onError or onComplete. These calls are ignored. Be 66 | * careful because this means that it is possible for the Subscriber 67 | * to "consume" events, even if the Iteratee doesn't. 68 | */ 69 | def iterateeToSubscriber[T, U](iter: Iteratee[T, U]): (Subscriber[T], Iteratee[T, U]) = { 70 | val subr = new impl.IterateeSubscriber(iter) 71 | val resultIter = subr.result 72 | (subr, resultIter) 73 | } 74 | 75 | /** 76 | * Adapt a Subscriber to an Iteratee. 77 | * 78 | * The Iteratee will attempt to give the Subscriber as many elements as 79 | * demanded. When the subscriber cancels, the iteratee enters the done 80 | * state. 81 | * 82 | * Feeding EOF into the iteratee will cause the Subscriber to be fed a 83 | * completion event, and the iteratee will go into the done state. 84 | * 85 | * The iteratee will not enter the Cont state until demand is requested 86 | * by the subscriber. 87 | * 88 | * This Iteratee will never enter the error state, unless the subscriber 89 | * deviates from the reactive streams spec. 90 | */ 91 | def subscriberToIteratee[T](subscriber: Subscriber[T]): Iteratee[T, Unit] = { 92 | new impl.SubscriberIteratee[T](subscriber) 93 | } 94 | 95 | /** 96 | * Adapt an Iteratee to a Publisher, publishing its Done value. If 97 | * the iteratee is *not* Done then an exception is published. 98 | * 99 | * This method is similar in its effect to Iteratee.run, which 100 | * extracts the final value from an Iteratee. However, unlike 101 | * Iteratee.run, this method will not feed an EOF input to the 102 | * Iteratee. 103 | */ 104 | def iterateeDoneToPublisher[T, U](iter: Iteratee[T, U]): Publisher[U] = { 105 | iterateeFoldToPublisher[T, U, U](iter, { 106 | case Step.Done(x, _) => Future.successful(x) 107 | case notDone: Step[T, U] => Future.failed(new Exception(s"Can only get value from Done iteratee: $notDone")) 108 | })(Execution.trampoline) 109 | } 110 | 111 | /** 112 | * Fold an Iteratee and publish its result. This method is used 113 | * by iterateeDoneToPublisher to extract the value of a Done iteratee. 114 | */ 115 | private def iterateeFoldToPublisher[T, U, V](iter: Iteratee[T, U], f: Step[T, U] => Future[V])(implicit ec: ExecutionContext): Publisher[V] = { 116 | val fut: Future[V] = iter.fold(f)(ec.prepare) 117 | val pubr: Publisher[V] = futureToPublisher(fut) 118 | pubr 119 | } 120 | 121 | /** 122 | * Adapt an Iteratee to a Processor, which consumes input and then 123 | * yields the iteratee's Done value. It uses iterateeToSubscriber and 124 | * iterateeDoneToPublisher to create each end of the Processor. 125 | */ 126 | def iterateeToProcessor[T, U](iter: Iteratee[T, U]): Processor[T, U] = { 127 | val (subr, resultIter) = iterateeToSubscriber(iter) 128 | val pubr = iterateeDoneToPublisher(resultIter) 129 | val proc = join(subr, pubr) 130 | proc 131 | } 132 | 133 | /** 134 | * Adapt an Enumerator to a Publisher. Each Subscriber will be 135 | * adapted to an Iteratee and applied to the Enumerator. Input of 136 | * type Input.El will result in calls to onNext. 137 | * 138 | * Either onError or onComplete will always be invoked as the 139 | * last call to the subscriber, the former happening if the 140 | * enumerator fails with an error, the latter happening when 141 | * the first of either Input.EOF is fed, or the enumerator 142 | * completes. 143 | * 144 | * If emptyElement is None then Input of type Input.Empty will 145 | * be ignored. If it is set to Some(x) then it will call onNext 146 | * with the value x. 147 | */ 148 | def enumeratorToPublisher[T](enum: Enumerator[T], emptyElement: Option[T] = None): Publisher[T] = 149 | new impl.EnumeratorPublisher(enum, emptyElement) 150 | 151 | /** 152 | * Adapt a Publisher to an Enumerator. This is achieved by 153 | * adapting any Iteratees into Subscribers using the 154 | * iterateeToSubscriber method. 155 | */ 156 | def publisherToEnumerator[T](pubr: Publisher[T]): Enumerator[T] = 157 | new impl.PublisherEnumerator(pubr) 158 | 159 | /** 160 | * Adapt an Enumeratee to a Processor. 161 | */ 162 | def enumerateeToProcessor[A, B](enumeratee: Enumeratee[A, B]): Processor[A, B] = { 163 | val (iter, enum) = Concurrent.joined[A] 164 | val (subr, _) = iterateeToSubscriber(iter) 165 | val pubr = enumeratorToPublisher(enum &> enumeratee) 166 | new impl.SubscriberPublisherProcessor(subr, pubr) 167 | } 168 | 169 | /** 170 | * Adapt a Processor to an Enumeratee. 171 | */ 172 | def processorToEnumeratee[A, B](processor: Processor[A, B]): Enumeratee[A, B] = { 173 | val iter = subscriberToIteratee(processor) 174 | val enum = publisherToEnumerator(processor) 175 | new Enumeratee[A, B] { 176 | override def applyOn[U](inner: Iteratee[B, U]): Iteratee[A, Iteratee[B, U]] = { 177 | import play.api.libs.iteratee.Execution.Implicits.trampoline 178 | iter.map { _ => 179 | Iteratee.flatten(enum(inner)) 180 | } 181 | } 182 | } 183 | } 184 | 185 | /** 186 | * Join a Subscriber and Publisher together to make a Processor. 187 | * The Processor delegates its Subscriber methods to the Subscriber 188 | * and its Publisher methods to the Publisher. The Processor 189 | * otherwise does nothing. 190 | */ 191 | def join[T, U](subr: Subscriber[T], pubr: Publisher[U]): Processor[T, U] = 192 | new impl.SubscriberPublisherProcessor(subr, pubr) 193 | 194 | } 195 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/IterateeSubscriberSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.specs2.mutable.Specification 7 | import play.api.libs.iteratee._ 8 | import scala.concurrent.duration.{ FiniteDuration => ScalaFiniteDuration, SECONDS } 9 | import scala.concurrent.{ Await, Promise } 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | import scala.util.{ Failure, Success, Try } 12 | 13 | class IterateeSubscriberSpec extends Specification { 14 | 15 | import PublisherEvents._ 16 | case class ContInput(input: Input[_]) 17 | case class Result(result: Any) 18 | 19 | class TestEnv[T] extends EventRecorder() with PublisherEvents[T] { 20 | 21 | @volatile 22 | var nextIterateePromise = Promise[Iteratee[T, T]]() 23 | def nextIteratee = Iteratee.flatten(nextIterateePromise.future) 24 | 25 | // Initialize 26 | { 27 | val iter = nextIteratee 28 | val subr = new IterateeSubscriber(iter) 29 | subr.result.unflatten.onComplete { tryStep: Try[Step[T, T]] => 30 | record(Result(tryStep)) 31 | 32 | } 33 | publisher.subscribe(subr) 34 | } 35 | 36 | def contStep(): Unit = { 37 | val oldPromise = nextIterateePromise 38 | nextIterateePromise = Promise[Iteratee[T, T]]() 39 | oldPromise.success(Cont { input => 40 | record(ContInput(input)) 41 | nextIteratee 42 | }) 43 | } 44 | 45 | def doneStep(result: T, remaining: Input[T]): Unit = { 46 | nextIterateePromise.success(Done(result, remaining)) 47 | } 48 | 49 | def errorStep(msg: String, input: Input[T]): Unit = { 50 | nextIterateePromise.success(Error(msg, input)) 51 | } 52 | 53 | } 54 | 55 | "IterateeSubscriber" should { 56 | "consume 1 item" in { 57 | val enum = Enumerator(1, 2, 3) >>> Enumerator.eof 58 | val pubr = new EnumeratorPublisher(enum) 59 | val iter = Iteratee.getChunks[Int] 60 | val subr = new IterateeSubscriber(iter) 61 | pubr.subscribe(subr) 62 | Await.result(subr.result.unflatten, ScalaFiniteDuration(2, SECONDS)) must_== Done(List(1, 2, 3), Input.EOF) 63 | } 64 | 65 | "consume one element (on-subscribe/cont-step/on-next/cont-step/on-complete/done-step)" in { 66 | val testEnv = new TestEnv[Int] 67 | testEnv.onSubscribe() 68 | testEnv.isEmptyAfterDelay() must beTrue 69 | 70 | testEnv.contStep() 71 | testEnv.next must_== RequestMore(1) 72 | testEnv.isEmptyAfterDelay() must beTrue 73 | 74 | testEnv.onNext(1) 75 | testEnv.next must_== ContInput(Input.El(1)) 76 | testEnv.isEmptyAfterDelay() must beTrue 77 | 78 | testEnv.contStep() 79 | testEnv.next must_== RequestMore(1) 80 | testEnv.isEmptyAfterDelay() must beTrue 81 | 82 | testEnv.onComplete() 83 | testEnv.next must_== ContInput(Input.EOF) 84 | testEnv.isEmptyAfterDelay() must beTrue 85 | 86 | testEnv.doneStep(123, Input.Empty) 87 | testEnv.next must_== Result(Success(Step.Done(123, Input.Empty))) 88 | testEnv.isEmptyAfterDelay() must beTrue 89 | } 90 | 91 | "consume one element (cont-step/on-subscribe/on-next/cont-step/on-complete/done-step)" in { 92 | val testEnv = new TestEnv[Int] 93 | 94 | testEnv.contStep() 95 | testEnv.isEmptyAfterDelay() must beTrue 96 | 97 | testEnv.onSubscribe() 98 | testEnv.next must_== RequestMore(1) 99 | testEnv.isEmptyAfterDelay() must beTrue 100 | 101 | testEnv.onNext(1) 102 | testEnv.next must_== ContInput(Input.El(1)) 103 | testEnv.isEmptyAfterDelay() must beTrue 104 | 105 | testEnv.contStep() 106 | testEnv.next must_== RequestMore(1) 107 | testEnv.isEmptyAfterDelay() must beTrue 108 | 109 | testEnv.onComplete() 110 | testEnv.next must_== ContInput(Input.EOF) 111 | testEnv.isEmptyAfterDelay() must beTrue 112 | 113 | testEnv.doneStep(123, Input.Empty) 114 | testEnv.next must_== Result(Success(Step.Done(123, Input.Empty))) 115 | testEnv.isEmptyAfterDelay() must beTrue 116 | } 117 | 118 | "send EOF to cont when publisher completes immediately, moving to cont (cont-step/on-complete/cont-step)" in { 119 | val testEnv = new TestEnv[Int] 120 | 121 | testEnv.contStep() 122 | testEnv.isEmptyAfterDelay() must beTrue 123 | 124 | testEnv.onComplete() 125 | testEnv.next must_== ContInput(Input.EOF) 126 | testEnv.isEmptyAfterDelay() must beTrue 127 | 128 | testEnv.contStep() 129 | testEnv.next must beLike { case Result(Success(Step.Cont(_))) => ok } 130 | testEnv.isEmptyAfterDelay() must beTrue 131 | } 132 | 133 | "send EOF to cont when publisher completes immediately, moving to error (on-complete/cont-step/error-step)" in { 134 | val testEnv = new TestEnv[Int] 135 | 136 | testEnv.onComplete() 137 | testEnv.isEmptyAfterDelay() must beTrue 138 | 139 | testEnv.contStep() 140 | testEnv.next must_== ContInput(Input.EOF) 141 | testEnv.isEmptyAfterDelay() must beTrue 142 | 143 | testEnv.errorStep("!!!", Input.EOF) 144 | testEnv.next must_== Result(Success(Step.Error("!!!", Input.EOF))) 145 | testEnv.isEmptyAfterDelay() must beTrue 146 | } 147 | 148 | "fail when publisher errors immediately (on-error)" in { 149 | val testEnv = new TestEnv[Int] 150 | 151 | val t = new Exception("%@^%#!") 152 | testEnv.onError(t) 153 | testEnv.next must_== Result(Failure(t)) 154 | testEnv.isEmptyAfterDelay() must beTrue 155 | } 156 | 157 | "fail when publisher errors immediately (cont-step/on-error)" in { 158 | val testEnv = new TestEnv[Int] 159 | 160 | testEnv.contStep() 161 | testEnv.isEmptyAfterDelay() must beTrue 162 | 163 | val t = new Exception("%@^%#!") 164 | testEnv.onError(t) 165 | testEnv.next must_== Result(Failure(t)) 166 | testEnv.isEmptyAfterDelay() must beTrue 167 | } 168 | 169 | "finish when iteratee is done immediately, cancel subscription (done-step/on-subscribe)" in { 170 | val testEnv = new TestEnv[Int] 171 | 172 | testEnv.doneStep(333, Input.El(99)) 173 | testEnv.next must_== Result(Success(Done(333, Input.El(99)))) 174 | testEnv.isEmptyAfterDelay() must beTrue 175 | 176 | testEnv.onSubscribe() 177 | testEnv.next must_== Cancel 178 | testEnv.isEmptyAfterDelay() must beTrue 179 | } 180 | 181 | "finish when iteratee is done after subscribe, cancel subscription (on-subscribe/done-step)" in { 182 | val testEnv = new TestEnv[Int] 183 | 184 | testEnv.onSubscribe() 185 | testEnv.isEmptyAfterDelay() must beTrue 186 | 187 | testEnv.doneStep(333, Input.El(99)) 188 | testEnv.next must_== Cancel 189 | testEnv.next must_== Result(Success(Done(333, Input.El(99)))) 190 | testEnv.isEmptyAfterDelay() must beTrue 191 | } 192 | 193 | "finish when iteratee is done immediately, ignore complete (done-step/on-complete)" in { 194 | val testEnv = new TestEnv[Int] 195 | 196 | testEnv.doneStep(333, Input.El(99)) 197 | testEnv.next must_== Result(Success(Done(333, Input.El(99)))) 198 | testEnv.isEmptyAfterDelay() must beTrue 199 | 200 | testEnv.onComplete() 201 | testEnv.isEmptyAfterDelay() must beTrue 202 | } 203 | 204 | "finish when iteratee is done immediately, ignore error (done-step/on-error)" in { 205 | val testEnv = new TestEnv[Int] 206 | 207 | testEnv.doneStep(333, Input.El(99)) 208 | testEnv.next must_== Result(Success(Done(333, Input.El(99)))) 209 | testEnv.isEmptyAfterDelay() must beTrue 210 | 211 | testEnv.onError(new Exception("x")) 212 | testEnv.isEmptyAfterDelay() must beTrue 213 | } 214 | 215 | "finish when iteratee errors immediately, cancel subscription (done-step/on-subscribe)" in { 216 | val testEnv = new TestEnv[Int] 217 | 218 | testEnv.errorStep("iteratee error", Input.El(99)) 219 | testEnv.next must_== Result(Success(Error("iteratee error", Input.El(99)))) 220 | testEnv.isEmptyAfterDelay() must beTrue 221 | 222 | testEnv.onSubscribe() 223 | testEnv.next must_== Cancel 224 | testEnv.isEmptyAfterDelay() must beTrue 225 | } 226 | 227 | "finish when iteratee errors after subscribe, cancel subscription (on-subscribe/done-step)" in { 228 | val testEnv = new TestEnv[Int] 229 | 230 | testEnv.onSubscribe() 231 | testEnv.isEmptyAfterDelay() must beTrue 232 | 233 | testEnv.errorStep("iteratee error", Input.El(99)) 234 | testEnv.next must_== Cancel 235 | testEnv.next must_== Result(Success(Error("iteratee error", Input.El(99)))) 236 | testEnv.isEmptyAfterDelay() must beTrue 237 | } 238 | 239 | } 240 | 241 | } 242 | -------------------------------------------------------------------------------- /streams/src/test/scala/play/api/libs/iteratee/streams/impl/EnumeratorPublisherSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import org.specs2.mutable.Specification 8 | import play.api.libs.iteratee.{ Concurrent, Enumerator, Input } 9 | import scala.concurrent.{ Await, Future, Promise } 10 | import scala.concurrent.duration._ 11 | 12 | import scala.concurrent.ExecutionContext.Implicits.global 13 | 14 | class EnumeratorPublisherSpec extends Specification { 15 | 16 | case object OnSubscribe 17 | case class OnError(t: Throwable) 18 | case class OnNext(element: Any) 19 | case object OnComplete 20 | case class RequestMore(elementCount: Int) 21 | case object Cancel 22 | case object GetSubscription 23 | 24 | class TestEnv[T] extends EventRecorder() { 25 | 26 | object subscriber extends Subscriber[T] { 27 | val subscription = Promise[Subscription]() 28 | override def onSubscribe(s: Subscription) = { 29 | record(OnSubscribe) 30 | subscription.success(s) 31 | } 32 | override def onError(t: Throwable) = record(OnError(t)) 33 | override def onNext(element: T) = record(OnNext(element)) 34 | override def onComplete() = record(OnComplete) 35 | } 36 | 37 | def forSubscription(f: Subscription => Any): Future[Unit] = { 38 | subscriber.subscription.future.map(f).map(_ => ()) 39 | } 40 | def request(elementCount: Int): Future[Unit] = { 41 | forSubscription { s => 42 | record(RequestMore(elementCount)) 43 | s.request(elementCount) 44 | } 45 | } 46 | def cancel(): Future[Unit] = { 47 | forSubscription { s => 48 | record(Cancel) 49 | s.cancel() 50 | } 51 | } 52 | 53 | } 54 | 55 | "EnumeratorPublisher" should { 56 | "enumerate one item" in { 57 | val testEnv = new TestEnv[Int] 58 | val enum = Enumerator(1) >>> Enumerator.eof 59 | val pubr = new EnumeratorPublisher(enum) 60 | pubr.subscribe(testEnv.subscriber) 61 | testEnv.next must_== OnSubscribe 62 | testEnv.request(1) 63 | testEnv.next must_== RequestMore(1) 64 | testEnv.next must_== OnNext(1) 65 | testEnv.request(1) 66 | testEnv.next must_== RequestMore(1) 67 | testEnv.next must_== OnComplete 68 | testEnv.isEmptyAfterDelay() must beTrue 69 | } 70 | "enumerate three items, with batched requests" in { 71 | val testEnv = new TestEnv[Int] 72 | val enum = Enumerator(1, 2, 3) >>> Enumerator.eof 73 | val pubr = new EnumeratorPublisher(enum) 74 | pubr.subscribe(testEnv.subscriber) 75 | testEnv.next must_== OnSubscribe 76 | testEnv.request(2) 77 | testEnv.next must_== RequestMore(2) 78 | testEnv.next must_== OnNext(1) 79 | testEnv.next must_== OnNext(2) 80 | testEnv.request(2) 81 | testEnv.next must_== RequestMore(2) 82 | testEnv.next must_== OnNext(3) 83 | testEnv.next must_== OnComplete 84 | testEnv.isEmptyAfterDelay() must beTrue 85 | } 86 | "be done enumerating after EOF" in { 87 | val testEnv = new TestEnv[Int] 88 | var enumDone = Promise[Boolean]() 89 | val enum = (Enumerator(1, 2, 3) >>> Enumerator.eof).onDoneEnumerating { 90 | enumDone.success(true) 91 | } 92 | val pubr = new EnumeratorPublisher(enum) 93 | pubr.subscribe(testEnv.subscriber) 94 | testEnv.next must_== OnSubscribe 95 | testEnv.request(4) 96 | testEnv.next must_== RequestMore(4) 97 | testEnv.next must_== OnNext(1) 98 | testEnv.next must_== OnNext(2) 99 | testEnv.next must_== OnNext(3) 100 | testEnv.next must_== OnComplete 101 | testEnv.isEmptyAfterDelay() must beTrue 102 | Await.result(enumDone.future, Duration(5, SECONDS)) must beTrue 103 | } 104 | 105 | "complete the subscriber when done enumerating without eof" in { 106 | val testEnv = new TestEnv[Int] 107 | val enum = Enumerator(1, 2, 3) 108 | val pubr = new EnumeratorPublisher(enum) 109 | pubr.subscribe(testEnv.subscriber) 110 | testEnv.next must_== OnSubscribe 111 | testEnv.request(4) 112 | testEnv.next must_== RequestMore(4) 113 | testEnv.next must_== OnNext(1) 114 | testEnv.next must_== OnNext(2) 115 | testEnv.next must_== OnNext(3) 116 | testEnv.next must_== OnComplete 117 | testEnv.isEmptyAfterDelay() must beTrue 118 | } 119 | 120 | "be done enumerating after being cancelled" in { 121 | val testEnv = new TestEnv[Int] 122 | val enumDone = Promise[Boolean]() 123 | val (broadcastEnum, channel) = Concurrent.broadcast[Int] 124 | val enum = broadcastEnum.onDoneEnumerating { 125 | enumDone.success(true) 126 | } 127 | val pubr = new EnumeratorPublisher(enum) 128 | pubr.subscribe(testEnv.subscriber) 129 | testEnv.next must_== OnSubscribe 130 | testEnv.request(4) 131 | testEnv.next must_== RequestMore(4) 132 | testEnv.isEmptyAfterDelay() must beTrue 133 | testEnv.cancel() 134 | testEnv.next must_== Cancel 135 | // Element push occurs after cancel, so will not generate an event. 136 | // However it is necessary to have an event so that the publisher's 137 | // Cont is satisfied. We want to advance the iteratee to pick up the 138 | // Done iteratee caused by the cancel. 139 | try { 140 | channel.push(0) 141 | Await.result(enumDone.future, Duration(5, SECONDS)) must beTrue 142 | } catch { 143 | case t: Throwable => 144 | // If it didn't work the first time, try again, since cancel only guarantees that the publisher will 145 | // eventually finish 146 | channel.push(0) 147 | Await.result(enumDone.future, Duration(5, SECONDS)) must beTrue 148 | } 149 | } 150 | "enumerate eof only" in { 151 | val testEnv = new TestEnv[Int] 152 | val enum: Enumerator[Int] = Enumerator.eof 153 | val pubr = new EnumeratorPublisher(enum) 154 | pubr.subscribe(testEnv.subscriber) 155 | testEnv.next must_== OnSubscribe 156 | testEnv.request(1) 157 | testEnv.next must_== RequestMore(1) 158 | testEnv.next must_== OnComplete 159 | testEnv.isEmptyAfterDelay() must beTrue 160 | } 161 | "by default, enumerate nothing for empty" in { 162 | val testEnv = new TestEnv[Int] 163 | val enum: Enumerator[Int] = Enumerator.enumInput(Input.Empty) >>> Enumerator.eof 164 | val pubr = new EnumeratorPublisher(enum) 165 | pubr.subscribe(testEnv.subscriber) 166 | testEnv.next must_== OnSubscribe 167 | testEnv.request(1) 168 | testEnv.next must_== RequestMore(1) 169 | testEnv.next must_== OnComplete 170 | testEnv.isEmptyAfterDelay() must beTrue 171 | } 172 | "be able to enumerate something for empty" in { 173 | val testEnv = new TestEnv[Int] 174 | val enum: Enumerator[Int] = Enumerator.enumInput(Input.Empty) >>> Enumerator.eof 175 | val pubr = new EnumeratorPublisher(enum, emptyElement = Some(-1)) 176 | pubr.subscribe(testEnv.subscriber) 177 | testEnv.next must_== OnSubscribe 178 | testEnv.request(1) 179 | testEnv.next must_== RequestMore(1) 180 | testEnv.next must_== OnNext(-1) 181 | testEnv.request(1) 182 | testEnv.next must_== RequestMore(1) 183 | testEnv.next must_== OnComplete 184 | testEnv.isEmptyAfterDelay() must beTrue 185 | } 186 | "handle errors when enumerating" in { 187 | val testEnv = new TestEnv[Int] 188 | val lotsOfItems = 0 until 25 189 | val exception = new Exception("x") 190 | val enum = Enumerator.flatten(Future.failed(exception)) 191 | val pubr = new EnumeratorPublisher[Nothing](enum) 192 | pubr.subscribe(testEnv.subscriber) 193 | testEnv.next must_== OnSubscribe 194 | testEnv.request(1) 195 | testEnv.next must_== RequestMore(1) 196 | testEnv.next must beLike { 197 | case OnError(e) => e.getMessage must_== exception.getMessage 198 | } 199 | testEnv.isEmptyAfterDelay() must beTrue 200 | } 201 | "enumerate 25 items" in { 202 | val testEnv = new TestEnv[Int] 203 | val lotsOfItems = 0 until 25 204 | val enum = Enumerator(lotsOfItems: _*) >>> Enumerator.eof 205 | val pubr = new EnumeratorPublisher(enum) 206 | pubr.subscribe(testEnv.subscriber) 207 | testEnv.next must_== OnSubscribe 208 | for (i <- lotsOfItems) { 209 | testEnv.request(1) 210 | testEnv.next must_== RequestMore(1) 211 | testEnv.next must_== OnNext(i) 212 | } 213 | testEnv.request(1) 214 | testEnv.next must_== RequestMore(1) 215 | testEnv.next must_== OnComplete 216 | testEnv.isEmptyAfterDelay() must beTrue 217 | } 218 | } 219 | 220 | } 221 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/CharEncodingSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import org.specs2.mutable._ 7 | import scala.concurrent.Await 8 | import scala.concurrent.duration.Duration 9 | 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | 12 | object CharEncodingSpec extends Specification { 13 | 14 | "CharEncodingSpec.decode()" should { 15 | 16 | "decode US-ASCII" in { 17 | val input = Seq( 18 | Array[Byte](0x48, 0x65, 0x6c), 19 | Array[Byte](0x6c, 0x6f, 0x20, 0x57), 20 | Array[Byte](0x6f, 0x72, 0x6c, 0x64) 21 | ) 22 | val result = Enumerator(input: _*) &> CharEncoding.decode("US-ASCII") |>>> Iteratee.consume[String]() 23 | Await.result(result, Duration.Inf) must be equalTo new String(input.flatten.toArray, "US-ASCII") 24 | } 25 | 26 | "decode UTF-8" in { 27 | val input = Seq( 28 | Array[Byte](0x48, 0x65, 0x6c), 29 | Array[Byte](0x6c, 0x6f, 0x20, 0x57), 30 | Array[Byte](0x6f, 0x72, 0x6c, 0x64) 31 | ) 32 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-8") |>>> Iteratee.consume[String]() 33 | Await.result(result, Duration.Inf) must be equalTo new String(input.flatten.toArray, "UTF-8") 34 | } 35 | 36 | "decode UTF-8 with split characters" in { 37 | val input = Seq( 38 | Array[Byte](0xe2.toByte), 39 | Array[Byte](0x82.toByte), 40 | Array[Byte](0xac.toByte), 41 | Array[Byte](0xf0.toByte), 42 | Array[Byte](0x9f.toByte), 43 | Array[Byte](0x82.toByte), 44 | Array[Byte](0xA5.toByte) 45 | ) 46 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-8") |>>> Iteratee.consume[String]() 47 | Await.result(result, Duration.Inf) must be equalTo "\u20ac\ud83c\udca5" 48 | } 49 | 50 | "decode UTF-16" in { 51 | val input = Seq( 52 | Array[Byte](0x00, 0x48, 0x00, 0x65, 0x00, 0x6c), 53 | Array[Byte](0x00, 0x6c, 0x00, 0x6f, 0x00, 0x20, 0x00, 0x57), 54 | Array[Byte](0x00, 0x6f, 0x00, 0x72, 0x00, 0x6c, 0x00, 0x64) 55 | ) 56 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-16") |>>> Iteratee.consume[String]() 57 | Await.result(result, Duration.Inf) must be equalTo "Hello World" 58 | } 59 | 60 | "decode UTF-16 with split characters" in { 61 | val input = Seq( 62 | Array[Byte](0x20), 63 | Array[Byte](0xac.toByte), 64 | Array[Byte](0xd8.toByte), 65 | Array[Byte](0x3c), 66 | Array[Byte](0xdc.toByte), 67 | Array[Byte](0xa5.toByte) 68 | ) 69 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-16") |>>> Iteratee.consume[String]() 70 | Await.result(result, Duration.Inf) must be equalTo "\u20ac\ud83c\udca5" 71 | } 72 | 73 | "decode UTF-32" in { 74 | val input = Seq( 75 | Array[Byte](0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x6c), 76 | Array[Byte](0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x57), 77 | Array[Byte](0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x6c, 0x00, 0x00, 0x00, 0x64) 78 | ) 79 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-32") |>>> Iteratee.consume[String]() 80 | Await.result(result, Duration.Inf) must be equalTo "Hello World" 81 | } 82 | 83 | "decode UTF-32 with split characters" in { 84 | val input = Seq( 85 | Array[Byte](0x00), 86 | Array[Byte](0x00), 87 | Array[Byte](0x20), 88 | Array[Byte](0xac.toByte), 89 | Array[Byte](0x00), 90 | Array[Byte](0x01), 91 | Array[Byte](0xf0.toByte), 92 | Array[Byte](0xa5.toByte) 93 | ) 94 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-32") |>>> Iteratee.consume[String]() 95 | Await.result(result, Duration.Inf) must be equalTo "\u20ac\ud83c\udca5" 96 | } 97 | 98 | "fail on invalid ASCII" in { 99 | val input = Seq( 100 | Array[Byte](0x80.toByte) 101 | ) 102 | val result = Enumerator(input: _*) &> CharEncoding.decode("US-ASCII") |>>> Iteratee.skipToEof 103 | val status = result.map { _ => "success" }.recover { case e => "failure" } 104 | Await.result(status, Duration.Inf) must be equalTo "failure" 105 | } 106 | 107 | "fail on invalid UTF-8" in { 108 | val input = Seq( 109 | Array[Byte](0xe2.toByte, 0xe2.toByte, 0xe2.toByte) 110 | ) 111 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-8") |>>> Iteratee.skipToEof 112 | val status = result.map { _ => "success" }.recover { case e => "failure" } 113 | Await.result(status, Duration.Inf) must be equalTo "failure" 114 | } 115 | 116 | "fail on invalid UTF-16" in { 117 | val input = Seq( 118 | Array[Byte](0xd8.toByte, 0x00), 119 | Array[Byte](0xd8.toByte, 0x00) 120 | ) 121 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-16") |>>> Iteratee.skipToEof 122 | val status = result.map { _ => "success" }.recover { case e => "failure" } 123 | Await.result(status, Duration.Inf) must be equalTo "failure" 124 | } 125 | 126 | "fail on invalid UTF-32" in { 127 | val input = Seq( 128 | Array[Byte](0x00) 129 | ) 130 | val result = Enumerator(input: _*) &> CharEncoding.decode("UTF-32") |>>> Iteratee.skipToEof 131 | val status = result.map { _ => "success" }.recover { case e => "failure" } 132 | Await.result(status, Duration.Inf) must be equalTo "failure" 133 | } 134 | 135 | } 136 | 137 | "CharEncodingSpec.encode()" should { 138 | 139 | "encode US-ASCII" in { 140 | val input = Seq( 141 | "Hel", 142 | "lo W", 143 | "orld" 144 | ) 145 | val result = Enumerator(input: _*) &> CharEncoding.encode("US-ASCII") |>>> Iteratee.consume[Array[Byte]]() 146 | Await.result(result, Duration.Inf) must be equalTo input.mkString.getBytes("US-ASCII") 147 | } 148 | 149 | "encode UTF-8" in { 150 | val input = Seq( 151 | "Hel", 152 | "lo W", 153 | "orld" 154 | ) 155 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-8") |>>> Iteratee.consume[Array[Byte]]() 156 | Await.result(result, Duration.Inf) must be equalTo input.mkString.getBytes("UTF-8") 157 | } 158 | 159 | "encode UTF-8 with split characters" in { 160 | val input = Seq( 161 | "\u20ac", 162 | "\ud83c", 163 | "\udca5" 164 | ) 165 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-8") |>>> Iteratee.consume[Array[Byte]]() 166 | Await.result(result, Duration.Inf) must be equalTo input.mkString.getBytes("UTF-8") 167 | } 168 | 169 | "encode UTF-16" in { 170 | val input = Seq( 171 | "Hel", 172 | "lo W", 173 | "orld" 174 | ) 175 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-16") |>>> Iteratee.consume[Array[Byte]]() 176 | Await.result(result, Duration.Inf) must be equalTo input.mkString.getBytes("UTF-16BE") 177 | } 178 | 179 | "encode UTF-16 with split characters" in { 180 | val input = Seq( 181 | "\u20ac", 182 | "\ud83c", 183 | "\udca5" 184 | ) 185 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-16") |>>> Iteratee.consume[Array[Byte]]() 186 | Await.result(result, Duration.Inf) must be equalTo input.mkString.getBytes("UTF-16BE") 187 | } 188 | 189 | "encode UTF-32" in { 190 | val input = Seq( 191 | "Hel", 192 | "lo W", 193 | "orld" 194 | ) 195 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-32") |>>> Iteratee.consume[Array[Byte]]() 196 | Await.result(result, Duration.Inf) must be equalTo input.mkString.getBytes("UTF-32") 197 | } 198 | 199 | "encode UTF-32 with split characters" in { 200 | val input = Seq( 201 | "\u20ac", 202 | "\ud83c", 203 | "\udca5" 204 | ) 205 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-32") |>>> Iteratee.consume[Array[Byte]]() 206 | Await.result(result, Duration.Inf) must be equalTo input.mkString.getBytes("UTF-32") 207 | } 208 | 209 | "fail on unmappable ASCII" in { 210 | val input = Seq("\u20ac") 211 | val result = Enumerator(input: _*) &> CharEncoding.encode("US-ASCII") |>>> Iteratee.skipToEof 212 | val status = result.map { _ => "success" }.recover { case e => "failure" } 213 | Await.result(status, Duration.Inf) must be equalTo "failure" 214 | } 215 | 216 | "fail on invalid Unicode with UTF-8" in { 217 | val input = Seq( 218 | "\ud83c" 219 | ) 220 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-8") |>>> Iteratee.skipToEof 221 | val status = result.map { _ => "success" }.recover { case e => "failure" } 222 | Await.result(status, Duration.Inf) must be equalTo "failure" 223 | } 224 | 225 | "fail on invalid Unicode with UTF-8" in { 226 | val input = Seq( 227 | "\ud83c" 228 | ) 229 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-16") |>>> Iteratee.skipToEof 230 | val status = result.map { _ => "success" }.recover { case e => "failure" } 231 | Await.result(status, Duration.Inf) must be equalTo "failure" 232 | } 233 | 234 | "fail on invalid Unicode with UTF-32" in { 235 | val input = Seq( 236 | "\ud83c" 237 | ) 238 | val result = Enumerator(input: _*) &> CharEncoding.encode("UTF-32") |>>> Iteratee.skipToEof 239 | val status = result.map { _ => "success" }.recover { case e => "failure" } 240 | Await.result(status, Duration.Inf) must be equalTo "failure" 241 | } 242 | 243 | } 244 | 245 | } 246 | -------------------------------------------------------------------------------- /streams/src/main/scala/play/api/libs/iteratee/streams/impl/IterateeSubscriber.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee.streams.impl 5 | 6 | import org.reactivestreams._ 7 | import play.api.libs.iteratee.concurrent.StateMachine 8 | import play.api.libs.iteratee._ 9 | 10 | import scala.concurrent.Promise 11 | 12 | private[streams] object IterateeSubscriber { 13 | /** 14 | * Internal state of the Subscriber. 15 | * 16 | * @tparam T type of elements. 17 | * @tparam R The result type of the Iteratee. 18 | */ 19 | sealed trait State[T, R] 20 | /** 21 | * A Subscriber that hasn't had onSubscribe called on it yet, and whose 22 | * Iteratee hasn't resolved to a Step yet. This is the initial state of the 23 | * Subscriber. 24 | * 25 | * @param result A Promise of the eventual result of this Subscriber. 26 | */ 27 | case class NotSubscribedNoStep[T, R](result: Promise[Iteratee[T, R]]) extends State[T, R] 28 | /** 29 | * A Subscriber that has had onSubscribe called on it but 30 | * doesn't yet know the current Step of its Iteratee. 31 | * 32 | * @param subs The Subscription the Subscriber is subscribed to. 33 | * @param result A Promise of the eventual result of this Subscriber. 34 | */ 35 | case class SubscribedNoStep[T, R](subs: Subscription, result: Promise[Iteratee[T, R]]) extends State[T, R] 36 | /** 37 | * A Subscriber that hasn't had onSubscribe called on it yet, but whose 38 | * Iteratee is known to have a Step of Cont. 39 | * 40 | * @param cont The current Step of the Iteratee. 41 | * @param result A Promise of the eventual result of this Subscriber. 42 | */ 43 | case class NotSubscribedWithCont[T, R](cont: Step.Cont[T, R], result: Promise[Iteratee[T, R]]) extends State[T, R] 44 | /** 45 | * A Subscriber that has had onSubscribe called, has a current Iteratee with 46 | * a Step of Cont and is currently waiting for an element that it has 47 | * requested from the Subscription. 48 | * 49 | * @param subs The Subscription the Subscriber is subscribed to. 50 | * @param cont The current Step of the Iteratee. 51 | * @param result A Promise of the eventual result of this Subscriber. 52 | */ 53 | case class SubscribedWithCont[T, R](subs: Subscription, cont: Step.Cont[T, R], result: Promise[Iteratee[T, R]]) extends State[T, R] 54 | /** 55 | * A Subscriber that has been completed with onComplete but whose Iteratee 56 | * hasn't resolved ot a Step yet. If the Iteratee resolves to Cont then it 57 | * Input.EOF will be fed to it. 58 | * 59 | * @param result A Promise of the eventual result of this Subscriber. 60 | */ 61 | case class CompletedNoStep[T, R](result: Promise[Iteratee[T, R]]) extends State[T, R] 62 | /** 63 | * A Subscriber that is finished. We don't track the precise reason for 64 | * being finished. 65 | * 66 | * @param resultIteratee The result of this Subscriber. 67 | */ 68 | case class Finished[T, R](resultIteratee: Iteratee[T, R]) extends State[T, R] 69 | } 70 | 71 | import streams.impl.IterateeSubscriber._ 72 | 73 | private[streams] class IterateeSubscriber[T, R, S](iter0: Iteratee[T, R]) 74 | extends StateMachine[State[T, R]](initialState = NotSubscribedNoStep(Promise[Iteratee[T, R]]())) with Subscriber[T] { 75 | 76 | // We immediately fold on the iteratee 77 | getNextStepFromIteratee(iter0) 78 | 79 | /** 80 | * The final result of this Iteratee, either a Done or Error Iteratee, 81 | * or the last Iteratee when the Subscription is completed. 82 | */ 83 | def result: Iteratee[T, R] = state match { 84 | case NotSubscribedNoStep(result) => 85 | promiseToIteratee(result) 86 | case SubscribedNoStep(subs, result) => 87 | promiseToIteratee(result) 88 | case NotSubscribedWithCont(cont, result) => 89 | promiseToIteratee(result) 90 | case SubscribedWithCont(subs, cont, result) => 91 | promiseToIteratee(result) 92 | case CompletedNoStep(result) => 93 | promiseToIteratee(result) 94 | case Finished(resultIteratee) => 95 | resultIteratee 96 | } 97 | 98 | // Streams methods 99 | 100 | override def onSubscribe(subs: Subscription): Unit = exclusive { 101 | case NotSubscribedNoStep(result) => 102 | state = SubscribedNoStep(subs, result) 103 | case SubscribedNoStep(subs, result) => 104 | throw new IllegalStateException("Can't subscribe twice") 105 | case NotSubscribedWithCont(cont, result) => 106 | subs.request(1) 107 | state = SubscribedWithCont(subs, cont, result) 108 | case SubscribedWithCont(subs, cont, result) => 109 | throw new IllegalStateException("Can't subscribe twice") 110 | case CompletedNoStep(result) => 111 | throw new IllegalStateException("Can't subscribe once completed") 112 | case Finished(resultIteratee) => 113 | subs.cancel() 114 | } 115 | 116 | override def onComplete(): Unit = exclusive { 117 | case NotSubscribedNoStep(result) => 118 | state = CompletedNoStep(result) 119 | case SubscribedNoStep(subs, result) => 120 | state = CompletedNoStep(result) 121 | case NotSubscribedWithCont(cont, result) => 122 | finishWithCompletedCont(cont, result) 123 | case SubscribedWithCont(subs, cont, result) => 124 | finishWithCompletedCont(cont, result) 125 | case CompletedNoStep(result) => 126 | throw new IllegalStateException("Can't complete twice") 127 | case Finished(resultIteratee) => 128 | () 129 | } 130 | 131 | override def onError(cause: Throwable): Unit = exclusive { 132 | case NotSubscribedNoStep(result) => 133 | finishWithError(cause, result) 134 | case SubscribedNoStep(subs, result) => 135 | finishWithError(cause, result) 136 | case NotSubscribedWithCont(cont, result) => 137 | finishWithError(cause, result) 138 | case SubscribedWithCont(subs, cont, result) => 139 | finishWithError(cause, result) 140 | case CompletedNoStep(result) => 141 | throw new IllegalStateException("Can't receive error once completed") 142 | case Finished(resultIteratee) => 143 | () 144 | } 145 | 146 | override def onNext(element: T): Unit = exclusive { 147 | case NotSubscribedNoStep(result) => 148 | throw new IllegalStateException("Got next element before subscribed") 149 | case SubscribedNoStep(subs, result) => 150 | throw new IllegalStateException("Got next element before requested") 151 | case NotSubscribedWithCont(cont, result) => 152 | throw new IllegalStateException("Got next element before subscribed") 153 | case SubscribedWithCont(subs, cont, result) => 154 | continueWithNext(subs, cont, element, result) 155 | case CompletedNoStep(result) => 156 | throw new IllegalStateException("Can't receive error once completed") 157 | case Finished(resultIteratee) => 158 | () 159 | } 160 | 161 | private def continueWithNext(subs: Subscription, cont: Step.Cont[T, R], element: T, result: Promise[Iteratee[T, R]]): Unit = { 162 | val nextIteratee = cont.k(Input.El(element)) 163 | getNextStepFromIteratee(nextIteratee) 164 | state = SubscribedNoStep(subs, result) 165 | } 166 | 167 | /** 168 | * Called when the iteratee folds to a Cont step. We may want to feed 169 | * an Input to the Iteratee. 170 | */ 171 | private def onContStep(cont: Step.Cont[T, R]): Unit = { 172 | exclusive { 173 | case NotSubscribedNoStep(result) => 174 | state = NotSubscribedWithCont(cont, result) 175 | case SubscribedNoStep(subs, result) => 176 | subs.request(1) 177 | state = SubscribedWithCont(subs, cont, result) 178 | case NotSubscribedWithCont(cont, result) => 179 | throw new IllegalStateException("Can't get cont twice") 180 | case SubscribedWithCont(subs, cont, result) => 181 | throw new IllegalStateException("Can't get cont twice") 182 | case CompletedNoStep(result) => 183 | finishWithCompletedCont(cont, result) 184 | case Finished(resultIteratee) => 185 | () 186 | } 187 | } 188 | 189 | /** 190 | * Called when the iteratee folds to a Done or Error step. We may want to 191 | * cancel our Subscription. 192 | */ 193 | private def onDoneOrErrorStep(doneOrError: Step[T, R]): Unit = exclusive { 194 | case NotSubscribedNoStep(result) => 195 | finishWithDoneOrErrorStep(doneOrError, result) 196 | case SubscribedNoStep(subs, result) => 197 | subs.cancel() 198 | finishWithDoneOrErrorStep(doneOrError, result) 199 | case NotSubscribedWithCont(cont, result) => 200 | throw new IllegalStateException("Can't get done or error while has cont") 201 | case SubscribedWithCont(subs, cont, result) => 202 | throw new IllegalStateException("Can't get done or error while has cont") 203 | case CompletedNoStep(result) => 204 | finishWithDoneOrErrorStep(doneOrError, result) 205 | case Finished(resultIteratee) => 206 | () 207 | } 208 | 209 | /** 210 | * Folds an Iteratee to get its Step. The Step is used to choose a method 211 | * to call. 212 | */ 213 | private def getNextStepFromIteratee(iter: Iteratee[T, R]): Unit = { 214 | iter.pureFold { 215 | case c @ Step.Cont(_) => onContStep(c) 216 | case d @ Step.Done(_, _) => onDoneOrErrorStep(d) 217 | case e @ Step.Error(_, _) => onDoneOrErrorStep(e) 218 | }(Execution.trampoline) 219 | } 220 | 221 | /** Flattens a Promise[Iteratee] to an Iteratee. */ 222 | private def promiseToIteratee(result: Promise[Iteratee[T, R]]) = Iteratee.flatten(result.future) 223 | 224 | /** 225 | * Finishes the Subscription when the Step is Cont and onComplete 226 | * has been called. This is done by feeding EOF to the Cont Iteratee. 227 | */ 228 | private def finishWithCompletedCont(cont: Step.Cont[T, R], result: Promise[Iteratee[T, R]]): Unit = { 229 | val nextIteratee = cont.k(Input.EOF) 230 | result.success(nextIteratee) 231 | state = Finished(nextIteratee) 232 | } 233 | 234 | /** 235 | * Finishes the Subscription when onError has been called. This is done by 236 | * setting the Iteratee Future to an failed state. 237 | */ 238 | private def finishWithError(cause: Throwable, result: Promise[Iteratee[T, R]]): Unit = { 239 | result.failure(cause) 240 | state = Finished(promiseToIteratee(result)) 241 | } 242 | 243 | /** 244 | * Finishes the Subscription when the Step is Done or Error. This is done by 245 | * setting the result to the Step's iteratee. 246 | */ 247 | private def finishWithDoneOrErrorStep(step: Step[T, R], result: Promise[Iteratee[T, R]]): Unit = { 248 | val nextIteratee = step.it 249 | result.success(nextIteratee) 250 | state = Finished(nextIteratee) 251 | } 252 | 253 | } 254 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/EnumerateesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import play.api.libs.iteratee.Execution.Implicits.{ defaultExecutionContext => dec } 7 | import scala.concurrent._ 8 | import scala.concurrent.duration.Duration 9 | 10 | import org.specs2.mutable._ 11 | 12 | object EnumerateesSpec extends Specification 13 | with IterateeSpecification with ExecutionSpecification { 14 | 15 | "Enumeratee.zip" should { 16 | 17 | "combine the final results into a pair" in { 18 | Await.result(Enumeratee.zip(Done[Int, Int](2), Done[Int, Int](3)).unflatten, Duration.Inf) must equalTo(Step.Done((2, 3), Input.Empty)) 19 | } 20 | 21 | } 22 | 23 | "Enumeratee.zipWith" should { 24 | 25 | "combine the final results" in { 26 | mustExecute(1) { zipEC => 27 | Await.result(Enumeratee.zipWith(Done[Int, Int](2), Done[Int, Int](3))(_ * _)(zipEC).unflatten, Duration.Inf) must equalTo(Step.Done(6, Input.Empty)) 28 | } 29 | } 30 | 31 | } 32 | 33 | "Enumeratee.mapInput" should { 34 | 35 | "transform each input" in { 36 | mustExecute(2) { mapEC => 37 | mustTransformTo(1, 2)(2, 4)(Enumeratee.mapInput[Int](_.map(_ * 2))(mapEC)) 38 | } 39 | } 40 | 41 | } 42 | 43 | "Enumeratee.mapConcatInput" should { 44 | 45 | "transform each input element into a sequence of inputs" in { 46 | mustExecute(2) { mapEC => 47 | mustTransformTo(1, 2)(1, 1, 2, 2)(Enumeratee.mapConcatInput[Int](x => List(Input.El(x), Input.Empty, Input.El(x)))(mapEC)) 48 | } 49 | } 50 | 51 | } 52 | 53 | "Enumeratee.mapConcat" should { 54 | 55 | "transform each input element into a sequence of input elements" in { 56 | mustExecute(2) { mapEC => 57 | mustTransformTo(1, 2)(1, 1, 2, 2)(Enumeratee.mapConcat[Int](x => List(x, x))(mapEC)) 58 | } 59 | } 60 | 61 | } 62 | 63 | "Enumeratee.mapFlatten" should { 64 | 65 | "transform each input element into the output of an enumerator" in { 66 | mustExecute(2) { mapFlattenEC => 67 | mustTransformTo(1, 2)(1, 1, 2, 2)(Enumeratee.mapFlatten[Int](x => Enumerator(x, x))(mapFlattenEC)) 68 | } 69 | } 70 | 71 | } 72 | 73 | "Enumeratee.mapInputFlatten" should { 74 | 75 | "transform each input" in { 76 | mustExecute(2) { mapEC => 77 | val eee = Enumeratee.mapInputFlatten[Int][Int] { 78 | case Input.El(x) => Enumerator(x * 2) 79 | case Input.Empty => Enumerator.empty 80 | case Input.EOF => Enumerator.empty 81 | }(mapEC) 82 | mustTransformTo(1, 2)(2, 4)(eee compose Enumeratee.take(2)) 83 | } 84 | } 85 | 86 | } 87 | 88 | "Enumeratee.mapInputM" should { 89 | 90 | "transform each input" in { 91 | mustExecute(2) { mapEC => 92 | mustTransformTo(1, 2)(2, 4)(Enumeratee.mapInputM[Int]((i: Input[Int]) => Future.successful(i.map(_ * 2)))(mapEC)) 93 | } 94 | } 95 | 96 | } 97 | 98 | "Enumeratee.mapM" should { 99 | 100 | "transform each input element" in { 101 | mustExecute(2) { mapEC => 102 | mustTransformTo(1, 2)(2, 4)(Enumeratee.mapM[Int]((x: Int) => Future.successful(x * 2))(mapEC)) 103 | } 104 | } 105 | 106 | } 107 | 108 | "Enumeratee.drop" should { 109 | 110 | "ignore 3 chunkes when applied with 3" in { 111 | 112 | val drop3AndConsume = Enumeratee.drop[String](3) &>> Iteratee.consume[String]() 113 | val enumerator = Enumerator(Range(1, 20).map(_.toString): _*) 114 | Await.result(enumerator |>>> drop3AndConsume, Duration.Inf) must equalTo(Range(4, 20).map(_.toString).mkString) 115 | 116 | } 117 | 118 | } 119 | 120 | "Enumeratee.dropWhile" should { 121 | 122 | "ignore chunks while predicate is valid" in { 123 | mustExecute(4) { dropWhileEC => 124 | val drop3AndConsume = Enumeratee.dropWhile[String](_ != "4")(dropWhileEC) &>> Iteratee.consume[String]() 125 | val enumerator = Enumerator(Range(1, 20).map(_.toString): _*) 126 | Await.result(enumerator |>>> drop3AndConsume, Duration.Inf) must equalTo(Range(4, 20).map(_.toString).mkString) 127 | } 128 | } 129 | 130 | } 131 | 132 | "Enumeratee.take" should { 133 | 134 | "pass only first 3 chunks to Iteratee when applied with 3" in { 135 | 136 | val take3AndConsume = Enumeratee.take[String](3) &>> Iteratee.consume() 137 | val enumerator = Enumerator(Range(1, 20).map(_.toString): _*) 138 | Await.result(enumerator |>>> take3AndConsume, Duration.Inf) must equalTo(List(1, 2, 3).map(_.toString).mkString) 139 | 140 | } 141 | 142 | "passes along what's left of chunks after taking 3" in { 143 | mustExecute(1) { flatMapEC => 144 | val take3AndConsume = (Enumeratee.take[String](3) &>> Iteratee.consume()).flatMap(_ => Iteratee.consume())(flatMapEC) 145 | val enumerator = Enumerator(Range(1, 20).map(_.toString): _*) 146 | Await.result(enumerator |>>> take3AndConsume, Duration.Inf) must equalTo(Range(4, 20).map(_.toString).mkString) 147 | } 148 | } 149 | 150 | "not execute callback on take 0" in { 151 | mustExecute(0, 0) { (generateEC, foldEC) => 152 | var triggered = false 153 | val enumerator = Enumerator.generateM { 154 | triggered = true 155 | Future(Some(1))(dec) 156 | }(generateEC) 157 | Await.result(enumerator &> Enumeratee.take(0) |>>> Iteratee.fold(0)((_: Int) + (_: Int))(foldEC), Duration.Inf) must equalTo(0) 158 | triggered must beFalse 159 | } 160 | } 161 | 162 | } 163 | 164 | "Enumeratee.takeWhile" should { 165 | 166 | "pass chunks until condition is not met" in { 167 | mustExecute(4) { takeWhileEC => 168 | val take3AndConsume = Enumeratee.takeWhile[String](_ != "4")(takeWhileEC) &>> Iteratee.consume() 169 | val enumerator = Enumerator(Range(1, 20).map(_.toString): _*) 170 | Await.result(enumerator |>>> take3AndConsume, Duration.Inf) must equalTo(List(1, 2, 3).map(_.toString).mkString) 171 | } 172 | } 173 | 174 | "passes along what's left of chunks after taking" in { 175 | mustExecute(4, 1, 0) { (takeWhileEC, consumeFlatMapEC, generateEC) => 176 | val take3AndConsume = (Enumeratee.takeWhile[String](_ != "4")(takeWhileEC) &>> Iteratee.consume()).flatMap(_ => Iteratee.consume())(consumeFlatMapEC) 177 | val enumerator = Enumerator(Range(1, 20).map(_.toString): _*) 178 | Await.result(enumerator |>>> take3AndConsume, Duration.Inf) must equalTo(Range(4, 20).map(_.toString).mkString) 179 | } 180 | } 181 | 182 | "pass input through while the predicate is met" in { 183 | mustExecute(3) { breakEC => 184 | mustTransformTo(1, 2, 3, 2, 1)(1, 2)(Enumeratee.takeWhile[Int](_ <= 2)(breakEC)) 185 | } 186 | } 187 | 188 | } 189 | 190 | "Enumeratee.breakE" should { 191 | 192 | "pass input through until the predicate is met" in { 193 | mustExecute(3) { breakEC => 194 | mustTransformTo(1, 2, 3, 2, 1)(1, 2)(Enumeratee.breakE[Int](_ > 2)(breakEC)) 195 | } 196 | } 197 | 198 | } 199 | 200 | "Enumeratee.onIterateeDone" should { 201 | 202 | "call the callback when the iteratee is done" in { 203 | mustExecute(1) { doneEC => 204 | val count = new java.util.concurrent.atomic.AtomicInteger() 205 | mustTransformTo(1, 2, 3)(1, 2, 3)(Enumeratee.onIterateeDone(() => count.incrementAndGet())(doneEC)) 206 | count.get() must equalTo(1) 207 | } 208 | } 209 | 210 | } 211 | 212 | "Enumeratee.onEOF" should { 213 | 214 | "call the callback on EOF" in { 215 | mustExecute(1) { eofEC => 216 | val count = new java.util.concurrent.atomic.AtomicInteger() 217 | mustTransformTo(1, 2, 3)(1, 2, 3)(Enumeratee.onEOF(() => count.incrementAndGet())(eofEC)) 218 | count.get() must equalTo(1) 219 | } 220 | } 221 | 222 | } 223 | 224 | "Traversable.take" should { 225 | 226 | "pass only first 3 elements to Iteratee when applied with 3" in { 227 | 228 | val take3AndConsume = Traversable.take[String](3) &>> Iteratee.consume() 229 | val enumerator = Enumerator("he", "ybbb", "bbb") 230 | Await.result(enumerator |>>> take3AndConsume, Duration.Inf) must equalTo("hey") 231 | 232 | } 233 | 234 | "pass along what's left after taking 3 elements" in { 235 | mustExecute(1) { consumeFlatMapEC => 236 | val take3AndConsume = (Traversable.take[String](3) &>> Iteratee.consume()).flatMap(_ => Iteratee.consume())(consumeFlatMapEC) 237 | val enumerator = Enumerator("he", "ybbb", "bbb") 238 | Await.result(enumerator |>>> take3AndConsume, Duration.Inf) must equalTo("bbbbbb") 239 | } 240 | } 241 | 242 | } 243 | 244 | "Enumeratee.map" should { 245 | 246 | "add one to each of the ints enumerated" in { 247 | mustExecute(4) { mapEC => 248 | val add1AndConsume = Enumeratee.map[Int](i => List(i + 1))(mapEC) &>> Iteratee.consume[List[Int]]() 249 | val enumerator = Enumerator(1, 2, 3, 4) 250 | Await.result(enumerator |>>> add1AndConsume, Duration.Inf) must equalTo(Seq(2, 3, 4, 5)) 251 | } 252 | } 253 | 254 | "infer its types correctly from previous enumeratee" in { 255 | mustExecute(0, 0) { (map1EC, map2EC) => 256 | val add1AndConsume = Enumeratee.map[Int](i => i + 1)(map1EC) ><> Enumeratee.map[Int](i => List(i))(map2EC) &>> 257 | Iteratee.consume[List[Int]]() 258 | val check: Iteratee[Int, List[Int]] = add1AndConsume 259 | true //this test is about compilation and if it compiles it means we got it right 260 | } 261 | } 262 | 263 | "infer its types correctly from the preceeding enumerator" in { 264 | mustExecute(0) { mapEC => 265 | val addOne = Enumerator(1, 2, 3, 4) &> Enumeratee.map[Int](i => i + 1)(mapEC) 266 | val check: Enumerator[Int] = addOne 267 | true //this test is about compilation and if it compiles it means we got it right 268 | } 269 | } 270 | 271 | } 272 | 273 | "Enumeratee.flatten" should { 274 | 275 | "passAlong a future enumerator" in { 276 | mustExecute(9) { sumEC => 277 | val passAlongFuture = Enumeratee.flatten { 278 | Future { 279 | Enumeratee.passAlong[Int] 280 | }(ExecutionContext.global) 281 | } 282 | val sum = Iteratee.fold[Int, Int](0)(_ + _)(sumEC) 283 | val enumerator = Enumerator(1, 2, 3, 4, 5, 6, 7, 8, 9) 284 | Await.result(enumerator |>>> passAlongFuture &>> sum, Duration.Inf) must equalTo(45) 285 | } 286 | } 287 | 288 | } 289 | 290 | "Enumeratee.filter" should { 291 | 292 | "only enumerate input that satisfies the predicate" in { 293 | mustExecute(6) { filterEC => 294 | mustTransformTo("One", "Two", "Three", "Four", "Five", "Six")("One", "Two", "Six")(Enumeratee.filter[String](_.length < 4)(filterEC)) 295 | } 296 | } 297 | 298 | } 299 | 300 | "Enumeratee.filterNot" should { 301 | 302 | "only enumerate input that doesn't satisfy the predicate" in { 303 | mustExecute(6) { filterEC => 304 | mustTransformTo("One", "Two", "Three", "Four", "Five", "Six")("Three", "Four", "Five")(Enumeratee.filterNot[String](_.length < 4)(filterEC)) 305 | } 306 | } 307 | 308 | } 309 | 310 | "Enumeratee.collect" should { 311 | 312 | "ignores input that doesn't satisfy the predicate and transform the input when matches" in { 313 | mustExecute(6) { collectEC => 314 | mustTransformTo("One", "Two", "Three", "Four", "Five", "Six")("ONE", "TWO", "SIX")(Enumeratee.collect[String] { case e @ ("One" | "Two" | "Six") => e.toUpperCase(java.util.Locale.ENGLISH) }(collectEC)) 315 | } 316 | } 317 | 318 | } 319 | 320 | "Enumeratee.grouped" should { 321 | 322 | "group input elements according to a folder iteratee" in { 323 | mustExecute(9, 7, 3) { (mapInputEC, foldEC, mapEC) => 324 | val folderIteratee = 325 | Enumeratee.mapInput[String] { 326 | case Input.El("Concat") => Input.EOF; 327 | case other => other 328 | }(mapInputEC) &>> 329 | Iteratee.fold[String, String]("")((s, e) => s + e)(foldEC) 330 | 331 | val result = 332 | Enumerator("He", "ll", "o", "Concat", "Wo", "r", "ld", "Concat", "!") &> 333 | Enumeratee.grouped(folderIteratee) ><> 334 | Enumeratee.map[String](List(_))(mapEC) |>>> 335 | Iteratee.consume[List[String]]() 336 | Await.result(result, Duration.Inf) must equalTo(List("Hello", "World", "!")) 337 | } 338 | } 339 | 340 | } 341 | 342 | "Enumeratee.grouped" should { 343 | "pass along what is consumed by the last folder iteratee on EOF" in { 344 | mustExecute(4, 3) { (splitEC, mapEC) => 345 | val upToSpace = Traversable.splitOnceAt[String, Char](c => c != '\n')(implicitly[String => scala.collection.TraversableLike[Char, String]], splitEC) &>> Iteratee.consume() 346 | 347 | val result = (Enumerator("dasdasdas ", "dadadasda\nshouldb\neinnext") &> Enumeratee.grouped(upToSpace) ><> Enumeratee.map[String](_ + "|")(mapEC)) |>>> Iteratee.consume[String]() 348 | Await.result(result, Duration.Inf) must equalTo("dasdasdas dadadasda|shouldb|einnext|") 349 | } 350 | } 351 | } 352 | 353 | "Enumeratee.scanLeft" should { 354 | 355 | "transform elements using a sate" in { 356 | mustExecute(4) { mapEC => 357 | val result = 358 | Enumerator(1, 2, 3, 4) &> 359 | Enumeratee.scanLeft[Int](0)(_ + _) ><> 360 | Enumeratee.map[Int](List(_))(mapEC) |>>> 361 | Iteratee.consume[List[Int]]() 362 | 363 | Await.result(result, Duration.Inf) must equalTo(List(1, 3, 6, 10)) 364 | } 365 | } 366 | 367 | } 368 | 369 | "Enumeratee.recover" should { 370 | 371 | "perform computations and log errors" in { 372 | mustExecute(3, 3) { (recoverEC, mapEC) => 373 | val eventuallyInput = Promise[Input[Int]]() 374 | val result = Enumerator(0, 2, 4) &> Enumeratee.recover[Int] { (_, input) => 375 | eventuallyInput.success(input) 376 | }(recoverEC) &> Enumeratee.map[Int] { i => 377 | 8 / i 378 | }(mapEC) |>>> Iteratee.getChunks // => List(4, 2) 379 | 380 | Await.result(result, Duration.Inf) must equalTo(List(4, 2)) 381 | Await.result(eventuallyInput.future, Duration.Inf) must equalTo(Input.El(0)) 382 | } 383 | } 384 | } 385 | 386 | } 387 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/ConcurrentSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import java.util.concurrent.{ TimeUnit, CountDownLatch } 7 | import java.util.concurrent.TimeUnit._ 8 | import java.util.concurrent.atomic.AtomicInteger 9 | import org.specs2.mutable._ 10 | import scala.concurrent._ 11 | import scala.concurrent.duration.Duration 12 | import scala.concurrent.ExecutionContext.Implicits.global 13 | import scala.util.Try 14 | 15 | object ConcurrentSpec extends Specification 16 | with IterateeSpecification with ExecutionSpecification { 17 | 18 | "Concurrent.broadcast (0-arg)" should { 19 | "broadcast the same to already registered iteratees" in { 20 | mustExecute(38) { foldEC => 21 | val (broadcaster, pushHere) = Concurrent.broadcast[String] 22 | val results = Future.sequence(Range(1, 20).map(_ => Iteratee.fold[String, String]("") { (s, e) => s + e }(foldEC)).map(broadcaster.apply).map(_.flatMap(_.run))) 23 | pushHere.push("beep") 24 | pushHere.push("beep") 25 | pushHere.eofAndEnd() 26 | Await.result(results, Duration.Inf) must equalTo(Range(1, 20).map(_ => "beepbeep")) 27 | } 28 | } 29 | "allow invoking end twice" in { 30 | val (broadcaster, pushHere) = Concurrent.broadcast[String] 31 | val result = broadcaster |>>> Iteratee.getChunks[String] 32 | pushHere.push("beep") 33 | pushHere.end() 34 | pushHere.end() 35 | Await.result(result, Duration.Inf) must_== Seq("beep") 36 | } 37 | "return the iteratee if already ended before the iteratee is attached" in { 38 | val (broadcaster, pushHere) = Concurrent.broadcast[String] 39 | pushHere.end() 40 | val result = broadcaster |>>> Iteratee.getChunks[String] 41 | Await.result(result, Duration.Inf) must_== Nil 42 | } 43 | "allow ending with an exception" in { 44 | val (broadcaster, pushHere) = Concurrent.broadcast[String] 45 | val result = broadcaster |>>> Iteratee.getChunks[String] 46 | pushHere.end(new RuntimeException("foo")) 47 | Await.result(result, Duration.Inf) must throwA[RuntimeException](message = "foo") 48 | } 49 | "update the end result after end is already called" in { 50 | val (broadcaster, pushHere) = Concurrent.broadcast[String] 51 | val result1 = broadcaster |>>> Iteratee.getChunks[String] 52 | pushHere.end(new RuntimeException("foo")) 53 | Await.result(result1, Duration.Inf) must throwA[RuntimeException](message = "foo") 54 | pushHere.end() 55 | val result2 = broadcaster |>>> Iteratee.getChunks[String] 56 | Await.result(result2, Duration.Inf) must_== Nil 57 | } 58 | } 59 | 60 | "Concurrent.buffer" should { 61 | 62 | def now = System.currentTimeMillis() 63 | 64 | "not slow down the enumerator if the iteratee is slow" in { 65 | val enumeratorFinished = new CountDownLatch(1) 66 | mustExecute(10) { bufferEC => 67 | // This enumerator emits elements as fast as it can and 68 | // signals that it has finished using a latch 69 | val fastEnumerator = Enumerator((1 to 10): _*).onDoneEnumerating { 70 | enumeratorFinished.countDown() 71 | } 72 | val slowIteratee = Iteratee.foldM(List[Int]()) { (s, e: Int) => 73 | Future { 74 | enumeratorFinished.await(waitTime.toMillis, TimeUnit.MILLISECONDS) 75 | s :+ e 76 | } 77 | } 78 | // Concurrent.buffer should buffer elements so that the the 79 | // fastEnumerator can complete even though the slowIteratee 80 | // won't consume anything until it has finished. 81 | val result = 82 | fastEnumerator &> 83 | Concurrent.buffer(20, (_: Input[Int]) => 1)(bufferEC) |>>> 84 | slowIteratee 85 | 86 | await(result) must_== ((1 to 10).to[List]) 87 | } 88 | } 89 | 90 | "throw an exception when buffer is full" in { 91 | testExecution { foldEC => 92 | val foldCount = new AtomicInteger() 93 | val p = Promise[List[Long]]() 94 | val stuckIteratee = Iteratee.foldM(List[Long]()) { (s, e: Long) => foldCount.incrementAndGet(); p.future }(foldEC) 95 | val fastEnumerator = Enumerator[Long](1, 2, 3, 4, 5, 6, 7, 8, 9, 10) 96 | val result = 97 | fastEnumerator &> 98 | Concurrent.buffer(7) |>>> 99 | stuckIteratee 100 | 101 | Await.result(result, Duration.Inf) must throwAn[Exception]("buffer overflow") 102 | foldEC.executionCount must equalTo(foldCount.get()) 103 | } 104 | } 105 | 106 | "drop intermediate unused input, swallow even the unused eof forcing u to pass it twice" in { 107 | testExecution { (flatMapEC, mapEC) => 108 | val p = Promise[List[Long]]() 109 | val slowIteratee = Iteratee.flatten(timeout(Cont[Long, List[Long]] { 110 | case Input.El(e) => Done(List(e), Input.Empty) 111 | case in => throw new MatchError(in) // Shouldn't occur, but here to suppress compiler warning 112 | }, Duration(100, MILLISECONDS))) 113 | val fastEnumerator = Enumerator[Long](1, 2, 3, 4, 5, 6, 7, 8, 9, 10) >>> Enumerator.eof 114 | val preparedMapEC = mapEC.prepare() 115 | val result = 116 | fastEnumerator |>>> 117 | (Concurrent.buffer(20) &>> 118 | slowIteratee).flatMap { l => Iteratee.getChunks.map(l ++ (_: List[Long]))(preparedMapEC) }(flatMapEC) 119 | 120 | Await.result(result, Duration.Inf) must not equalTo (List(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) 121 | flatMapEC.executionCount must beGreaterThan(0) 122 | mapEC.executionCount must equalTo(flatMapEC.executionCount) 123 | } 124 | } 125 | 126 | } 127 | 128 | "Concurrent.lazyAndErrIfNotReady" should { 129 | 130 | "return an error if the iteratee is taking too long" in { 131 | // Create an iteratee that never finishes. This means that our 132 | // Concurrent.lazyAndErrIfNotReady timeout will always fire. 133 | // Once we've got our timeout we release the iteratee so that 134 | // it can finish normally. 135 | val gotResult = new CountDownLatch(1) 136 | val slowIteratee = Iteratee.foldM(List[Int]()) { (s, e: Int) => 137 | Future { 138 | gotResult.await(waitTime.toMillis, TimeUnit.MILLISECONDS) 139 | s :+ e 140 | } 141 | } 142 | 143 | val fastEnumerator = Enumerator((1 to 10): _*) >>> Enumerator.eof 144 | val result = Try(await(fastEnumerator &> Concurrent.lazyAndErrIfNotReady(50) |>>> slowIteratee)) 145 | // We've got our result (hopefully a timeout), so let the iteratee 146 | // complete. 147 | gotResult.countDown() 148 | result.get must throwA[Exception]("iteratee is taking too long") 149 | } 150 | 151 | } 152 | 153 | "Concurrent.unicast" should { 154 | "allow to push messages and end" in { 155 | mustExecute(2, 2) { (unicastEC, foldEC) => 156 | val a = "FOO" 157 | val b = "bar" 158 | val startCount = new AtomicInteger() 159 | val completeCount = new AtomicInteger() 160 | val errorCount = new AtomicInteger() 161 | val enumerator = Concurrent.unicast[String]( 162 | c => { 163 | startCount.incrementAndGet() 164 | c.push(a) 165 | c.push(b) 166 | c.eofAndEnd() 167 | }, 168 | () => completeCount.incrementAndGet(), 169 | (_: String, _: Input[String]) => errorCount.incrementAndGet() 170 | )(unicastEC) 171 | val promise = (enumerator |>> Iteratee.fold[String, String]("")(_ ++ _)(foldEC)).flatMap(_.run) 172 | 173 | Await.result(promise, Duration.Inf) must equalTo(a + b) 174 | startCount.get() must equalTo(1) 175 | completeCount.get() must equalTo(0) 176 | errorCount.get() must equalTo(0) 177 | } 178 | } 179 | 180 | "call the onComplete callback when the iteratee is done" in { 181 | mustExecute(2) { unicastEC => 182 | val completed = Promise[String] 183 | 184 | val enumerator = Concurrent.unicast[String](onStart = { c => 185 | c.push("foo") 186 | c.push("bar") 187 | }, onComplete = { 188 | completed.success("called") 189 | })(unicastEC) 190 | 191 | val future = enumerator |>>> Cont { 192 | case Input.El(data) => Done(data) 193 | case _ => Done("didn't get data") 194 | } 195 | 196 | Await.result(future, Duration.Inf) must_== "foo" 197 | Await.result(completed.future, Duration.Inf) must_== "called" 198 | } 199 | } 200 | 201 | "call the onError callback when the iteratee encounters an error" in { 202 | mustExecute(2) { unicastEC => 203 | val error = Promise[String] 204 | 205 | val enumerator = Concurrent.unicast[String](onStart = { c => 206 | c.push("foo") 207 | c.push("bar") 208 | }, onError = { (err, input) => 209 | error.success(err) 210 | })(unicastEC) 211 | 212 | enumerator |>> Cont { 213 | case Input.El(data) => Error(data, Input.Empty) 214 | case in => Error("didn't get data", in) 215 | } 216 | 217 | Await.result(error.future, Duration.Inf) must_== "foo" 218 | } 219 | } 220 | 221 | "allow invoking end twice" in { 222 | mustExecute(1) { unicastEC => 223 | val endInvokedTwice = new CountDownLatch(1) 224 | val enumerator = Concurrent.unicast[String](onStart = { c => 225 | c.end() 226 | c.end() 227 | endInvokedTwice.countDown() 228 | })(unicastEC) 229 | 230 | Await.result(enumerator |>>> Iteratee.getChunks[String], Duration.Inf) must_== Nil 231 | endInvokedTwice.await(10, TimeUnit.SECONDS) must_== true 232 | } 233 | } 234 | 235 | "allow ending with an exception" in { 236 | mustExecute(1) { unicastEC => 237 | val enumerator = Concurrent.unicast[String](onStart = { c => 238 | c.end(new RuntimeException("foo")) 239 | })(unicastEC) 240 | 241 | val result = enumerator |>>> Iteratee.getChunks[String] 242 | Await.result(result, Duration.Inf) must throwA[RuntimeException](message = "foo") 243 | } 244 | } 245 | 246 | "only use the invocation of the first end" in { 247 | mustExecute(1) { unicastEC => 248 | val endInvokedTwice = new CountDownLatch(1) 249 | val enumerator = Concurrent.unicast[String](onStart = { c => 250 | c.end() 251 | c.end(new RuntimeException("foo")) 252 | endInvokedTwice.countDown() 253 | })(unicastEC) 254 | 255 | val result = enumerator |>>> Iteratee.getChunks[String] 256 | endInvokedTwice.await(10, TimeUnit.SECONDS) must_== true 257 | Await.result(result, Duration.Inf) must_== Nil 258 | } 259 | } 260 | 261 | } 262 | 263 | "Concurrent.broadcast (2-arg)" should { 264 | "call callback in the correct ExecutionContext" in { 265 | mustExecute(1) { callbackEC => 266 | val (e0, c) = Concurrent.broadcast[Int] 267 | val interestCount = new AtomicInteger() 268 | val interestDone = new CountDownLatch(1) 269 | val (e2, b) = Concurrent.broadcast(e0, { f => 270 | interestCount.incrementAndGet() 271 | interestDone.countDown() 272 | })(callbackEC) 273 | val i = e2 |>>> Iteratee.getChunks[Int] 274 | c.push(1) 275 | c.push(2) 276 | c.push(3) 277 | c.eofAndEnd() 278 | Await.result(i, Duration.Inf) must equalTo(List(1, 2, 3)) 279 | interestDone.await(30, SECONDS) must beTrue 280 | interestCount.get() must equalTo(1) 281 | } 282 | } 283 | } 284 | 285 | "Concurrent.patchPanel" should { 286 | 287 | "perform patching in the correct ExecutionContext" in { 288 | mustExecute(1) { ppEC => 289 | val e = Concurrent.patchPanel[Int] { pp => 290 | pp.patchIn(Enumerator.eof) 291 | }(ppEC) 292 | Await.result(e |>>> Iteratee.getChunks[Int], Duration.Inf) must equalTo(Nil) 293 | } 294 | } 295 | } 296 | 297 | "Concurrent.joined" should { 298 | "join the iteratee and enumerator if the enumerator is applied first" in { 299 | val (iteratee, enumerator) = Concurrent.joined[String] 300 | val result = enumerator |>>> Iteratee.getChunks[String] 301 | val unitResult = Enumerator("foo", "bar") |>>> iteratee 302 | await(result) must_== Seq("foo", "bar") 303 | await(unitResult) must_== (()) 304 | } 305 | "join the iteratee and enumerator if the iteratee is applied first" in { 306 | val (iteratee, enumerator) = Concurrent.joined[String] 307 | val unitResult = Enumerator("foo", "bar") |>>> iteratee 308 | val result = enumerator |>>> Iteratee.getChunks[String] 309 | await(result) must_== Seq("foo", "bar") 310 | await(unitResult) must_== (()) 311 | } 312 | "join the iteratee and enumerator if the enumerator is applied during the iteratees run" in { 313 | val (iteratee, enumerator) = Concurrent.joined[String] 314 | val (broadcast, channel) = Concurrent.broadcast[String] 315 | val unitResult = broadcast |>>> iteratee 316 | channel.push("foo") 317 | Thread.sleep(10) 318 | val result = enumerator |>>> Iteratee.getChunks[String] 319 | channel.push("bar") 320 | channel.end() 321 | await(result) must_== Seq("foo", "bar") 322 | await(unitResult) must_== (()) 323 | } 324 | "break early from infinite enumerators" in { 325 | val (iteratee, enumerator) = Concurrent.joined[String] 326 | val infinite = Enumerator.repeat("foo") 327 | val unitResult = infinite |>>> iteratee 328 | val head = enumerator |>>> Iteratee.head 329 | await(head) must beSome("foo") 330 | await(unitResult) must_== (()) 331 | } 332 | } 333 | 334 | "Concurrent.runPartial" should { 335 | "redeem the iteratee with the result and the partial enumerator" in { 336 | val (a, remaining) = await(Concurrent.runPartial(Enumerator("foo", "bar"), Iteratee.head[String])) 337 | a must beSome("foo") 338 | await(remaining |>>> Iteratee.getChunks[String]) must_== Seq("bar") 339 | } 340 | "work when there is no input left in the enumerator" in { 341 | val (a, remaining) = await(Concurrent.runPartial(Enumerator("foo", "bar"), Iteratee.getChunks[String])) 342 | a must_== Seq("foo", "bar") 343 | await(remaining |>>> Iteratee.getChunks[String]) must_== Nil 344 | } 345 | } 346 | } 347 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/EnumeratorsSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import java.nio.file.Files 7 | 8 | import org.specs2.mutable._ 9 | import java.io.{ ByteArrayInputStream, File, FileOutputStream, OutputStream } 10 | import java.util.concurrent.{ CountDownLatch, TimeUnit } 11 | import java.util.concurrent.atomic.AtomicInteger 12 | import play.api.libs.iteratee.Execution.Implicits.{ defaultExecutionContext => dec } 13 | import scala.concurrent.{ Promise, Future, Await } 14 | import scala.concurrent.duration.Duration 15 | 16 | object EnumeratorsSpec extends Specification 17 | with IterateeSpecification with ExecutionSpecification { 18 | 19 | "Enumerator's interleave" should { 20 | 21 | "mix it with another enumerator into one" in { 22 | mustExecute(8) { foldEC => 23 | val e1 = Enumerator(List(1), List(3), List(5), List(7)) 24 | val e2 = Enumerator(List(2), List(4), List(6), List(8)) 25 | val e = e1 interleave e2 26 | val kk = e |>>> Iteratee.fold(List.empty[Int])((r, e: List[Int]) => r ++ e)(foldEC) 27 | val result = Await.result(kk, Duration.Inf) 28 | result.diff(Seq(1, 2, 3, 4, 5, 6, 7, 8)) must equalTo(Seq()) 29 | } 30 | } 31 | 32 | "yield when both enumerators EOF" in { 33 | mustExecute(8) { foldEC => 34 | val e1 = Enumerator(List(1), List(3), List(5), List(7)) >>> Enumerator.enumInput(Input.EOF) 35 | val e2 = Enumerator(List(2), List(4), List(6), List(8)) >>> Enumerator.enumInput(Input.EOF) 36 | val e = e1 interleave e2 37 | val kk = e |>>> Iteratee.fold(List.empty[Int])((r, e: List[Int]) => r ++ e)(foldEC) 38 | val result = Await.result(kk, Duration.Inf) 39 | result.diff(Seq(1, 2, 3, 4, 5, 6, 7, 8)) must equalTo(Seq()) 40 | } 41 | } 42 | 43 | "yield when iteratee is done!" in { 44 | mustExecute(7) { foldEC => 45 | val e1 = Enumerator(List(1), List(3), List(5), List(7)) 46 | val e2 = Enumerator(List(2), List(4), List(6), List(8)) 47 | val e = e1 interleave e2 48 | val kk = e |>>> Enumeratee.take(7) &>> Iteratee.fold(List.empty[Int])((r, e: List[Int]) => r ++ e)(foldEC) 49 | val result = Await.result(kk, Duration.Inf) 50 | result.length must equalTo(7) 51 | } 52 | } 53 | 54 | "not necessarily go alternatively between two enumerators" in { 55 | mustExecute(1, 2) { (onDoneEC, unfoldEC) => 56 | val firstDone = Promise[Unit] 57 | val e1 = Enumerator(1, 2, 3, 4).onDoneEnumerating(firstDone.success(Unit))(onDoneEC) 58 | val e2 = Enumerator.unfoldM[Boolean, Int](true) { first => if (first) firstDone.future.map(_ => Some((false, 5))) else Future.successful(None) }(unfoldEC) 59 | val result = Await.result((e1 interleave e2) |>>> Iteratee.getChunks[Int], Duration.Inf) 60 | result must_== Seq(1, 2, 3, 4, 5) 61 | } 62 | } 63 | 64 | } 65 | 66 | "Enumerator.enumerate " should { 67 | "generate an Enumerator from a singleton Iterator" in { 68 | mustExecute(1) { foldEC => 69 | val iterator = scala.collection.Iterator.single[Int](3) 70 | val futureOfResult = Enumerator.enumerate(iterator) |>>> 71 | Enumeratee.take(1) &>> 72 | Iteratee.fold(List.empty[Int])((r, e: Int) => e :: r)(foldEC) 73 | val result = Await.result(futureOfResult, Duration.Inf) 74 | result(0) must equalTo(3) 75 | result.length must equalTo(1) 76 | } 77 | } 78 | 79 | "take as much element as in the iterator in the right order" in { 80 | mustExecute(50) { foldEC => 81 | val iterator = scala.collection.Iterator.range(0, 50) 82 | val futureOfResult = Enumerator.enumerate(iterator) |>>> 83 | Enumeratee.take(100) &>> 84 | Iteratee.fold(Seq.empty[Int])((r, e: Int) => r :+ e)(foldEC) 85 | val result = Await.result(futureOfResult, Duration.Inf) 86 | result.length must equalTo(50) 87 | result(0) must equalTo(0) 88 | result(49) must equalTo(49) 89 | } 90 | } 91 | "work with Seq too" in { 92 | mustExecute(6) { foldEC => 93 | val seq = List(1, 2, 3, 7, 42, 666) 94 | val futureOfResult = Enumerator.enumerate(seq) |>>> 95 | Enumeratee.take(100) &>> 96 | Iteratee.fold(Seq.empty[Int])((r, e: Int) => r :+ e)(foldEC) 97 | val result = Await.result(futureOfResult, Duration.Inf) 98 | result.length must equalTo(6) 99 | result(0) must equalTo(1) 100 | result(4) must equalTo(42) 101 | } 102 | } 103 | } 104 | 105 | /*"Enumerator's PatchPanel" should { 106 | 107 | "allow to patch in different Enumerators" in { 108 | import play.api.libs.iteratee.concurrent.Promise 109 | val pp = Promise[Concurrent.PatchPanel[Int]]() 110 | val e = Concurrent.patchPanel[Int](p => pp.redeem(p)) 111 | val i1 = Iteratee.fold[Int,Int](0){(s,i) => println(i);s+i} 112 | val sum = e |>> i1 113 | val p = pp.future.await.get 114 | p.patchIn(Enumerator(1,2,3,4)) 115 | p.patchIn(Enumerator.eof) 116 | sum.flatMap(_.run).value1.get must equalTo(10) 117 | } 118 | 119 | }*/ 120 | 121 | "Enumerator.apply" should { 122 | "enumerate zero args" in { 123 | mustEnumerateTo()(Enumerator()) 124 | } 125 | "enumerate 1 arg" in { 126 | mustEnumerateTo(1)(Enumerator(1)) 127 | } 128 | "enumerate more than 1 arg" in { 129 | mustEnumerateTo(1, 2)(Enumerator(1, 2)) 130 | mustEnumerateTo(1, 2, 3)(Enumerator(1, 2, 3)) 131 | } 132 | } 133 | 134 | "Enumerator" should { 135 | 136 | "call onDoneEnumerating callback" in { 137 | mustExecute(1) { onDoneEC => 138 | val count = new java.util.concurrent.atomic.AtomicInteger() 139 | mustEnumerateTo(1, 2, 3)(Enumerator(1, 2, 3).onDoneEnumerating(count.incrementAndGet())(onDoneEC)) 140 | count.get() must equalTo(1) 141 | } 142 | } 143 | 144 | "call onDoneEnumerating callback when an error is encountered" in { 145 | mustExecute(1) { onDoneEC => 146 | val count = new java.util.concurrent.atomic.AtomicInteger() 147 | mustPropagateFailure( 148 | Enumerator(1, 2, 3).onDoneEnumerating(count.incrementAndGet())(onDoneEC) 149 | ) 150 | count.get() must_== 1 151 | } 152 | } 153 | 154 | "transform input elements with map" in { 155 | mustExecute(3) { mapEC => 156 | mustEnumerateTo(2, 4, 6)(Enumerator(1, 2, 3).map(_ * 2)(mapEC)) 157 | } 158 | } 159 | 160 | "transform input with map" in { 161 | mustExecute(3) { mapEC => 162 | mustEnumerateTo(2, 4, 6)(Enumerator(1, 2, 3).mapInput(_.map(_ * 2))(mapEC)) 163 | } 164 | } 165 | 166 | "be transformed to another Enumerator using flatMap" in { 167 | mustExecute(3, 30) { (flatMapEC, foldEC) => 168 | val e = Enumerator(10, 20, 30).flatMap(i => Enumerator((i until i + 10): _*))(flatMapEC) 169 | val it = Iteratee.fold[Int, Int](0)((sum, x) => sum + x)(foldEC) 170 | Await.result(e |>>> it, Duration.Inf) must equalTo((10 until 40).sum) 171 | } 172 | } 173 | 174 | } 175 | 176 | "Enumerator.generateM" should { 177 | "generate a stream of values until the expression is None" in { 178 | mustExecute(12, 11) { (generateEC, foldEC) => 179 | val a = (0 to 10).toList 180 | val it = a.iterator 181 | 182 | val enumerator = Enumerator.generateM(Future(if (it.hasNext) Some(it.next()) else None))(generateEC) 183 | 184 | Await.result(enumerator |>>> Iteratee.fold[Int, String]("")(_ + _)(foldEC), Duration.Inf) must equalTo("012345678910") 185 | } 186 | } 187 | 188 | "Can be composed with another enumerator (doesn't send EOF)" in { 189 | mustExecute(12, 12) { (generateEC, foldEC) => 190 | val a = (0 to 10).toList 191 | val it = a.iterator 192 | 193 | val enumerator = Enumerator.generateM(Future(if (it.hasNext) Some(it.next()) else None))(generateEC) >>> Enumerator(12) 194 | 195 | Await.result(enumerator |>>> Iteratee.fold[Int, String]("")(_ + _)(foldEC), Duration.Inf) must equalTo("01234567891012") 196 | } 197 | } 198 | 199 | } 200 | 201 | "Enumerator.callback1" should { 202 | "Call onError on iteratee's error state" in { 203 | val it = Error[String]("foo", Input.Empty) 204 | val errorCount = new AtomicInteger(0) 205 | 206 | val enum = Enumerator.fromCallback1[String]( 207 | b => Future.successful(None), 208 | () => (), 209 | (msg, input) => 210 | errorCount.incrementAndGet() 211 | ) 212 | 213 | val result = enum |>>> it 214 | 215 | Await.ready(result, Duration(30, TimeUnit.SECONDS)) 216 | errorCount.get() must equalTo(1) 217 | } 218 | 219 | "Call onError on future failure" in { 220 | val it1 = Iteratee.fold1[String, String](Future.successful(""))((_, _) => Future.failed(new RuntimeException())) 221 | val it2 = Iteratee.fold1[String, String](Future.failed(new RuntimeException()))((_, _) => Future.failed(new RuntimeException())) 222 | val errorCount = new AtomicInteger(0) 223 | 224 | val enum = Enumerator.fromCallback1[String]( 225 | b => Future.successful(Some("")), 226 | () => (), 227 | (msg, input) => 228 | errorCount.incrementAndGet() 229 | ) 230 | 231 | val result1 = enum |>>> it1 232 | val result2 = enum |>>> it2 233 | Await.ready(result1.zip(result2), Duration(2, TimeUnit.SECONDS)) 234 | 235 | errorCount.get() must equalTo(2) 236 | } 237 | 238 | "generate a stream of values until the expression is None" in { 239 | mustExecute(5) { callbackEC => 240 | val it = (1 to 3).iterator // FIXME: Probably not thread-safe 241 | val completeCount = new AtomicInteger(0) 242 | val completeDone = new CountDownLatch(1) 243 | val errorCount = new AtomicInteger(0) 244 | val enumerator = Enumerator.fromCallback1( 245 | b => Future(if (it.hasNext) Some((b, it.next())) else None), 246 | () => { 247 | completeCount.incrementAndGet() 248 | completeDone.countDown() 249 | }, 250 | (_: String, _: Input[(Boolean, Int)]) => errorCount.incrementAndGet() 251 | )(callbackEC) 252 | mustEnumerateTo((true, 1), (false, 2), (false, 3))(enumerator) 253 | completeDone.await(30, TimeUnit.SECONDS) must beTrue 254 | completeCount.get() must equalTo(1) 255 | errorCount.get() must equalTo(0) 256 | } 257 | } 258 | } 259 | 260 | "Enumerator.fromStream" should { 261 | "read bytes from a stream" in { 262 | mustExecute(3) { fromStreamEC => 263 | val s = "hello" 264 | val enumerator = Enumerator.fromStream(new ByteArrayInputStream(s.getBytes))(fromStreamEC).map(new String(_)) 265 | mustEnumerateTo(s)(enumerator) 266 | } 267 | } 268 | "close the stream" in { 269 | class CloseableByteArrayInputStream(bytes: Array[Byte]) extends ByteArrayInputStream(bytes) { 270 | @volatile var closed = false 271 | 272 | override def close() = { 273 | closed = true 274 | } 275 | } 276 | 277 | "when done normally" in { 278 | val stream = new CloseableByteArrayInputStream(Array.empty) 279 | mustExecute(2) { fromStreamEC => 280 | Await.result(Enumerator.fromStream(stream)(fromStreamEC)(Iteratee.ignore), Duration.Inf) 281 | stream.closed must beTrue 282 | } 283 | } 284 | "when completed abnormally" in { 285 | val stream = new CloseableByteArrayInputStream("hello".getBytes) 286 | mustExecute(2) { fromStreamEC => 287 | mustPropagateFailure(Enumerator.fromStream(stream)(fromStreamEC)) 288 | stream.closed must beTrue 289 | } 290 | } 291 | } 292 | } 293 | 294 | "Enumerator.fromFile" should { 295 | "read bytes from a file" in { 296 | mustExecute(3) { fromFileEC => 297 | val f = File.createTempFile("EnumeratorSpec", "fromFile") 298 | try { 299 | val s = "hello" 300 | val out = new FileOutputStream(f) 301 | out.write(s.getBytes) 302 | out.close() 303 | val enumerator = Enumerator.fromFile(f)(fromFileEC).map(new String(_)) 304 | mustEnumerateTo(s)(enumerator) 305 | } finally { 306 | f.delete() 307 | } 308 | } 309 | } 310 | } 311 | 312 | "Enumerator.fromPath" should { 313 | "read bytes from a path" in { 314 | mustExecute(3) { fromPathEC => 315 | val f = Files.createTempFile("EnumeratorSpec", "fromPath") 316 | try { 317 | val s = "hello" 318 | val out = Files.newOutputStream(f) 319 | out.write(s.getBytes) 320 | out.close() 321 | val enumerator = Enumerator.fromPath(f)(fromPathEC).map(new String(_)) 322 | mustEnumerateTo(s)(enumerator) 323 | } finally { 324 | Files.delete(f) 325 | } 326 | } 327 | } 328 | } 329 | 330 | "Enumerator.unfoldM" should { 331 | "Can be composed with another enumerator (doesn't send EOF)" in { 332 | mustExecute(12, 12) { (foldEC, unfoldEC) => 333 | val enumerator = Enumerator.unfoldM[Int, Int](0)(s => Future(if (s > 10) None else Some((s + 1, s + 1))))(unfoldEC) >>> Enumerator(12) 334 | 335 | Await.result(enumerator |>>> Iteratee.fold[Int, String]("")(_ + _)(foldEC), Duration.Inf) must equalTo("123456789101112") 336 | } 337 | } 338 | } 339 | 340 | "Enumerator.unfold" should { 341 | "unfolds a value into input for an enumerator" in { 342 | mustExecute(5) { unfoldEC => 343 | val enumerator = Enumerator.unfold[Int, Int](0)(s => if (s > 3) None else Some((s + 1, s)))(unfoldEC) 344 | mustEnumerateTo(0, 1, 2, 3)(enumerator) 345 | } 346 | } 347 | } 348 | 349 | "Enumerator.repeat" should { 350 | "supply input from a by-name arg" in { 351 | mustExecute(3) { repeatEC => 352 | val count = new AtomicInteger(0) 353 | val fut = Enumerator.repeat(count.incrementAndGet())(repeatEC) |>>> (Enumeratee.take(3) &>> Iteratee.getChunks[Int]) 354 | Await.result(fut, Duration.Inf) must equalTo(List(1, 2, 3)) 355 | } 356 | } 357 | } 358 | 359 | "Enumerator.repeatM" should { 360 | "supply input from a by-name arg" in { 361 | mustExecute(3) { repeatEC => 362 | val count = new AtomicInteger(0) 363 | val fut = Enumerator.repeatM(Future.successful(count.incrementAndGet()))(repeatEC) |>>> (Enumeratee.take(3) &>> Iteratee.getChunks[Int]) 364 | Await.result(fut, Duration.Inf) must equalTo(List(1, 2, 3)) 365 | } 366 | } 367 | } 368 | 369 | "Enumerator.outputStream" should { 370 | "produce the same value written in the OutputStream" in { 371 | mustExecute(1, 2) { (outputEC, foldEC) => 372 | val a = "FOO" 373 | val b = "bar" 374 | val enumerator = Enumerator.outputStream { outputStream => 375 | outputStream.write(a.toArray.map(_.toByte)) 376 | outputStream.write(b.toArray.map(_.toByte)) 377 | outputStream.close() 378 | }(outputEC) 379 | val promise = (enumerator |>>> Iteratee.fold[Array[Byte], Array[Byte]](Array[Byte]())(_ ++ _)(foldEC)) 380 | Await.result(promise, Duration.Inf).map(_.toChar).foldLeft("")(_ + _) must equalTo(a + b) 381 | } 382 | } 383 | 384 | "not block" in { 385 | mustExecute(1) { outputEC => 386 | var os: OutputStream = null 387 | val osReady = new CountDownLatch(1) 388 | val enumerator = Enumerator.outputStream { o => os = o; osReady.countDown() }(outputEC) 389 | val promiseIteratee = Promise[Iteratee[Array[Byte], Array[Byte]]] 390 | val future = enumerator |>>> Iteratee.flatten(promiseIteratee.future) 391 | osReady.await(30, TimeUnit.SECONDS) must beTrue 392 | // os should now be set 393 | os.write("hello".getBytes) 394 | os.write(" ".getBytes) 395 | os.write("world".getBytes) 396 | os.close() 397 | promiseIteratee.success(Iteratee.consume[Array[Byte]]()) 398 | Await.result(future, Duration("10s")) must_== "hello world".getBytes 399 | } 400 | } 401 | } 402 | 403 | } 404 | -------------------------------------------------------------------------------- /iteratees/src/test/scala/play/api/libs/iteratee/IterateesSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009-2016 Lightbend Inc. 3 | */ 4 | package play.api.libs.iteratee 5 | 6 | import org.specs2.mutable._ 7 | import scala.concurrent.{ Future, ExecutionContext } 8 | import scala.util.{ Failure, Try } 9 | 10 | object IterateesSpec extends Specification 11 | with IterateeSpecification with ExecutionSpecification { 12 | 13 | def checkFoldResult[A, E](i: Iteratee[A, E], expected: Step[A, E]) = { 14 | mustExecute(1) { foldEC => 15 | await(i.fold(s => Future.successful(s))(foldEC)) must equalTo(expected) 16 | } 17 | } 18 | 19 | def checkFoldTryResult[A, E](i: Iteratee[A, E], expected: Try[Step[A, E]]) = { 20 | mustExecute(0) { foldEC => 21 | Try(await(i.fold(s => Future.successful(s))(foldEC))) must equalTo(expected) 22 | } 23 | } 24 | 25 | def checkUnflattenResult[A, E](i: Iteratee[A, E], expected: Step[A, E]) = { 26 | await(i.unflatten) must equalTo(expected) 27 | } 28 | 29 | def checkImmediateFoldFailure[A, E](i: Iteratee[A, E]) = { 30 | mustExecute(1) { foldEC => 31 | val e = new Exception("exception") 32 | val result = ready(i.fold(_ => throw e)(foldEC)) 33 | result.value must equalTo(Some(Failure(e))) 34 | } 35 | } 36 | 37 | def checkFutureFoldFailure[A, E](i: Iteratee[A, E]) = { 38 | mustExecute(1, 1) { (foldEC, folderEC) => 39 | val e = new Exception("exception") 40 | val preparedFolderEC = folderEC.prepare() 41 | val result = ready(i.fold(_ => Future(throw e)(preparedFolderEC))(foldEC)) 42 | result.value must equalTo(Some(Failure(e))) 43 | } 44 | } 45 | 46 | def mustTranslate3To(x: Int)(f: Iteratee[Int, Int] => Iteratee[Int, Int]) = { 47 | await(f(Done(3)).unflatten) must equalTo(Step.Done(x, Input.Empty)) 48 | } 49 | 50 | "Flattened iteratees" should { 51 | 52 | val i = Iteratee.flatten(Future.successful(Done(1, Input.El("x")))) 53 | 54 | "delegate folding to their promised iteratee" in { 55 | checkFoldResult(i, Step.Done(1, Input.El("x"))) 56 | } 57 | 58 | "return errors in flattened failed future" in { 59 | val ex = new Exception("exception message") 60 | val flattenedFailedFuture = Iteratee.flatten(Future.failed(ex)) 61 | checkFoldTryResult[Nothing, Nothing](flattenedFailedFuture, Failure(ex)) 62 | } 63 | 64 | "return immediate fold errors in promise" in { 65 | checkImmediateFoldFailure(i) 66 | } 67 | 68 | "return future fold errors in promise" in { 69 | checkFutureFoldFailure(i) 70 | } 71 | 72 | "support unflattening their state" in { 73 | checkUnflattenResult(i, Step.Done(1, Input.El("x"))) 74 | } 75 | 76 | } 77 | 78 | "Done iteratees" should { 79 | 80 | val i = Done(1, Input.El("x")) 81 | 82 | "fold with their result and unused input" in { 83 | checkFoldResult(i, Step.Done(1, Input.El("x"))) 84 | } 85 | 86 | "return immediate fold errors in promise" in { 87 | checkImmediateFoldFailure(i) 88 | } 89 | 90 | "return future fold errors in promise" in { 91 | checkFutureFoldFailure(i) 92 | } 93 | 94 | "fold input with fold1" in { 95 | mustExecute(1) { foldEC => 96 | mustTranslate3To(5)(it => Iteratee.flatten(it.fold1( 97 | (a, i) => Future.successful(Done(a + 2, i)), 98 | _ => ???, 99 | (_, _) => ??? 100 | )(foldEC))) 101 | } 102 | } 103 | 104 | "fold input with pureFold" in { 105 | mustExecute(1) { foldEC => 106 | mustTranslate3To(9)(it => Iteratee.flatten(it.pureFold(_ => Done[Int, Int](9))(foldEC))) 107 | } 108 | } 109 | 110 | "fold input with pureFlatFold" in { 111 | mustExecute(1) { foldEC => 112 | mustTranslate3To(9)(_.pureFlatFold(_ => Done[Int, Int](9))(foldEC)) 113 | } 114 | } 115 | 116 | "fold input with flatFold0" in { 117 | mustExecute(1) { foldEC => 118 | mustTranslate3To(9)(_.flatFold0(_ => Future.successful(Done[Int, Int](9)))(foldEC)) 119 | } 120 | } 121 | 122 | "fold input with flatFold" in { 123 | mustExecute(1) { foldEC => 124 | mustTranslate3To(9)(_.flatFold( 125 | (_, _) => Future.successful(Done[Int, Int](9)), 126 | _ => ???, 127 | (_, _) => ??? 128 | )(foldEC)) 129 | } 130 | } 131 | 132 | "support unflattening their state" in { 133 | checkUnflattenResult(i, Step.Done(1, Input.El("x"))) 134 | } 135 | 136 | "flatMap directly to result when no remaining input" in { 137 | mustExecute(1) { flatMapEC => 138 | await(Done(3).flatMap((x: Int) => Done[Int, Int](x * 2))(flatMapEC).unflatten) must equalTo(Step.Done(6, Input.Empty)) 139 | } 140 | } 141 | 142 | "flatMap result and process remaining input with Done" in { 143 | mustExecute(1) { flatMapEC => 144 | await(Done(3, Input.El("remaining")).flatMap((x: Int) => Done[String, Int](x * 2))(flatMapEC).unflatten) must equalTo(Step.Done(6, Input.El("remaining"))) 145 | } 146 | } 147 | 148 | "flatMap result and process remaining input with Cont" in { 149 | mustExecute(1) { flatMapEC => 150 | await(Done(3, Input.El("remaining")).flatMap((x: Int) => Cont(in => Done[String, Int](x * 2, in)))(flatMapEC).unflatten) must equalTo(Step.Done(6, Input.El("remaining"))) 151 | } 152 | } 153 | 154 | "flatMap result and process remaining input with Error" in { 155 | mustExecute(1) { flatMapEC => 156 | await(Done(3, Input.El("remaining")).flatMap((x: Int) => Error("error", Input.El("bad")))(flatMapEC).unflatten) must equalTo(Step.Error("error", Input.El("bad"))) 157 | } 158 | } 159 | 160 | "flatMap result with flatMapM" in { 161 | mustExecute(1) { flatMapEC => 162 | mustTranslate3To(6)(_.flatMapM((x: Int) => Future.successful(Done[Int, Int](x * 2)))(flatMapEC)) 163 | } 164 | } 165 | 166 | "fold result with flatMapInput" in { 167 | mustExecute(1) { flatMapEC => 168 | mustTranslate3To(1)(_.flatMapInput(_ => Done(1))(flatMapEC)) 169 | } 170 | } 171 | 172 | "concatenate unused input with flatMapTraversable" in { 173 | mustExecute(1) { flatMapEC => 174 | await(Done(3, Input.El(List(1, 2))).flatMapTraversable(_ => Done[List[Int], Int](4, Input.El(List(3, 4))))( 175 | implicitly[List[Int] => scala.collection.TraversableLike[Int, List[Int]]], 176 | implicitly[scala.collection.generic.CanBuildFrom[List[Int], Int, List[Int]]], 177 | flatMapEC 178 | ).unflatten) must equalTo(Step.Done(4, Input.El(List(1, 2, 3, 4)))) 179 | } 180 | } 181 | 182 | } 183 | 184 | "Cont iteratees" should { 185 | 186 | val k: Input[String] => Iteratee[String, Int] = x => ??? 187 | val i = Cont(k) 188 | 189 | "fold with their continuation" in { 190 | checkFoldResult(i, Step.Cont(k)) 191 | } 192 | 193 | "return immediate fold errors in promise" in { 194 | checkImmediateFoldFailure(i) 195 | } 196 | 197 | "return future fold errors in promise" in { 198 | checkFutureFoldFailure(i) 199 | } 200 | 201 | "support unflattening their state" in { 202 | checkUnflattenResult(i, Step.Cont(k)) 203 | } 204 | 205 | "flatMap recursively" in { 206 | mustExecute(1) { flatMapEC => 207 | await(Iteratee.flatten(Cont[Int, Int](_ => Done(3)).flatMap((x: Int) => Done[Int, Int](x * 2))(flatMapEC).feed(Input.El(11))).unflatten) must equalTo(Step.Done(6, Input.Empty)) 208 | } 209 | } 210 | 211 | "not overflow the stack when called recursively" in { 212 | // Find how much recursion is needed to overflow the stack 213 | // on the current Java runtime. 214 | def overflows(n: Int): Boolean = { 215 | def recurseTimes(n: Int): Unit = { 216 | if (n == 0) () else identity(recurseTimes(n - 1)) 217 | } 218 | try { 219 | recurseTimes(n) 220 | false // Didn't overflow 221 | } catch { 222 | case _: StackOverflowError => true 223 | } 224 | } 225 | val overflowDepth: Int = (12 until 20).map(1 << _).find(overflows).get 226 | 227 | import ExecutionContext.Implicits.global 228 | val unitDone: Iteratee[Unit, Unit] = Done(()) 229 | val flatMapped: Iteratee[Unit, Unit] = (0 until overflowDepth).foldLeft[Iteratee[Unit, Unit]](Cont(_ => unitDone)) { 230 | case (it, _) => it.flatMap(_ => unitDone) 231 | } 232 | await(await(flatMapped.feed(Input.EOF)).unflatten) must equalTo(Step.Done((), Input.Empty)) 233 | } 234 | 235 | } 236 | 237 | "Error iteratees" should { 238 | 239 | val i = Error("msg", Input.El("x")) 240 | 241 | "fold with their message and the input that caused the error" in { 242 | checkFoldResult(i, Step.Error("msg", Input.El("x"))) 243 | } 244 | 245 | "return immediate fold errors in promise" in { 246 | checkImmediateFoldFailure(i) 247 | } 248 | 249 | "return future fold errors in promise" in { 250 | checkFutureFoldFailure(i) 251 | } 252 | 253 | "support unflattening their state" in { 254 | checkUnflattenResult(i, Step.Error("msg", Input.El("x"))) 255 | } 256 | 257 | "flatMap to an error" in { 258 | mustExecute(0) { flatMapEC => 259 | await(Error("msg", Input.El("bad")).flatMap((x: Int) => Done("done"))(flatMapEC).unflatten) must equalTo(Step.Error("msg", Input.El("bad"))) 260 | } 261 | } 262 | 263 | } 264 | 265 | "Iteratees fed multiple inputs" should { 266 | 267 | "map the final iteratee's result (with map)" in { 268 | mustExecute(4, 1) { (foldEC, mapEC) => 269 | await(Enumerator(1, 2, 3, 4) |>>> Iteratee.fold[Int, Int](0)(_ + _)(foldEC).map(_ * 2)(mapEC)) must equalTo(20) 270 | } 271 | } 272 | 273 | "map the final iteratee's result (with mapM)" in { 274 | mustExecute(4, 1) { (foldEC, mapEC) => 275 | await(Enumerator(1, 2, 3, 4) |>>> Iteratee.fold[Int, Int](0)(_ + _)(foldEC).mapM(x => Future.successful(x * 2))(mapEC)) must equalTo(20) 276 | } 277 | } 278 | 279 | } 280 | 281 | "Iteratee.fold" should { 282 | 283 | "fold input" in { 284 | mustExecute(4) { foldEC => 285 | await(Enumerator(1, 2, 3, 4) |>>> Iteratee.fold[Int, Int](0)(_ + _)(foldEC)) must equalTo(10) 286 | } 287 | } 288 | 289 | } 290 | 291 | "Iteratee.foldM" should { 292 | 293 | "fold input" in { 294 | mustExecute(4) { foldEC => 295 | await(Enumerator(1, 2, 3, 4) |>>> Iteratee.foldM[Int, Int](0)((x, y) => Future.successful(x + y))(foldEC)) must equalTo(10) 296 | } 297 | } 298 | 299 | } 300 | 301 | "Iteratee.fold2" should { 302 | 303 | "fold input" in { 304 | mustExecute(4) { foldEC => 305 | val folder = (x: Int, y: Int) => Future.successful((x + y, false)) 306 | await(Enumerator(1, 2, 3, 4) |>>> Iteratee.fold2[Int, Int](0)(folder)(foldEC)) must equalTo(10) 307 | } 308 | } 309 | 310 | "fold input, stopping early" in { 311 | mustExecute(3) { foldEC => 312 | val folder = (x: Int, y: Int) => Future.successful((x + y, y > 2)) 313 | await(Enumerator(1, 2, 3, 4) |>>> Iteratee.fold2[Int, Int](0)(folder)(foldEC)) must equalTo(6) 314 | } 315 | } 316 | 317 | } 318 | 319 | "Iteratee.foldM" should { 320 | 321 | "fold input" in { 322 | mustExecute(4) { foldEC => 323 | await(Enumerator(1, 2, 3, 4) |>>> Iteratee.fold1[Int, Int](Future.successful(0))((x, y) => Future.successful(x + y))(foldEC)) must equalTo(10) 324 | } 325 | } 326 | 327 | } 328 | 329 | "Iteratee.recover" should { 330 | 331 | val expected = "expected" 332 | val unexpected = "should not be returned" 333 | 334 | "do nothing on a Done iteratee" in { 335 | mustExecute(1) { implicit foldEC => 336 | val it = done(expected).recover { case t: Throwable => unexpected } 337 | val actual = await(Enumerator(unexpected) |>>> it) 338 | actual must equalTo(expected) 339 | } 340 | } 341 | 342 | "do nothing on an eventually Done iteratee" in { 343 | mustExecute(1) { implicit foldEC => 344 | val it = delayed(done(expected)).recover { case t: Throwable => unexpected } 345 | val actual = await(Enumerator(unexpected) |>>> it) 346 | actual must equalTo(expected) 347 | } 348 | } 349 | 350 | "recover with the expected fallback value from an Error iteratee" in { 351 | mustExecute(2) { implicit foldEC => 352 | val it = error(unexpected).recover { case t: Throwable => expected } 353 | val actual = await(Enumerator(unexpected) |>>> it) 354 | actual must equalTo(expected) 355 | } 356 | } 357 | 358 | "leave the Error iteratee unchanged if the Exception type doesn't match the partial function" in { 359 | mustExecute(1) { implicit foldEC => 360 | val it = error(expected).recover { case t: IllegalArgumentException => unexpected } 361 | val actual = await((Enumerator(unexpected) |>>> it).failed) 362 | actual.getMessage must equalTo(expected) 363 | } 364 | } 365 | 366 | "recover with the expected fallback value from an eventually Error iteratee" in { 367 | mustExecute(2) { implicit foldEC => 368 | val it = delayed(error(unexpected)).recover { case t: Throwable => expected } 369 | val actual = await(Enumerator(unexpected) |>>> it) 370 | actual must equalTo(expected) 371 | } 372 | } 373 | 374 | "recover with the expected fallback value from an iteratee that eventually throws an exception" in { 375 | mustExecute(2) { implicit foldEC => 376 | val it = delayed(throw new RuntimeException(unexpected)).recover { case t: Throwable => expected } 377 | val actual = await(Enumerator(unexpected) |>>> it) 378 | actual must equalTo(expected) 379 | } 380 | } 381 | 382 | "do nothing on a Cont iteratee that becomes Done with input" in { 383 | mustExecute(2) { implicit foldEC => 384 | val it = cont(input => done(input)).recover { case t: Throwable => unexpected } 385 | val actual = await(Enumerator(expected) |>>> it) 386 | actual must equalTo(expected) 387 | } 388 | } 389 | 390 | "do nothing on an eventually Cont iteratee that becomes Done with input" in { 391 | mustExecute(2) { implicit foldEC => 392 | val it = delayed(cont(input => done(input))).recover { case t: Throwable => unexpected } 393 | val actual = await(Enumerator(expected) |>>> it) 394 | actual must equalTo(expected) 395 | } 396 | } 397 | 398 | "do nothing on a Cont iteratee that eventually becomes Done with input" in { 399 | mustExecute(2) { implicit foldEC => 400 | val it = cont(input => delayed(done(input))).recover { case t: Throwable => unexpected } 401 | val actual = await(Enumerator(expected) |>>> it) 402 | actual must equalTo(expected) 403 | } 404 | } 405 | 406 | "do nothing on an Cont iteratee that eventually becomes Done with input after several steps" in { 407 | mustExecute(4) { implicit foldEC => 408 | val it = delayed( 409 | cont(input1 => delayed( 410 | cont(input2 => delayed( 411 | cont(input3 => delayed( 412 | done(input1 + input2 + input3) 413 | )) 414 | )) 415 | )) 416 | ).recover { case t: Throwable => unexpected } 417 | val actual = await(Enumerator(expected, expected, expected) |>>> it) 418 | actual must equalTo(expected * 3) 419 | } 420 | } 421 | 422 | "recover with the expected fallback value from a Cont iteratee that eventually becomes an Error iteratee after several steps" in { 423 | mustExecute(5) { implicit foldEC => 424 | val it = delayed( 425 | cont(input1 => delayed( 426 | cont(input2 => delayed( 427 | cont(input3 => delayed( 428 | error(input1 + input2 + input3) 429 | )) 430 | )) 431 | )) 432 | ).recover { case t: Throwable => expected } 433 | val actual = await(Enumerator(unexpected, unexpected, unexpected) |>>> it) 434 | actual must equalTo(expected) 435 | } 436 | } 437 | } 438 | 439 | "Iteratee.recoverM" should { 440 | 441 | val expected = "expected" 442 | val unexpected = "should not be returned" 443 | 444 | "do nothing on a Done iteratee" in { 445 | mustExecute(1) { implicit foldEC => 446 | val it = done(expected).recoverM { case t: Throwable => Future.successful(unexpected) }(foldEC) 447 | val actual = await(Enumerator(unexpected) |>>> it) 448 | actual must equalTo(expected) 449 | } 450 | } 451 | 452 | "do nothing on a Done iteratee even if the recover block gets a failed Future" in { 453 | mustExecute(1) { implicit foldEC => 454 | val it = done(expected).recoverM { case t: Throwable => Future.failed(new RuntimeException(unexpected)) }(foldEC) 455 | val actual = await(Enumerator(unexpected) |>>> it) 456 | actual must equalTo(expected) 457 | } 458 | } 459 | 460 | "recover with the expected fallback Future from an Error iteratee" in { 461 | mustExecute(2) { implicit foldEC => 462 | val it = error(unexpected).recoverM { 463 | case t: Throwable => Future.successful(expected) 464 | }(foldEC) 465 | val actual = await(Enumerator(unexpected) |>>> it) 466 | actual must equalTo(expected) 467 | } 468 | } 469 | 470 | "leave the Error iteratee unchanged if the Exception type doesn't match the partial function" in { 471 | mustExecute(1) { implicit foldEC => 472 | val it = error(expected).recoverM { case t: IllegalArgumentException => Future.successful(unexpected) }(foldEC) 473 | val actual = await((Enumerator(unexpected) |>>> it).failed) 474 | actual.getMessage must equalTo(expected) 475 | } 476 | } 477 | 478 | "return a failed Future if you try to recover from an Error iteratee with a failed Future" in { 479 | val exCount = { 480 | if (util.Properties.versionNumberString startsWith "2.12") 1 481 | else 2 482 | 483 | /* 484 | With Scala <= 2.11, `Future(..)` call `prepare()` on the given 485 | `ExecutionContext` (see https://github.com/scala/scala/blob/2.11.x/src/library/scala/concurrent/Future.scala#L494 and https://github.com/scala/scala/blob/2.11.x/src/library/scala/concurrent/impl/Future.scala#L31 ), 486 | whereas since Scala 2.12 there is no longer such call (see https://github.com/scala/scala/blob/2.12.x/src/library/scala/concurrent/Future.scala#L652 ). 487 | 488 | Consequently there, one `Future` created with `StepIteratee.pureFold` 489 | is no longer executed with the prepared `TestExecutionContext`, 490 | and so the associated count is not incremented. 491 | */ 492 | } 493 | 494 | mustExecute(exCount) { implicit foldEC => 495 | val exception = new RuntimeException(expected) 496 | val it = error(unexpected).recoverM { 497 | case t: Throwable => Future.failed(exception) 498 | } 499 | val actual = await((Enumerator(unexpected) |>>> it).failed) 500 | actual must equalTo(exception) 501 | } 502 | } 503 | } 504 | 505 | "Iteratee.recoverWith" should { 506 | val expected = "expected" 507 | val unexpected = "should not be returned" 508 | 509 | "do nothing on a Done iteratee" in { 510 | mustExecute(1) { implicit foldEC => 511 | val it = done(expected).recoverWith { case t: Throwable => done(unexpected) } 512 | val actual = await(Enumerator(unexpected) |>>> it) 513 | actual must equalTo(expected) 514 | } 515 | } 516 | 517 | "do nothing on a Done iteratee even if the recover block gets an error Iteratee" in { 518 | mustExecute(1) { implicit foldEC => 519 | val it = done(expected).recoverWith { case t: Throwable => error(unexpected) } 520 | val actual = await(Enumerator(unexpected) |>>> it) 521 | actual must equalTo(expected) 522 | } 523 | } 524 | 525 | "recover with the expected fallback Iteratee from an Error iteratee" in { 526 | mustExecute(1) { implicit foldEC => 527 | val it = error(unexpected).recoverWith { case t: Throwable => done(expected) } 528 | val actual = await(Enumerator(unexpected) |>>> it) 529 | actual must equalTo(expected) 530 | } 531 | } 532 | 533 | "leave the Error iteratee unchanged if the Exception type doesn't match the partial function" in { 534 | mustExecute(1) { implicit foldEC => 535 | val it = error(expected).recoverWith { case t: IllegalArgumentException => done(unexpected) } 536 | val actual = await((Enumerator(unexpected) |>>> it).failed) 537 | actual.getMessage must equalTo(expected) 538 | } 539 | } 540 | 541 | "return a failed Future if you try to recover from an Error iteratee with an Error iteratee" in { 542 | mustExecute(1) { implicit foldEC => 543 | val it = error(unexpected).recoverWith { case t: Throwable => error(expected) } 544 | val actual = await((Enumerator(unexpected) |>>> it).failed) 545 | actual.getMessage must equalTo(expected) 546 | } 547 | } 548 | } 549 | 550 | "Iteratee.consume" should { 551 | 552 | "return its concatenated input" in { 553 | val s = List(List(1, 2), List(3), List(4, 5)) 554 | val r = List(1, 2, 3, 4, 5) 555 | await(Enumerator.enumerateSeq1(s) |>>> Iteratee.consume[List[Int]]()) must equalTo(r) 556 | } 557 | 558 | } 559 | 560 | "Iteratee.getChunks" should { 561 | 562 | "return its input as a list" in { 563 | val s = List(1, 2, 3, 4, 5) 564 | await(Enumerator.enumerateSeq1(s) |>>> Iteratee.getChunks[Int]) must equalTo(s) 565 | } 566 | 567 | } 568 | 569 | "Iteratee.takeUpTo" should { 570 | 571 | def takenAndNotTaken[E](n: Int): Iteratee[E, (Seq[E], Seq[E])] = { 572 | import ExecutionContext.Implicits.global 573 | for { 574 | seq1 <- Iteratee.takeUpTo(n) 575 | seq2 <- Iteratee.getChunks 576 | } yield (seq1, seq2) 577 | } 578 | 579 | "take 0 elements from 0" in { 580 | await(Enumerator() |>>> takenAndNotTaken(0)) must equalTo((Seq(), Seq())) 581 | } 582 | 583 | "take 0 elements from 0 when asked for 2" in { 584 | await(Enumerator() |>>> takenAndNotTaken(2)) must equalTo((Seq(), Seq())) 585 | } 586 | 587 | "take 1 element from 2" in { 588 | await(Enumerator(1, 2) |>>> takenAndNotTaken(1)) must equalTo((Seq(1), Seq(2))) 589 | } 590 | 591 | "take 2 elements from 2" in { 592 | await(Enumerator(1, 2) |>>> takenAndNotTaken(2)) must equalTo((Seq(1, 2), Seq())) 593 | } 594 | 595 | "take 2 elements from 2 when asked for 3" in { 596 | await(Enumerator(1, 2) |>>> takenAndNotTaken(3)) must equalTo((Seq(1, 2), Seq())) 597 | } 598 | 599 | "skip Input.Empty when taking elements" in { 600 | val enum = Enumerator(1, 2) >>> Enumerator.enumInput(Input.Empty) >>> Enumerator(3, 4) 601 | await(enum |>>> takenAndNotTaken(3)) must equalTo((Seq(1, 2, 3), Seq(4))) 602 | } 603 | 604 | } 605 | 606 | "Iteratee.isEmpty" should { 607 | 608 | def isEmptyThenRest[E]: Iteratee[E, (Boolean, Seq[E])] = { 609 | import ExecutionContext.Implicits.global 610 | for { 611 | empty <- Iteratee.isEmpty 612 | seq <- Iteratee.getChunks 613 | } yield (empty, seq) 614 | } 615 | 616 | "be true for a stream with only EOF" in { 617 | await(Enumerator() |>>> isEmptyThenRest) must equalTo((true, Seq())) 618 | } 619 | 620 | "be true for a stream with Empty and EOF" in { 621 | val enum = Enumerator.enumInput(Input.Empty) >>> Enumerator.eof 622 | await(enum |>>> isEmptyThenRest) must equalTo((true, Seq())) 623 | } 624 | 625 | "be false for a stream with one element" in { 626 | await(Enumerator(1) |>>> isEmptyThenRest) must equalTo((false, Seq(1))) 627 | } 628 | 629 | "be false for a stream with two elements" in { 630 | await(Enumerator(1, 2) |>>> isEmptyThenRest) must equalTo((false, Seq(1, 2))) 631 | } 632 | 633 | "be false for a stream with empty and element inputs" in { 634 | val enum = Enumerator.enumInput(Input.Empty) >>> Enumerator(1, 2) 635 | await(enum |>>> isEmptyThenRest) must equalTo((false, Seq(1, 2))) 636 | } 637 | 638 | } 639 | 640 | "Iteratee.takeUpTo and Iteratee.isEmpty" should { 641 | 642 | def process[E](n: Int): Iteratee[E, (Seq[E], Boolean, Seq[E])] = { 643 | import ExecutionContext.Implicits.global 644 | for { 645 | seq1 <- Iteratee.takeUpTo(n) 646 | emptyAfterSeq1 <- Iteratee.isEmpty 647 | seq2 <- Iteratee.getChunks 648 | } yield (seq1, emptyAfterSeq1, seq2) 649 | } 650 | 651 | "take 0 elements and be empty from 0" in { 652 | await(Enumerator() |>>> process(0)) must equalTo((Seq(), true, Seq())) 653 | } 654 | 655 | "take 0 elements from 0 and be empty when asked for 2" in { 656 | await(Enumerator() |>>> process(2)) must equalTo((Seq(), true, Seq())) 657 | } 658 | 659 | "take 1 element and not be empty from 2" in { 660 | await(Enumerator(1, 2) |>>> process(1)) must equalTo((Seq(1), false, Seq(2))) 661 | } 662 | 663 | "take 2 elements and be empty from 2" in { 664 | await(Enumerator(1, 2) |>>> process(2)) must equalTo((Seq(1, 2), true, Seq())) 665 | } 666 | 667 | "take 2 elements and be empty from 2 when asked for 3" in { 668 | await(Enumerator(1, 2) |>>> process(3)) must equalTo((Seq(1, 2), true, Seq())) 669 | } 670 | 671 | "skip Input.Empty when taking elements" in { 672 | val enum = Enumerator(1, 2) >>> Enumerator.enumInput(Input.Empty) >>> Enumerator(3, 4) 673 | await(enum |>>> process(3)) must equalTo((Seq(1, 2, 3), false, Seq(4))) 674 | } 675 | 676 | } 677 | 678 | "Iteratee.ignore" should { 679 | 680 | "never throw an OutOfMemoryError when consuming large input" in { 681 | // Work out how many arrays we'd need to create to trigger an OutOfMemoryError 682 | val arraySize = 1000000 683 | val tooManyArrays = (Runtime.getRuntime.maxMemory / arraySize).toInt + 1 684 | val iterator = Iterator.range(0, tooManyArrays).map(_ => new Array[Byte](arraySize)) 685 | import play.api.libs.iteratee.Execution.Implicits.defaultExecutionContext 686 | await(Enumerator.enumerate(iterator) |>>> Iteratee.ignore[Array[Byte]]) must_== (()) 687 | } 688 | 689 | } 690 | 691 | } 692 | --------------------------------------------------------------------------------