├── .gitignore ├── .idea └── compiler.xml ├── .travis.yml ├── LICENSE ├── README.md ├── RELEASE_NOTES.md ├── build.sbt ├── project ├── build.properties └── plugins.sbt └── src ├── main ├── resources │ └── log4j.properties └── scala │ ├── es │ └── ucm │ │ └── fdi │ │ └── sscheck │ │ ├── gen │ │ ├── Batch.scala │ │ ├── BatchGen.scala │ │ ├── Buildables.scala │ │ ├── PDStream.scala │ │ ├── PDStreamGen.scala │ │ ├── RDDGen.scala │ │ ├── ReGen.scala │ │ └── UtilsGen.scala │ │ ├── matcher │ │ └── specs2 │ │ │ └── package.scala │ │ ├── package.scala │ │ ├── prop │ │ ├── UtilsProp.scala │ │ └── tl │ │ │ ├── DStreamTLProperty.scala │ │ │ └── Formula.scala │ │ └── spark │ │ ├── SharedSparkContext.scala │ │ ├── SharedSparkContextBeforeAfterAll.scala │ │ └── streaming │ │ ├── SharedStreamingContext.scala │ │ ├── SharedStreamingContextBeforeAfterEach.scala │ │ ├── StreamingContextUtils.scala │ │ ├── TestInputStream.scala │ │ └── Utils.scala │ └── org │ └── apache │ └── spark │ └── streaming │ └── dstream │ └── SscheckFriendlyInputDStream.scala └── test └── scala └── es └── ucm └── fdi └── sscheck ├── gen ├── PDStreamGenTest.scala ├── ReGenTest.scala ├── TLGenTest.scala └── UtilsGenTest.scala ├── prop └── tl │ └── FormulaTest.scala └── spark ├── SharedSparkContextBeforeAfterAllTest.scala ├── demo ├── StreamingFormulaDemo1.scala ├── StreamingFormulaDemo2.scala ├── StreamingFormulaManyArgsDemo.scala └── StreamingFormulaQuantDemo.scala ├── simple └── SimpleStreamingFormulas.scala └── streaming ├── ScalaCheckStreamingTest.scala └── SharedStreamingContextBeforeAfterEachTest.scala /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | .cache 6 | .history 7 | .lib/ 8 | dist/* 9 | target/ 10 | lib_managed/ 11 | src_managed/ 12 | project/boot/ 13 | project/plugins/project/ 14 | 15 | # Scala-IDE specific 16 | .scala_dependencies 17 | .worksheet 18 | 19 | .classpath 20 | .project 21 | -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.10.6 4 | - 2.11.8 5 | jdk: 6 | - oraclejdk7 7 | script: 8 | - sbt ++$TRAVIS_SCALA_VERSION test 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sscheck 2 | Utilities for using ScalaCheck with Spark and Spark Streaming, based on Specs2 3 | 4 | [Jenkins](https://juanrhcubox.duckdns.org:8080/) 5 | 6 | Use linear temporal logic to write ScalaCheck properties for Spark Streaming programs, see the [**Quickstart**](https://github.com/juanrh/sscheck/wiki/Quickstart) for details. See also 7 | 8 | * **scaladoc** 9 | - [scala 2.10](http://juanrh.github.io/doc/sscheck/scala-2.10/api) 10 | - [scala 2.11](http://juanrh.github.io/doc/sscheck/scala-2.11/api) 11 | * sbt dependency 12 | 13 | ```scala 14 | lazy val sscheckVersion = "0.3.2" 15 | libraryDependencies += "es.ucm.fdi" %% "sscheck" % sscheckVersion 16 | resolvers += Resolver.bintrayRepo("juanrh", "maven") 17 | ``` 18 | See latest version in [bintray](https://bintray.com/juanrh/maven/sscheck/view) 19 | 20 | # Acknowledgements 21 | This work has been partially supported by MICINN Spanish project StrongSoft (TIN2012-39391-C04-04), by the 22 | Spanish MINECO project CAVI-ART (TIN2013-44742-C4-3-R), and by the Comunidad de Madrid project [N-Greens Software-CM](http://n-greens-cm.org/) (S2013/ICE-2731). 23 | 24 | Some parts of this code are based on or have been taken from [Spark Testing Base](https://github.com/holdenk/spark-testing-base) by Holden Karau 25 | -------------------------------------------------------------------------------- /RELEASE_NOTES.md: -------------------------------------------------------------------------------- 1 | # sscheck 0.3.2 2 | Minor maintenance release 3 | 4 | * replace `Now` by `BindNext`: #53 and move overloads of `Formula.now` that don't generate a result immediately to overloads of `Formula.next`, so the DSL is more clear 5 | * add `beEqualAsSetTo` RDD matcher 6 | * minor scaladoc fixes 7 | * rename overloads of forAllDStream for more than 1 argument to avoid having to specify the type parameters in all usages 8 | 9 | # sscheck 0.3.1 10 | Bug fixes 11 | 12 | * Ensure we don't try to `cache()` generated batches more than once, which throws a Spark exception. 13 | * Fix formatting of print of generated DStreams. 14 | * Bug fix: the state of the formula has to be updated even for empty batches, because the state and output of stateful operators might be updated anyway (e.g. `PairDStreamFunctions.reduceByKeyAndWindow`) 15 | * Fix base case of safeWordLength [#51](https://github.com/juanrh/sscheck/issues/51). 16 | 17 | # sscheck 0.3.0 18 | First order quantifiers, many DStreams in properties, and performance improvements 19 | 20 | * Add first order quantifiers on letters and optionally use atoms time in now, in the style of TPTL [#46](https://github.com/juanrh/sscheck/pull/46). 21 | * Support several DStreams in properties: 1 input 1 derived, 2 input 1 derived, 1 input 2 derived, 2 input 2 derived [#16](https://github.com/juanrh/sscheck/pull/16). 22 | * Lazy next form, so the system might scales to complex formulas or with long timeouts [#25](https://github.com/juanrh/sscheck/pull/25). Only Next is lazy, which is enough combined with a nested next formula generation. 23 | * Simplify concurrency: synchronization to wait for the Streaming Context to stop; improve usage of parallel collections. 24 | * Fix safeWordLength to return an Option, as it cannot be always statically computed due to first order quantifiers. 25 | * Update to Spark 1.6.2 to get rid of some bugs. 26 | 27 | # sscheck 0.2.4 28 | Added cross Scala version compatibility, 2.10 and 2.11, see [#42](https://github.com/juanrh/sscheck/pull/42) 29 | 30 | # sscheck 0.2.3 31 | Bug fixing and code cleanup 32 | 33 | * Remove dependency to spark-testing-base and multisets in order to fix [#36](https://github.com/juanrh/sscheck/issues/36) 34 | * Remove unused code from preliminary approaches that were later discarded 35 | * Update Spark to version 1.6.0 36 | 37 | # sscheck 0.2.2 38 | Update Spark to version 1.6.0 39 | 40 | # sscheck 0.2.1 41 | Bug fix implementation of temporal properties. Uses of `DStreamProp.forAll` combining with extending the trait `SharedStreamingContextBeforeAfterEach` should be replaced by extending the trait `DStreamTLProperty` and calling `forAllDStream`. This solves: 42 | 43 | * Execution of a test case is now independent from others, as a new streaming context is created for each test case. This is particularly important for stateful DStream transformations 44 | * Replaced uses `DynSingleSeqQueueInputDStream` by `TestInputStream` from [spark-testing-base](https://github.com/holdenk/spark-testing-base), which implements checkpointing correctly 45 | * fixed [#32](https://github.com/juanrh/sscheck/issues/32) and [#31](https://github.com/juanrh/sscheck/issues/31) 46 | 47 | # sscheck 0.2.0 - Temporal logic generators and properties 48 | First implementation of a temporal logic for testing Spark Streaming with ScalaCheck. This allows to define: 49 | 50 | * Generators for DStream defined by temporal logic formulas. 51 | * Properties for testing functions over DStream, using a ScalaCheck generator, and a propositional temporal logic formula as the assertion. DStreamProp.forAll defines a property that is universally quantified over the generated test cases. 52 | 53 | # sscheck 0.1.0 - RDD generators 54 | 55 | Shared Spark context for ScalaCheck generators based on parallelization of lists, through the integration of ScalaCheck and specs2 56 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "sscheck" 2 | 3 | organization := "es.ucm.fdi" 4 | 5 | version := "0.3.3-SNAPSHOT" 6 | 7 | scalaVersion := "2.11.8" 8 | 9 | crossScalaVersions := Seq("2.10.6", "2.11.8") 10 | 11 | licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0")) 12 | 13 | bintrayPackageLabels := Seq("testing") 14 | 15 | bintrayVcsUrl := Some("git@github.com:juanrh/sscheck.git") 16 | 17 | lazy val sparkVersion = "1.6.2" 18 | 19 | lazy val specs2Version = "3.8.4" 20 | 21 | // Use `sbt doc` to generate scaladoc, more on chapter 14.8 of "Scala Cookbook" 22 | 23 | // show all the warnings: http://stackoverflow.com/questions/9415962/how-to-see-all-the-warnings-in-sbt-0-11 24 | scalacOptions ++= Seq("-feature", "-unchecked", "-deprecation") 25 | 26 | // if parallel test execution is not disabled and several test suites using 27 | // SparkContext (even through SharedSparkContext) are running then tests fail randomly 28 | parallelExecution := false 29 | 30 | // Could be interesting at some point 31 | // resourceDirectory in Compile := baseDirectory.value / "main/resources" 32 | // resourceDirectory in Test := baseDirectory.value / "main/resources" 33 | 34 | // Configure sbt to add the resources path to the eclipse project http://stackoverflow.com/questions/14060131/access-configuration-resources-in-scala-ide 35 | // This is critical so log4j.properties is found by eclipse 36 | EclipseKeys.createSrc := EclipseCreateSrc.Default + EclipseCreateSrc.Resource 37 | 38 | // Spark 39 | libraryDependencies += "org.apache.spark" %% "spark-core" % sparkVersion 40 | 41 | libraryDependencies += "org.apache.spark" %% "spark-streaming" % sparkVersion 42 | 43 | // additional libraries: NOTE as we are writing a testing library they should also be available for main 44 | libraryDependencies += "org.scalatest" %% "scalatest" % "2.2.6" 45 | 46 | libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.13.1" 47 | 48 | libraryDependencies += "org.specs2" %% "specs2-core" % specs2Version 49 | 50 | libraryDependencies += "org.specs2" %% "specs2-scalacheck" % specs2Version 51 | 52 | libraryDependencies += "org.specs2" %% "specs2-matcher-extra" % specs2Version 53 | 54 | libraryDependencies += "org.specs2" %% "specs2-junit" % specs2Version 55 | 56 | libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.21" 57 | 58 | resolvers ++= Seq( 59 | "MVN Repository.com" at "http://mvnrepository.com/artifact/", 60 | "scalaz-bintray" at "http://dl.bintray.com/scalaz/releases", 61 | "Spark Packages Repo" at "http://dl.bintray.com/spark-packages/maven" 62 | ) 63 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.17 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // Eclipse support 2 | // resolvers += Classpaths.typesafeResolver 3 | 4 | addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "2.5.0") 5 | 6 | addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0") 7 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set everything to be logged to the console 2 | # use this for the repo so travis log is not bigger than 4MB 3 | log4j.rootCategory=WARN, console 4 | # standard detail 5 | # log4j.rootCategory=INFO, console 6 | # more detail 7 | # log4j.rootCategory=DEBUG, console 8 | log4j.appender.console=org.apache.log4j.ConsoleAppender 9 | log4j.appender.console.target=System.err 10 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 11 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 12 | 13 | # Settings to quiet third party logs that are too verbose 14 | log4j.logger.org.spark-project.jetty=WARN 15 | log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR 16 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO 17 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO 18 | -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/Batch.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import scala.language.implicitConversions 4 | 5 | object Batch { 6 | def empty[A] : Batch[A] = new Batch(points = List():_*) 7 | 8 | implicit def seq2batch[A](seq : Seq[A]) : Batch[A] = Batch(seq:_*) 9 | } 10 | 11 | /** Objects of this class represent batches of elements 12 | * in a discrete data stream 13 | * */ 14 | case class Batch[A](points : A*) extends Seq[A] { 15 | override def toSeq : Seq[A] = points 16 | 17 | override def apply(idx : Int) = points.apply(idx) 18 | override def iterator = points.iterator 19 | override def length = points.length 20 | 21 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/BatchGen.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.Gen 4 | import org.scalacheck.Shrink 5 | import org.scalacheck.Shrink.shrink 6 | import scala.language.implicitConversions 7 | import es.ucm.fdi.sscheck.prop.tl.Timeout 8 | 9 | /** Implicit conversions removed from BatchGen companion. This is needed 10 | * because we cannot populate the name space with standard names for HO generators 11 | * like of() or ofN(), but we also want to have these implicit conversions available 12 | */ 13 | object BatchGenConversions { 14 | implicit def genBatch2BatchGen[A](bg : Gen[Batch[A]]) : BatchGen[A] = BatchGen(bg) 15 | 16 | implicit def batchGen2seqGen[A](g : Gen[Batch[A]]) : Gen[Seq[A]] = g.map(_.toSeq) 17 | implicit def seqGen2batchGen[A](g : Gen[Seq[A]]) : Gen[Batch[A]] = g.map(Batch(_:_*)) 18 | } 19 | 20 | /** 21 | * Generators for Batch, a class for representing batches of elements 22 | * in a discrete data stream. This is a local representation, but you 23 | * can use `import es.ucm.fdi.sscheck.gen.RDDGen._` to get an automatic 24 | * conversion from `Gen[Batch[A]]` to `Gen[RDD[A]]` 25 | * 26 | * All the temporal generators defined in this object are sized generators, but the size parameters 27 | * only affects the number of batches in the output DStream, not the size of the batches. 28 | * The size of the batches can be changed by using Gen.resize in the definition of the batch 29 | * generator that is passed to these HO generators 30 | * 31 | * On the other hand for generators of arbitrary DStreams like Gen.resize(5, arbitrary[DStream[Int]]) 32 | * Gen.resize has effect, both in the number of batches and in the size of the batches, as those 33 | * arbitrary instances are based on arbitrary for lists 34 | * 35 | * TODO That should be tested 36 | // by using Gen.resize(5, TLGen.always(Batch(List(0, 1)))) (constant batch size) 37 | // and then in Gen.resize(3, TLGen.always(BatchGen.ofNtoM(1, 5, Gen.choose(0, 3)))) 38 | // the point is that we always have 3 batches, but Gen.resize doesn't affect the batch 39 | // size but only the number of batches 40 | // 41 | * 42 | * */ 43 | object BatchGen { 44 | import BatchGenConversions._ 45 | 46 | def apply[A](bg : Gen[Batch[A]]) : BatchGen[A] = new BatchGen(bg) 47 | 48 | /** Shrink function for Batch 49 | * 50 | * Although a Batch is a Seq, we need this simple definition of 51 | * shrink as the default doesn't work properly 52 | * 53 | * Note shrinking assumes no particular structure on the batches, but 54 | * that is no problem as there is no temporal structure in the 55 | * Batch generators: note generators like BatchGen.now() are in 56 | * fact generators of PDStream 57 | * */ 58 | implicit def shrinkBatch[A] : Shrink[Batch[A]] = Shrink( batch => 59 | // unwrap the underlying Seq, shrink the Seq, and rewrap 60 | shrink(batch.toSeq).map(Batch(_:_*)) 61 | ) 62 | 63 | /** @return a generator of Batch that generates its elements from g 64 | * */ 65 | def of[T](g : => Gen[T]) : Gen[Batch[T]] = { 66 | import Buildables.buildableBatch 67 | Gen.containerOf[Batch, T](g) 68 | } 69 | 70 | /** @return a generator of Batch that generates its elements from g 71 | * */ 72 | def ofN[T](n : Int, g : Gen[T]) : Gen[Batch[T]] = { 73 | import Buildables.buildableBatch 74 | Gen.containerOfN[Batch, T](n, g) 75 | } 76 | 77 | /** @return a generator of Batch that generates its elements from g 78 | * */ 79 | def ofNtoM[T](n : Int, m : Int, g : Gen[T]) : Gen[Batch[T]] = { 80 | import Buildables.buildableBatch 81 | UtilsGen.containerOfNtoM[Batch, T](n, m, g) 82 | } 83 | 84 | /** @return a dstream generator with exactly bg as the only batch 85 | * */ 86 | def now[A](bg : Gen[Batch[A]]) : Gen[PDStream[A]] = { 87 | bg.map(PDStream(_)) 88 | } 89 | 90 | /** @return a dstream generator that has exactly two batches, the first one 91 | * is emptyBatch, and the second one is bg 92 | * */ 93 | def next[A](bg : Gen[Batch[A]]) : Gen[PDStream[A]] = { 94 | PDStreamGen.next(now(bg)) 95 | } 96 | 97 | /** @return a dstream generator that has n empty batches followed by bg 98 | * */ 99 | def laterN[A](n : Int, bg : Gen[Batch[A]]) : Gen[PDStream[A]] = { 100 | PDStreamGen.laterN(n, now(bg)) 101 | } 102 | 103 | /** 104 | * @return a generator of DStream that repeats batches generated by gb1 until 105 | * a batch generated by bg2 is produced 106 | * 107 | * Note bg2 always occurs eventually, so this is not a weak until. When bg2 occurs 108 | * then the generated DStream finishes 109 | * This generator is exclusive in the sense that when bg2 finally happens then bg1 110 | * doesn't occur. 111 | * 112 | * Example: 113 | * 114 | * Gen.resize(10, 115 | always(BatchGen.ofN(2, 3)) 116 | + until(BatchGen.ofN(2, 0), BatchGen.ofN(2, 1)) 117 | ). sample //> res34: Option[es.ucm.fdi.sscheck.DStream[Int]] = Some(DStream(Batch(3, 3, 0 118 | //| , 0), Batch(3, 3, 0, 0), Batch(3, 3, 1, 1), Batch(3, 3), Batch(3, 3), Batch 119 | //| (3, 3), Batch(3, 3), Batch(3, 3), Batch(3, 3), Batch(3, 3))) 120 | * */ 121 | def until[A](bg1 : Gen[Batch[A]], bg2 : Gen[Batch[A]]) : Gen[PDStream[A]] = 122 | PDStreamGen.until(now(bg1), now(bg2)) 123 | def until[A](bg1 : Gen[Batch[A]], bg2 : Gen[Batch[A]], t : Timeout) : Gen[PDStream[A]] = 124 | PDStreamGen.until(now(bg1), now(bg2), t) 125 | 126 | def eventually[A](bg : Gen[Batch[A]]) : Gen[PDStream[A]] = 127 | // note true is represented here as Batch.empty, as true 128 | // asks for nothing to happen 129 | // until(Batch.empty : Batch[A], bg) 130 | PDStreamGen.eventually(now(bg)) 131 | def eventually[A](bg : Gen[Batch[A]], t : Timeout) : Gen[PDStream[A]] = 132 | PDStreamGen.eventually(now(bg), t) 133 | 134 | def always[A](bg : Gen[Batch[A]]) : Gen[PDStream[A]] = 135 | PDStreamGen.always(now(bg)) 136 | def always[A](bg : Gen[Batch[A]], t : Timeout) : Gen[PDStream[A]] = 137 | PDStreamGen.always(now(bg), t) 138 | 139 | /** 140 | * Generator for the weak version of LTL release operator: either bg2 141 | * happens forever, or it happens until bg1 happens, including the 142 | * moment when bg1 happens 143 | * */ 144 | def release[A](bg1 : Gen[Batch[A]], bg2 : Gen[Batch[A]]) : Gen[PDStream[A]] = 145 | PDStreamGen.release(now(bg1), now(bg2)) 146 | def release[A](bg1 : Gen[Batch[A]], bg2 : Gen[Batch[A]], t : Timeout) : Gen[PDStream[A]] = 147 | PDStreamGen.release(now(bg1), now(bg2), t) 148 | } 149 | 150 | class BatchGen[A](self : Gen[Batch[A]]) { 151 | import BatchGenConversions._ 152 | /** Returns the generator that results from concatenating the sequences 153 | * generated by the generator wrapped by this, and other. This only makes 154 | * sense when ordered batches are considered 155 | * */ 156 | def ++(other : Gen[Batch[A]]) : Gen[Batch[A]] = UtilsGen.concSeq(self, other) 157 | 158 | /** @return a generator for the concatenation of the generated batches 159 | */ 160 | def +(other : Gen[Batch[A]]) : Gen[Batch[A]] = 161 | for { 162 | b1 <- self 163 | b2 <- other 164 | } yield b1 ++ b2 165 | 166 | def next : Gen[PDStream[A]] = BatchGen.next(self) 167 | def laterN(n : Int) : Gen[PDStream[A]] = BatchGen.laterN(n, self) 168 | def eventually : Gen[PDStream[A]] = BatchGen.eventually(self) 169 | def until(other : Gen[Batch[A]]) : Gen[PDStream[A]] = BatchGen.until(self, other) 170 | def always : Gen[PDStream[A]] = BatchGen.always(self) 171 | def release(other : Gen[Batch[A]]) : Gen[PDStream[A]] = BatchGen.release(self, other) 172 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/Buildables.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.util.Buildable 4 | import scala.language.{implicitConversions,higherKinds,postfixOps} 5 | 6 | object Buildables { 7 | /** Buildable for Seq, so we can use Gen.containerOf and Arbitrary.arbitrary 8 | * using Seq as a container 9 | * */ 10 | implicit def buildableSeq[T] = new Buildable[T, Seq[T]] { 11 | def builder = new collection.mutable.Builder[T,Seq[T]] { 12 | var xs : List[T] = List() 13 | def +=(x: T) = { 14 | xs = x :: xs 15 | this 16 | } 17 | def clear = xs = List() 18 | def result = xs reverse // note it is important to respect the generation order 19 | } 20 | } 21 | 22 | /** Builds a Buildable as a transformation of a given buildable, both element wise and 23 | * on the container 24 | * */ 25 | def mapBuildable[T1, C1[_], T2, C2[_]] 26 | (elemMapping : T2 => T1, resultMapping : C1[T1] => C2[T2])(buildable : Buildable[T1, C1[T1]]) 27 | : Buildable[T2, C2[T2]] = new Buildable[T2, C2[T2]] { 28 | def builder = new collection.mutable.Builder[T2,C2[T2]] { 29 | var nestedBuilder : collection.mutable.Builder[T1, C1[T1]] = buildable.builder 30 | def +=(x : T2) = { 31 | nestedBuilder += elemMapping(x) 32 | this 33 | } 34 | def clear = nestedBuilder.clear 35 | def result = nestedBuilder.mapResult(resultMapping).result 36 | } 37 | } 38 | /** Builds a Buildable as a transformation of a given buildable, by tranforming 39 | * the result with collection.mutable.Builder.mapResult 40 | * */ 41 | def mapBuildable[T, R1, R2] 42 | (resultMapping : R1 => R2)(buildable : Buildable[T, R1]) 43 | : Buildable[T, R2] = new Buildable[T, R2] { 44 | def builder = new collection.mutable.Builder[T,R2] { 45 | var nestedBuilder : collection.mutable.Builder[T, R1] = buildable.builder 46 | def +=(x : T) = { 47 | nestedBuilder += x 48 | this 49 | } 50 | def clear = nestedBuilder.clear 51 | def result = nestedBuilder.mapResult(resultMapping).result 52 | } 53 | } 54 | 55 | /** A Buildable for building an object Batch[T] from its elements of type T 56 | * */ 57 | implicit def buildableBatch[T] : Buildable[T, Batch[T]] = { 58 | /* alternative implementation based on the overload of mapBuildable, implies additional 59 | * calls to identity 60 | 61 | mapBuildable(identity[T], (xs : List[T]) => Batch(xs))(implicitly[Buildable[T, List[T]]]) 62 | */ 63 | mapBuildable((xs : List[T]) => Batch(xs:_*))(implicitly[Buildable[T, List[T]]]) 64 | } 65 | 66 | /** A Buildable for building an object DStream[T] from its batches of type Batch[T] 67 | * */ 68 | implicit def buildablePDStreamFromBatch[T] : Buildable[Batch[T], PDStream[T]] = 69 | mapBuildable((batches : List[Batch[T]]) => PDStream(batches:_*))( 70 | implicitly[Buildable[Batch[T], List[Batch[T]]]]) 71 | 72 | 73 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/PDStream.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalatest.matchers.{Matcher, MatchResult} 4 | import scala.language.implicitConversions 5 | 6 | object PDStream { 7 | def empty[A] : PDStream[A] = new PDStream(List():_*) 8 | 9 | implicit def batchSeq2dstream[A](batches : Seq[Batch[A]]) : PDStream[A] = PDStream(batches:_*) 10 | implicit def seqSeq2dstream[A](batches : Seq[Seq[A]]) : PDStream[A] = PDStream(batches.map(Batch(_:_*)):_*) 11 | } 12 | 13 | /** An object of this class represents a finite prefix of a discrete data streams, 14 | * aka prefix DStream or just PDStream 15 | * */ 16 | case class PDStream[A](batches : Batch[A]*) extends Seq[Batch[A]] { 17 | override def toSeq : Seq[Batch[A]] = batches 18 | 19 | override def apply(idx : Int) = batches.apply(idx) 20 | override def iterator = batches.iterator 21 | override def length = batches.length 22 | 23 | // Note def ++(other : DStream[A]) : DStream[A] is inherited from Seq[_] 24 | 25 | /** @return a DStream for the batch-by-batch concatenation of this 26 | * and other. Note we fill both PDStreams with empty batches. This 27 | * implies both PDStreams are implicitly treated as they where 28 | * infinitely extended with empty batches 29 | */ 30 | def #+(other : PDStream[A]) : PDStream[A] = { 31 | batches. zipAll(other, Batch.empty, Batch.empty) 32 | . map(xs12 => xs12._1 ++ xs12._2) 33 | } 34 | 35 | /** @return true iff each batch of this dstream is contained in the batch 36 | * at the same position in other. Note this implies that true can be 37 | * returned for cases when other has more batches than this 38 | * */ 39 | def subsetOf(other : PDStream[A]) : Boolean = { 40 | batches 41 | .zip(other.batches) 42 | .map({case (thisBatch, otherBatch) => 43 | thisBatch.forall(otherBatch.contains(_)) 44 | }) 45 | .forall(identity[Boolean]) 46 | } 47 | } 48 | 49 | trait DStreamMatchers { 50 | class DStreamSubsetOf[A](expectedSuperDStream : PDStream[A]) extends Matcher[PDStream[A]] { 51 | override def apply(observedDStream : PDStream[A]) : MatchResult = { 52 | // FIXME reimplement with Inspector for better report 53 | MatchResult(observedDStream.subsetOf(expectedSuperDStream), 54 | s"""$observedDStream is not a pointwise subset of $expectedSuperDStream""", 55 | s"""$observedDStream is a pointwise subset of $expectedSuperDStream""") 56 | } 57 | } 58 | def beSubsetOf[A](expectedSuperDStream : PDStream[A]) = new DStreamSubsetOf(expectedSuperDStream) 59 | } 60 | object DStreamMatchers extends DStreamMatchers 61 | -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/PDStreamGen.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.Gen 4 | import org.scalacheck.Arbitrary 5 | import org.scalacheck.Arbitrary.arbitrary 6 | import org.scalacheck.Shrink 7 | import org.scalacheck.Shrink.shrink 8 | import BatchGen.now 9 | import scala.language.implicitConversions 10 | import es.ucm.fdi.sscheck.prop.tl.Timeout 11 | 12 | /** Implicit conversions removed from BatchGen companion. This is needed 13 | * because we cannot populate the name space with standard names for HO generators 14 | * like of() or ofN(), but we also want to have these implicit conversions available 15 | */ 16 | object PDStreamGenConversions { 17 | implicit def genDStream2DStreamGen[A](dsg : Gen[PDStream[A]]) : PDStreamGen[A] = PDStreamGen(dsg) 18 | 19 | implicit def dstreamGen2batchSeqGen[A](gs : Gen[PDStream[A]]) : Gen[Seq[Batch[A]]] = gs.map(_.toSeq) 20 | implicit def batchSeqGen2dstreamGen[A](gs : Gen[Seq[Batch[A]]]) : Gen[PDStream[A]] = gs.map(PDStream(_:_*)) 21 | } 22 | 23 | /** 24 | * All the temporal generators defined in this object are sized generators, but the size parameters 25 | * only affects the number of batches in the output PDStream, not the size of the batches. 26 | * The size of the batches can be changed by using Gen.resize in the definition of the batch 27 | * generator that is passed to these HO generators 28 | * 29 | * On the other hand for generators of arbitrary PDStreams like Gen.resize(5, arbitrary[DStream[Int]]) 30 | * Gen.resize has effect, both in the number of batches and in the size of the batches, as those 31 | * arbitrary instances are based on arbitrary for lists 32 | * 33 | * TODO That should be tested 34 | // by using Gen.resize(5, TLGen.always(Batch(List(0, 1)))) (constant batch size) 35 | // and then in Gen.resize(3, TLGen.always(BatchGen.ofNtoM(1, 5, Gen.choose(0, 3)))) 36 | // the point is that we always have 3 batches, but Gen.resize doesn't affect the batch 37 | // size but only the number of batches 38 | // 39 | * 40 | * */ 41 | object PDStreamGen { 42 | import PDStreamGenConversions._ 43 | 44 | def apply[A](dsg : Gen[PDStream[A]]) : PDStreamGen[A] = new PDStreamGen(dsg) 45 | 46 | /** Arbitrary generator for DStream. Only works if an Arbitrary for Batch is also present. 47 | * Note this hasn't been automatically derived from Buildables.buildableDStreamFromBatch 48 | * for the same reason Gen.buildableOf has to be used with DStream instead of Gen.containerOf 49 | * */ 50 | implicit def arbDStream[A](implicit arbBatch : Arbitrary[Batch[A]]) : Arbitrary[PDStream[A]] = 51 | Arbitrary(PDStreamGen.of(arbitrary[Batch[A]])) 52 | 53 | /* This makes no sense for temporal generators, see third comment 54 | * at issue #10 55 | */ 56 | // /** Shrink function for DStream 57 | // * */ 58 | // implicit def shrinkDStream[A] : Shrink[PDStream[A]] = Shrink(pdstream => 59 | // // unwrap the underlying Seq, shrink the Seq, and rewrap 60 | // shrink(pdstream.toSeq).map(PDStream(_:_*)) 61 | // ) 62 | 63 | /** @return a generator of DStream that generates its batches from bg 64 | * */ 65 | def of[T](bg : => Gen[Batch[T]]) : Gen[PDStream[T]] = { 66 | import Buildables.buildablePDStreamFromBatch 67 | Gen.buildableOf[PDStream[T], Batch[T]](bg) 68 | } 69 | /** @return a generator of DStream that generates its batches from bg 70 | * */ 71 | def ofN[T](n : Int, bg : Gen[Batch[T]]) : Gen[PDStream[T]] = { 72 | import Buildables.buildablePDStreamFromBatch 73 | Gen.buildableOfN[PDStream[T], Batch[T]](n, bg) 74 | } 75 | /** @return a generator of DStream that generates its batches from bg 76 | * */ 77 | def ofNtoM[T](n : Int, m : Int, bg : Gen[Batch[T]]) : Gen[PDStream[T]] = { 78 | import Buildables.buildablePDStreamFromBatch 79 | UtilsGen.buildableOfNtoM[PDStream[T], Batch[T]](n, m, bg) 80 | } 81 | 82 | /** @return a generator for the batch-by-batch concatenation of the PDStreams 83 | * generated by gs1 and gs2. Note we fill either PDStreams with empty batches 84 | * This implies PDStreams are implicitly treated as they where infinitely extended 85 | * with empty batches 86 | */ 87 | def dstreamUnion[A](gs1 : Gen[PDStream[A]], gs2 : Gen[PDStream[A]]) : Gen[PDStream[A]] = { 88 | for { 89 | xs1 <- gs1 90 | xs2 <- gs2 91 | } yield xs1 #+ xs2 92 | } 93 | 94 | /** @return a dstream generator that has an empty batch followed by the batches 95 | * from dsg 96 | * */ 97 | def next[A](dsg : Gen[PDStream[A]]) : Gen[PDStream[A]] = { 98 | laterN(1, dsg) 99 | } 100 | 101 | /** @return a dstream generator that has n empty batches followed by the batches 102 | * from dsg 103 | * */ 104 | def laterN[A](n : Int, dsg : Gen[PDStream[A]]) : Gen[PDStream[A]] = 105 | for { 106 | ds <- dsg 107 | blanks = Seq.fill(n)(Batch.empty : Batch[A]) 108 | } yield blanks ++ ds 109 | 110 | /** 111 | * This is sized generator where the size acts like the timeout, so 112 | * - with size <= 0 returns the empty PDStream, as no word can 113 | * succeed with a timeout of 0 114 | * - with size 1 returns a PDStream generated by dsg2 115 | * - otherwise dsg2 happens after between 0 and size-1 instants 116 | * */ 117 | def until[A](dsg1 : Gen[PDStream[A]], dsg2 : Gen[PDStream[A]]) : Gen[PDStream[A]] = 118 | Gen.sized { size => 119 | if (size <= 0) 120 | Gen.const(PDStream.empty) 121 | else if (size == 1) 122 | dsg2 123 | else 124 | for { 125 | proofOffset <- Gen.choose(0, size-1) 126 | prefix <- always(dsg1, Timeout(proofOffset)) 127 | dsg2Proof <- laterN(proofOffset, dsg2) 128 | } yield prefix #+ dsg2Proof 129 | } 130 | 131 | def until[A](dsg1 : Gen[PDStream[A]], dsg2 : Gen[PDStream[A]], t : Timeout) : Gen[PDStream[A]] = 132 | Gen.resize(t.instants, until(dsg1, dsg2)) 133 | 134 | /** Sized generator where the size acts like the timeout, so 135 | * - with size <= 0 returns the empty PDStream 136 | * - otherwise dsg happens at some moment between 0 and size instants 137 | */ 138 | def eventually[A](dsg : Gen[PDStream[A]]) : Gen[PDStream[A]] = 139 | Gen.sized { size => 140 | if (size <0) 141 | Gen.const(PDStream.empty) 142 | else 143 | PDStreamGen.ofNtoM(0, size -1, Batch.empty : Batch[A]) ++ dsg 144 | } 145 | def eventually[A](dsg : Gen[PDStream[A]], t : Timeout) : Gen[PDStream[A]] = 146 | Gen.resize(t.instants, eventually(dsg)) 147 | 148 | /** Sized generator where the size acts like the timeout, so 149 | * - with size <= 0 returns the empty PDStream 150 | * - otherwise dsg happens size times from now 151 | */ 152 | def always[A](dsg : Gen[PDStream[A]]) : Gen[PDStream[A]] = 153 | Gen.sized { size => 154 | if (size <= 0) // supporting size == 0 is needed by the calls to always from until and release 155 | Gen.const(PDStream.empty) 156 | else 157 | (for { 158 | offset <- 0 until size 159 | } yield laterN(offset, dsg) 160 | ) 161 | .reduce(dstreamUnion(_,_)) 162 | } 163 | def always[A](dsg : Gen[PDStream[A]], t : Timeout) : Gen[PDStream[A]] = 164 | Gen.resize(t.instants, always(dsg)) 165 | 166 | /** 167 | * Generator for the weak version of LTL release operator: either dsg2 168 | * happens forever, or it happens until dsg1 happens, including the 169 | * moment when dsg1 happens 170 | * 171 | * This is a sized generator where the size acts like the timeout, so 172 | * - with size <= 0 returns the empty PDStream 173 | * - otherwise, if dsg1 happens then it happens after between 0 and size-1 instants 174 | * */ 175 | def release[A](dsg1 : Gen[PDStream[A]], dsg2 : Gen[PDStream[A]]) : Gen[PDStream[A]] = 176 | Gen.sized { size => 177 | if (size <= 0) 178 | Gen.const(PDStream.empty) 179 | else if (size == 1) for { 180 | isReleased <- arbitrary[Boolean] 181 | proof <- if (isReleased) (dsg1 + dsg2) else dsg2 182 | } yield proof 183 | else for { 184 | isReleased <- arbitrary[Boolean] 185 | ds <- if (!isReleased) 186 | always(dsg2, Timeout(size)) 187 | else for { 188 | proofOffset <- Gen.choose(0, size-1) 189 | prefix <- always(dsg2, Timeout(proofOffset)) 190 | ending <- laterN(proofOffset, dsg1 + dsg2) 191 | } yield prefix #+ ending 192 | } yield ds 193 | } 194 | 195 | def release[A](dsg1 : Gen[PDStream[A]], dsg2 : Gen[PDStream[A]], t : Timeout) : Gen[PDStream[A]] = 196 | Gen.resize(t.instants, release(dsg1, dsg2)) 197 | } 198 | 199 | class PDStreamGen[A](self : Gen[PDStream[A]]) { 200 | import PDStreamGen._ 201 | import PDStreamGenConversions._ 202 | /** Returns the generator that results from concatenating the sequences 203 | * generated by the generator wrapped by this, and other 204 | * */ 205 | def ++(other : Gen[PDStream[A]]) : Gen[PDStream[A]] = UtilsGen.concSeq(self, other) 206 | 207 | /** @return a generator for the batch-by-batch concatenation of the PDStreams 208 | * generated by the generator wrapped by this, and other 209 | * Note we fill either PDStreams with empty batches. This implies PDStreams 210 | * are implicitly treated as they where infinitely extended with empty batches 211 | */ 212 | def +(other : Gen[PDStream[A]]) : Gen[PDStream[A]] = dstreamUnion(self, other) 213 | 214 | def next : Gen[PDStream[A]] = PDStreamGen.next(self) 215 | def laterN(n : Int) : Gen[PDStream[A]] = PDStreamGen.laterN(n, self) 216 | def eventually : Gen[PDStream[A]] = PDStreamGen.eventually(self) 217 | def until(other : Gen[PDStream[A]]) : Gen[PDStream[A]] = PDStreamGen.until(self, other) 218 | def always : Gen[PDStream[A]] = PDStreamGen.always(self) 219 | def release(other : Gen[PDStream[A]]) : Gen[PDStream[A]] = PDStreamGen.release(self, other) 220 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/RDDGen.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.Gen 4 | 5 | import org.apache.spark._ 6 | import org.apache.spark.rdd.RDD 7 | 8 | import scala.reflect.ClassTag 9 | import scala.language.implicitConversions 10 | 11 | import es.ucm.fdi.sscheck.spark.Parallelism 12 | 13 | /** Generators for RDDs and implicit conversions from Seq and Seq generators to RDD and RDD 14 | * generators. All the functions are based on parallelizing generated sequences 15 | * 16 | * Instead of defining a class holding a spark context and a [[es.ucm.fdi.sscheck.spark.Parallelism]] 17 | * object specifying the number of partitions to use on parallelization, these have to be explicitly 18 | * provided when calling the methods, although these arguments have been made implicit to simulate 19 | * state if wanted, like it is done in [[es.ucm.fdi.sscheck.spark.SharedSparkContextBeforeAfterAll]] 20 | * */ 21 | object RDDGen { 22 | /** Convert a ScalaCheck generator of Seq into a generator of RDD 23 | * */ 24 | implicit def seqGen2RDDGen[A](sg : Gen[Seq[A]]) 25 | (implicit aCt: ClassTag[A], sc : SparkContext, parallelism : Parallelism) : Gen[RDD[A]] = 26 | sg.map(sc.parallelize(_, numSlices = parallelism.numSlices)) 27 | 28 | /** Convert a sequence into a RDD 29 | * */ 30 | implicit def seq2RDD[A](seq : Seq[A])(implicit aCt: ClassTag[A], sc : SparkContext, parallelism : Parallelism) : RDD[A] = 31 | sc.parallelize(seq, numSlices=parallelism.numSlices) 32 | 33 | /** @return a generator of RDD that generates its elements from g 34 | * */ 35 | def of[A](g : => Gen[A]) 36 | (implicit aCt: ClassTag[A], sc : SparkContext, parallelism : Parallelism) 37 | : Gen[RDD[A]] = 38 | // this way is much simpler that implementing this with a ScalaCheck 39 | // Buildable, because that implies defining a wrapper to convert RDD into Traversable 40 | seqGen2RDDGen(Gen.listOf(g)) 41 | 42 | /** @return a generator of RDD that generates its elements from g 43 | * */ 44 | def ofN[A](n : Int, g : Gen[A]) 45 | (implicit aCt: ClassTag[A], sc : SparkContext, parallelism : Parallelism) 46 | : Gen[RDD[A]] = { 47 | seqGen2RDDGen(Gen.listOfN(n, g)) 48 | } 49 | 50 | /** @return a generator of RDD that generates its elements from g 51 | * */ 52 | def ofNtoM[A](n : Int, m : Int, g : => Gen[A]) 53 | (implicit aCt: ClassTag[A], sc : SparkContext, parallelism : Parallelism) 54 | : Gen[RDD[A]] = 55 | seqGen2RDDGen(UtilsGen.containerOfNtoM[List, A](n, m, g)) 56 | } 57 | 58 | -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/ReGen.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.Gen 4 | 5 | import es.ucm.fdi.sscheck.gen.UtilsGen.containerOfNtoM; 6 | import Buildables.buildableSeq 7 | import UtilsGen.{containerOfNtoM} 8 | 9 | import scala.language.postfixOps 10 | 11 | /** 12 | Generators for simple regexp, as generators of Seq[A] 13 | */ 14 | object ReGen { 15 | 16 | val epsilon = Gen.const(List()) 17 | 18 | def symbol[A](s : A) : Gen[Seq[A]] = Gen.const(List(s)) 19 | 20 | // def alt[A](g1 : Gen[Seq[A]], g2 : Gen[Seq[A]]) : Gen[Seq[A]] = Gen.oneOf(g1, g2) 21 | def alt[A](gs : Gen[Seq[A]]*) : Gen[Seq[A]] = { 22 | val l = gs.length 23 | // require (l > 0, "alt needs at least one alternative") 24 | if (l == 0) 25 | epsilon 26 | else if (l == 1) 27 | gs(0) 28 | else 29 | Gen.oneOf(gs(0), gs(1), gs.slice(2, l):_*) 30 | } 31 | 32 | def conc[A](g1 : Gen[Seq[A]], g2 : Gen[Seq[A]]) : Gen[Seq[A]] = { 33 | for { 34 | xs <- g1 35 | ys <- g2 36 | } yield xs ++ ys 37 | } 38 | 39 | def star[A](g : Gen[Seq[A]]) : Gen[Seq[A]] = { 40 | for { 41 | xs <- Gen.containerOf(g) 42 | } yield xs flatten 43 | } 44 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/gen/UtilsGen.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.Gen 4 | import org.scalacheck.util.Buildable 5 | import Buildables.buildableSeq 6 | 7 | import scala.language.{postfixOps,higherKinds} 8 | 9 | object UtilsGen { 10 | /** Like containerOfN but with variable number of elements 11 | * */ 12 | def containerOfNtoM[C[_], T] 13 | (n : Int, m : Int, g : Gen[T]) 14 | (implicit evb: Buildable[T, C[T]], evt: (C[T]) => Traversable[T]) 15 | : Gen[C[T]] = { 16 | for { 17 | i <- Gen.choose(n, m) 18 | xs <- Gen.containerOfN[C, T](i, g) 19 | } yield xs 20 | } 21 | 22 | def buildableOfNtoM[C, T] 23 | (n : Int, m : Int, g : Gen[T]) 24 | (implicit evb: Buildable[T, C], evt: (C) => Traversable[T]) 25 | : Gen[C] = { 26 | for { 27 | i <- Gen.choose(n, m) 28 | xs <- Gen.buildableOfN[C, T](i, g) 29 | } yield xs 30 | } 31 | 32 | /** Generates n sequences from g and concatenates them 33 | * */ 34 | def repN[A](n : Int, g : Gen[Seq[A]]) : Gen[Seq[A]] = { 35 | for { 36 | xs <- Gen.containerOfN(n, g) 37 | } yield xs flatten 38 | } 39 | 40 | /** Generates i sequences from g, with i between n and m, and concatenates them 41 | * */ 42 | def repNtoM[A](n : Int, m : Int, g : Gen[Seq[A]]) : Gen[Seq[A]] = { 43 | for { 44 | xs <- containerOfNtoM(n, m, g) 45 | } yield xs flatten 46 | } 47 | 48 | /** @return the generator that results from concatenating the sequences 49 | * generated by g1 and g2 50 | * */ 51 | def concSeq[A](g1 : Gen[Seq[A]], g2 : Gen[Seq[A]]) : Gen[Seq[A]] = { 52 | for { 53 | xs <- g1 54 | ys <- g2 55 | } yield xs ++ ys 56 | } 57 | 58 | /** @return if gen is present then a generator that wraps in Some 59 | * all the values generated by the value in gen, otherwise return 60 | * a generator that always returns None 61 | * */ 62 | def optGenToGenOpt[A](gen: Option[Gen[A]]): Gen[Option[A]] = 63 | gen.fold(Gen.const[Option[A]](None)){_.map(Some(_))} 64 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/matcher/specs2/package.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.matcher { 2 | 3 | package specs2 { 4 | import org.apache.spark.rdd.RDD 5 | import org.specs2.matcher.Matcher 6 | import org.specs2.matcher.MatchersImplicits._ 7 | import scalaz.syntax.std.boolean._ 8 | 9 | object RDDMatchers { 10 | /** Number of records to show on failing predicates 11 | * */ 12 | private val numErrors = 4 13 | 14 | /** @return a matcher that checks whether predicate holds for all the records of 15 | * an RDD or not. 16 | * 17 | * NOTE: in case exceptions like the following are generated when using a closure for the 18 | * the predicate, use the other variant of foreachRecord() to explicitly specify the context 19 | * available to the closure 20 | * {{{ 21 | * Driver stacktrace:,org.apache.spark.SparkException: Job aborted due to stage failure: 22 | * Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost): 23 | * java.io.InvalidClassException: org.specs2.execute.Success; no valid constructor 24 | * }}} 25 | * */ 26 | def foreachRecord[T](predicate: T => Boolean): Matcher[RDD[T]] = { (rdd: RDD[T]) => 27 | val failingRecords = rdd.filter(! predicate(_)) 28 | ( 29 | failingRecords.isEmpty, 30 | "each record fulfils the predicate", 31 | s"predicate failed for records ${failingRecords.take(numErrors).mkString(", ")} ..." 32 | ) 33 | } 34 | 35 | def foreachRecord[T,C](predicateContext: C)(toPredicate: C => (T => Boolean)): Matcher[RDD[T]] = { 36 | val predicate = toPredicate(predicateContext) 37 | foreachRecord(predicate) 38 | } 39 | 40 | /** @return a matcher that checks whether predicate holds for at least one of the records of 41 | * an RDD or not. 42 | * 43 | * NOTE: in case exceptions like the following are generated when using a closure for the 44 | * the predicate, use the other variant of foreachRecord() to explicitly specify the context 45 | * available to the closure 46 | * {{{ 47 | * Driver stacktrace:,org.apache.spark.SparkException: Job aborted due to stage failure: 48 | * Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost): 49 | * java.io.InvalidClassException: org.specs2.execute.Success; no valid constructor 50 | * }}} 51 | * */ 52 | def existsRecord[T](predicate: T => Boolean): Matcher[RDD[T]] = { (rdd: RDD[T]) => 53 | val exampleRecords = rdd.filter(predicate(_)) 54 | ( 55 | ! exampleRecords.isEmpty, 56 | "some record fulfils the predicate", 57 | s"predicate failed for all the records" 58 | ) 59 | } 60 | 61 | def existsRecord[T,C](predicateContext: C)(toPredicate: C => (T => Boolean)): Matcher[RDD[T]] = { 62 | val predicate = toPredicate(predicateContext) 63 | existsRecord(predicate) 64 | } 65 | 66 | /** @return a Matcher that checks that both RDDs are equal as sets. It is is recommended to 67 | * cache both RDDs to avoid recomputation 68 | * */ 69 | def beEqualAsSetTo[T](actualRDD: RDD[T]): Matcher[RDD[T]] = { (expectedRDD: RDD[T]) => 70 | val inActualNotInExpected = actualRDD.subtract(expectedRDD) 71 | val inExpectedNotInActual = expectedRDD.subtract(actualRDD) 72 | lazy val errorMsg: String = 73 | List( 74 | (!inActualNotInExpected.isEmpty) option 75 | s"unexpected records: ${inActualNotInExpected.take(numErrors).mkString(",")} ...", 76 | (!inExpectedNotInActual.isEmpty) option 77 | s"missing records: ${inExpectedNotInActual.take(numErrors).mkString(",")} ..." 78 | ).filter(_.isDefined) 79 | .map(_.get).mkString(",") 80 | ( 81 | inActualNotInExpected.isEmpty && inExpectedNotInActual.isEmpty, 82 | "both RDDs contain the same records", 83 | errorMsg 84 | ) 85 | } 86 | 87 | // TODO: idea for predicate with partial functions rdd.collect{case record => true}.toLocalIterator.hasNext must beTrue 88 | } 89 | } 90 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/package.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi { 2 | package object sscheck { 3 | type TestCaseId = Int 4 | } 5 | 6 | package sscheck { 7 | import java.util.concurrent.atomic.AtomicInteger 8 | 9 | /** This class can be used to generate unique test case identifiers per each instance. 10 | * */ 11 | class TestCaseIdCounter { 12 | val counter = new AtomicInteger(0) 13 | /** @return a fresh TestCaseId. This method is thread safe 14 | * */ 15 | def nextId() : TestCaseId = counter.getAndIncrement() 16 | } 17 | } 18 | } 19 | 20 | -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/prop/UtilsProp.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.prop 2 | 3 | import org.scalacheck.{Properties, Gen} 4 | import org.scalacheck.Prop.{forAll, exists, AnyOperators} 5 | import org.scalacheck.Prop 6 | import org.scalatest._ 7 | import org.scalatest.Matchers._ 8 | import org.specs2.matcher.MatchFailureException 9 | import scala.util.{Try, Success, Failure} 10 | import org.scalacheck.util.Pretty 11 | 12 | object UtilsProp { 13 | def safeProp[P <% Prop](p : => P) : Prop = { 14 | Try(p) match { 15 | case Success(pVal) => pVal 16 | case Failure(t) => t match { 17 | case _: TestFailedException => Prop.falsified 18 | case _: MatchFailureException[_] => Prop.falsified 19 | case _ => Prop.exception(t) 20 | } 21 | } 22 | } 23 | 24 | def safeExists[A, P](g: Gen[A])(f: (A) => P) 25 | (implicit pv: (P) => Prop, pp: (A) => Pretty): Prop = { 26 | exists(g)((x : A) => safeProp(pv(f(x))))(identity[Prop], pp) 27 | // This doesn't work for reasons unknown 28 | // exists(g)((x : A) => Prop.secure(pv(f(x))))(identity[Prop], pp) 29 | } 30 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/prop/tl/DStreamTLProperty.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.prop.tl 2 | 3 | import org.scalacheck.Gen 4 | import org.scalacheck.Prop 5 | import org.scalacheck.util.Pretty 6 | import org.apache.spark._ 7 | import org.apache.spark.rdd.RDD 8 | import org.apache.spark.storage.StorageLevel 9 | import org.apache.spark.streaming.dstream.DStream 10 | import org.apache.spark.streaming.{StreamingContext, Duration} 11 | import org.apache.spark.streaming.{Time => SparkTime} 12 | import scala.reflect.ClassTag 13 | import scala.concurrent.{Future, ExecutionContext} 14 | import java.util.concurrent.Executors 15 | import scala.util.{Try, Success, Failure} 16 | import org.slf4j.LoggerFactory 17 | import scala.util.Properties.lineSeparator 18 | import scala.language.implicitConversions 19 | import es.ucm.fdi.sscheck.gen.UtilsGen.optGenToGenOpt 20 | import es.ucm.fdi.sscheck.{TestCaseIdCounter,TestCaseId} 21 | import es.ucm.fdi.sscheck.spark.{SharedSparkContextBeforeAfterAll,Parallelism} 22 | import es.ucm.fdi.sscheck.spark.streaming 23 | import es.ucm.fdi.sscheck.spark.streaming.TestInputStream 24 | 25 | import DStreamTLProperty.SSeq 26 | import com.google.common.collect.Synchronized 27 | 28 | /* 29 | * TODO 30 | * - see https://issues.apache.org/jira/browse/SPARK-12009 and see if update to Spark 2.0.0 fixes 31 | * 32 | 16/10/03 04:09:55 ERROR StreamingListenerBus: StreamingListenerBus has already stopped! Dropping event StreamingListenerBatchCompleted(BatchInfo(1475467783650 ms,Map(),1475467783676,Some(1475467795274),Some(1475467795547),Map(0 -> OutputOperationInfo(1475467783650 ms,0,foreachRDD at DStreamTLProperty.scala:258,org.apache.spark.streaming.dstream.DStream.foreachRDD(DStream.scala:668) 33 | es.ucm.fdi.sscheck.prop.tl.TestCaseContext$.es$ucm$fdi$sscheck$prop$tl$TestCaseContext$$printDStream(DStreamTLProperty.scala:258) 34 | es.ucm.fdi.sscheck.prop.tl.TestCaseContext.init(DStreamTLProperty.scala:334) 35 | * 36 | * that is an error message than is thrown during the graceful stop of the streaming context, 37 | * and DStreamTLProperty.scala:258 is the foreachRDD at TestCaseContext.printDStream() 38 | * 39 | * - see https://issues.apache.org/jira/browse/SPARK-14701 and see if update to Spark 2.0.0 fixes 40 | * 41 | java.util.concurrent.RejectedExecutionException: Task org.apache.spark.streaming.CheckpointWriter$CheckpointWriteHandler@1a08d2 rejected from java.util.concurrent.ThreadPoolExecutor@b03cb4[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 70] 42 | at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047) 43 | at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823) 44 | at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369) 45 | at org.apache.spark.streaming.CheckpointWriter.write(Checkpoint.scala:278) 46 | at org.apache.spark.streaming.scheduler.JobGenerator.doCheckpoint(JobGenerator.scala:295) 47 | at org.apache.spark.streaming.scheduler.JobGenerator.org$apache$spark$streaming$scheduler$JobGenerator$$processEvent(JobGenerator.scala:184) 48 | at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:87) 49 | at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:86) 50 | at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) 51 | 16/10/03 04:34:17 ERROR StreamingListenerBus: StreamingListenerBus has already stopped! Dropping event StreamingListenerBatchCompleted(BatchInfo(1475469252300 52 | * */ 53 | 54 | object DStreamTLProperty { 55 | @transient private val logger = LoggerFactory.getLogger("DStreamTLProperty") 56 | 57 | type SSeq[A] = Seq[Seq[A]] 58 | type SSGen[A] = Gen[SSeq[A]] 59 | } 60 | 61 | trait DStreamTLProperty 62 | extends SharedSparkContextBeforeAfterAll { 63 | 64 | import DStreamTLProperty.{logger,SSeq,SSGen} 65 | 66 | /** Override for custom configuration 67 | * */ 68 | def batchDuration : Duration 69 | 70 | /** Override for custom configuration 71 | * Disabled by default because it is quite costly 72 | * */ 73 | def enableCheckpointing : Boolean = false 74 | 75 | /** Override for custom configuration 76 | * Whether the input and output RDDs for each test case should be cached 77 | * (with RDD.cache) or not, true by default 78 | * 79 | * Set to false if you get"Uncaught exception: RDDBlockId not found in driver-heartbeater", 80 | * "java.lang.ClassNotFoundException: org.apache.spark.storage.RDDBlockId", 81 | * "ERROR TaskSchedulerImpl: Lost executor driver on localhost: Executor heartbeat timed out" or 82 | * "ExecutorLostFailure (executor driver exited caused by one of the running tasks) Reason: Executor heartbeat timed out" 83 | * as it is a known problem https://issues.apache.org/jira/browse/SPARK-10722, that was reported 84 | * as fixed in Spark 1.6.2 in SPARK-10722 85 | */ 86 | def catchRDDs: Boolean = true 87 | 88 | /** Override for custom configuration 89 | * Maximum number of micro batches that the test case will wait for, 100 by default 90 | * */ 91 | def maxNumberBatchesPerTestCase: Int = 100 92 | 93 | /** @return a newly created streaming context, for which no DStream or action has 94 | * been defined, and that it's not started 95 | * 96 | * Precondition: no streaming context is currently started 97 | * in this JVM 98 | * */ 99 | private def buildFreshStreamingContext() : StreamingContext = { 100 | val __sc = sc() 101 | logger.warn(s"creating test Spark Streaming context") 102 | val newSsc = new StreamingContext(__sc, batchDuration) 103 | if (enableCheckpointing) { 104 | val checkpointDir = streaming.Utils.createTempDir().toString 105 | logger.warn(s"configuring Spark Streaming checkpoint directory ${checkpointDir}") 106 | newSsc.checkpoint(checkpointDir) 107 | } 108 | newSsc 109 | } 110 | 111 | // 1 in 1 out 112 | /** @return a ScalaCheck property that is executed by: 113 | * - generating a prefix of a DStream with g1 114 | * - generating a derived DStream with gt1 115 | * - checking formula on those DStreams 116 | * 117 | * The property is satisfied iff all the test cases satisfy the formula. 118 | * A new streaming context is created for each test case to isolate its 119 | * execution, which is particularly relevant if gt1 is stateful 120 | * 121 | * WARNING: the resulting Prop cannot be configured for parallel execution of 122 | * test cases, in order to avoid having more than a single StreamingContext 123 | * started in the same JVM 124 | * */ 125 | def forAllDStream[I1:ClassTag,O1:ClassTag]( 126 | gen1: SSGen[I1])( 127 | trans1: (DStream[I1]) => DStream[O1])( 128 | formula: Formula[(RDD[I1], RDD[O1])])( 129 | implicit pp1: SSeq[I1] => Pretty): Prop = { 130 | 131 | type U = (RDD[I1], RDD[O1]) 132 | implicit def atomsAdapter(a1: Option[RDD[I1]], a2: Option[RDD[Unit]], 133 | a3: Option[RDD[O1]], a4: Option[RDD[Unit]]): U = 134 | (a1.get, a3.get) 135 | forAllDStreamOption[I1,Unit,O1,Unit,U]( 136 | gen1, None)( 137 | (ds1, ds2) => trans1(ds1.get), 138 | None)( 139 | formula) 140 | } 141 | 142 | // 1 in 2 out 143 | /** 1 input - 2 outputs version of [[DStreamTLProperty.forAllDStream[I1,O1]:Prop*]]. 144 | * */ 145 | def forAllDStream12[I1:ClassTag,O1:ClassTag,O2:ClassTag]( 146 | gen1: SSGen[I1])( 147 | trans1: (DStream[I1]) => DStream[O1], 148 | trans2: (DStream[I1]) => DStream[O2])( 149 | formula: Formula[(RDD[I1], RDD[O1], RDD[O2])])( 150 | implicit pp1: SSeq[I1] => Pretty): Prop = { 151 | 152 | type U = (RDD[I1], RDD[O1], RDD[O2]) 153 | implicit def atomsAdapter(a1: Option[RDD[I1]], a2: Option[RDD[Unit]], 154 | a3: Option[RDD[O1]], a4: Option[RDD[O2]]): U = 155 | (a1.get, a3.get, a4.get) 156 | forAllDStreamOption[I1,Unit,O1,O2,U]( 157 | gen1, None)( 158 | (ds1, ds2) => trans1(ds1.get), 159 | Some((ds1, ds2) => trans2(ds1.get)))( 160 | formula) 161 | } 162 | 163 | // 2 in 1 out 164 | /** 2 inputs - 2 output version of [[DStreamTLProperty.forAllDStream[I1,O1]:Prop*]]. 165 | * */ 166 | def forAllDStream21[I1:ClassTag,I2:ClassTag,O1:ClassTag]( 167 | gen1: SSGen[I1], gen2: SSGen[I2])( 168 | trans1: (DStream[I1], DStream[I2]) => DStream[O1])( 169 | formula: Formula[(RDD[I1], RDD[I2], RDD[O1])])( 170 | implicit pp1: SSeq[I1] => Pretty, pp2: SSeq[I2] => Pretty): Prop = { 171 | 172 | type U = (RDD[I1], RDD[I2], RDD[O1]) 173 | implicit def atomsAdapter(a1: Option[RDD[I1]], a2: Option[RDD[I2]], 174 | a3: Option[RDD[O1]], a4: Option[RDD[Unit]]): U = 175 | (a1.get, a2.get, a3.get) 176 | forAllDStreamOption[I1,I2,O1,Unit,U]( 177 | gen1, Some(gen2))( 178 | (ds1, ds2) => trans1(ds1.get, ds2.get), 179 | None)( 180 | formula) 181 | } 182 | 183 | // 2 in 2 out 184 | /** 2 inputs - 2 outputs version of [[DStreamTLProperty.forAllDStream[I1,O1]:Prop*]]. 185 | * */ 186 | def forAllDStream22[I1:ClassTag,I2:ClassTag,O1:ClassTag,O2:ClassTag]( 187 | gen1: SSGen[I1], gen2: SSGen[I2])( 188 | trans1: (DStream[I1], DStream[I2]) => DStream[O1], 189 | trans2: (DStream[I1], DStream[I2]) => DStream[O2])( 190 | formula: Formula[(RDD[I1], RDD[I2], RDD[O1], RDD[O2])])( 191 | implicit pp1: SSeq[I1] => Pretty, pp2: SSeq[I2] => Pretty): Prop = { 192 | 193 | type U = (RDD[I1], RDD[I2], RDD[O1], RDD[O2]) 194 | implicit def atomsAdapter(a1: Option[RDD[I1]], a2: Option[RDD[I2]], 195 | a3: Option[RDD[O1]], a4: Option[RDD[O2]]): U = 196 | (a1.get, a2.get, a3.get, a4.get) 197 | forAllDStreamOption[I1,I2,O1,O2,U]( 198 | gen1, Some(gen2))( 199 | (ds1, ds2) => trans1(ds1.get, ds2.get), 200 | Some((ds1, ds2) => trans2(ds1.get, ds2.get)))( 201 | formula) 202 | } 203 | 204 | // TODO: add scaladoc 205 | private def forAllDStreamOption[I1:ClassTag,I2:ClassTag, 206 | O1:ClassTag,O2:ClassTag, 207 | U]( 208 | g1: SSGen[I1], g2: Option[SSGen[I2]])( 209 | gt1: (Option[DStream[I1]], Option[DStream[I2]]) => DStream[O1], 210 | gtOpt2: Option[(Option[DStream[I1]], Option[DStream[I2]]) => DStream[O2]])( 211 | formula: Formula[U])( 212 | implicit pp1: SSeq[I1] => Pretty, pp2: SSeq[I2] => Pretty, 213 | atomsAdapter: (Option[RDD[I1]], Option[RDD[I2]], Option[RDD[O1]], Option[RDD[O2]]) => U 214 | ): Prop = { 215 | 216 | val formulaNext = formula.nextFormula 217 | // test case id counter / generator 218 | val testCaseIdCounter = new TestCaseIdCounter 219 | 220 | // Create a new streaming context per test case, and use it to create a new TestCaseContext 221 | // that will use TestInputStream from spark-testing-base to create new input and output 222 | // dstreams, and register a foreachRDD action to evaluate the formula 223 | Prop.forAllNoShrink (g1, optGenToGenOpt(g2)) { (testCase1: SSeq[I1], testCaseOpt2: Option[SSeq[I2]]) => 224 | // Setup new test case 225 | val testCaseId : TestCaseId = testCaseIdCounter.nextId() 226 | // create, start and stop context for each test case 227 | // create a fresh streaming context for this test case, and pass it unstarted to 228 | // a new TestCaseContext, which will setup the streams and actions, and start the streaming context 229 | val freshSsc = buildFreshStreamingContext() 230 | val testCaseContext = 231 | new TestCaseContext[I1,I2,O1,O2,U](testCase1, testCaseOpt2, 232 | gt1, gtOpt2, 233 | formulaNext, atomsAdapter)( 234 | freshSsc, parallelism, catchRDDs, maxNumberBatchesPerTestCase) 235 | logger.warn(s"starting test case $testCaseId") 236 | testCaseContext.init() 237 | /* 238 | * We have to ensure we block until the StreamingContext is stopped before creating a new StreamingContext 239 | * for the next test case, otherwise we get "java.lang.IllegalStateException: Only one StreamingContext may 240 | * be started in this JVM" (see SPARK-2243) 241 | * */ 242 | testCaseContext.waitForCompletion() 243 | // In case there was a timeout waiting for test execution, again to avoid more than one StreamingContext 244 | // note this does nothing if it was already stopped 245 | testCaseContext.stop() 246 | 247 | val testCaseResult = testCaseContext.result 248 | 249 | // Note: ScalaCheck will show the correct test case that caused the counterexample 250 | // because only the test case that generated that counterexample will fail. Anyway we could 251 | // use testCaseResult.mapMessage here to add testCaseDstream to the message of 252 | // testCaseResult if we needed it 253 | logger.warn(s"finished test case $testCaseId with result $testCaseResult") 254 | testCaseResult match { 255 | case Prop.True => Prop.passed 256 | case Prop.Proof => Prop.proved 257 | case Prop.False => Prop.falsified 258 | case Prop.Undecided => Prop.passed //Prop.undecided FIXME make configurable 259 | case Prop.Exception(e) => Prop.exception(e) 260 | } 261 | } 262 | } 263 | } 264 | 265 | class PropExecutionException(msg : String) 266 | extends RuntimeException(msg) 267 | object TestCaseTimeoutException { 268 | def apply(batchInterval: Long, batchCompletionTimeout: Long): TestCaseTimeoutException = { 269 | val msg = s"Timeout to complete batch after ${batchCompletionTimeout} ms, expected batch interval was ${batchInterval} ms" 270 | new TestCaseTimeoutException(msg = msg) 271 | } 272 | } 273 | case class TestCaseTimeoutException(msg: String) 274 | extends PropExecutionException(msg) 275 | 276 | object TestCaseContext { 277 | @transient private val logger = LoggerFactory.getLogger("TestCaseContext") 278 | 279 | // Constants used for printing a sample of the generated values for each batch 280 | private val msgHeader = "-"*43 281 | private val numSampleRecords = 4 282 | 283 | /* A single execution context for all test cases because we are limited to 284 | * one test case per JVM due to the 1 streaming context per JVM limitation 285 | * of SPARK-2243 286 | */ 287 | implicit val ec = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(1)) 288 | 289 | /** Print some elements of dstream to stdout 290 | */ 291 | private def printDStream[A](dstream: DStream[A], dstreamName: String): Unit = 292 | dstream.foreachRDD { (rdd, time) => 293 | if (!rdd.isEmpty()) { 294 | val numRecords = rdd.count 295 | println(s"""${msgHeader} 296 | Time: ${time} - ${dstreamName} (${numRecords} records) 297 | ${msgHeader}""") 298 | if (numRecords > 0) println(rdd.take(numSampleRecords).mkString(lineSeparator)) 299 | println("...") 300 | } 301 | } 302 | /** Launch a trivial action on dstream to force its computation 303 | */ 304 | private def touchDStream[A](dstream: DStream[A]): Unit = 305 | dstream.foreachRDD {rdd => {}} 306 | 307 | /* Note https://issues.apache.org/jira/browse/SPARK-10722 causes 308 | * "Uncaught exception: RDDBlockId not found in driver-heartbeater", 309 | * "java.lang.ClassNotFoundException: org.apache.spark.storage.RDDBlockId", 310 | * "ERROR TaskSchedulerImpl: Lost executor driver on localhost: Executor heartbeat timed out" and 311 | * "ExecutorLostFailure (executor driver exited caused by one of the running tasks) Reason: Executor heartbeat timed out" 312 | * but that reported as fixed in v 1.6.2 in SPARK-10722 313 | */ 314 | private def getBatchForNow[T](ds: DStream[T], time: SparkTime, catchRDD: Boolean): RDD[T] = { 315 | val batch = ds.slice(time, time).head 316 | if (catchRDD && batch.getStorageLevel == StorageLevel.NONE) batch.cache else batch 317 | } 318 | } 319 | /** 320 | * Objects of this class define the DStreams involved in the test case execution 321 | * from the test case and the provided DStream, maintain a formula object and 322 | * register an action to evaluate that formula. 323 | * 324 | * ssc should be fresh and not started yet 325 | */ 326 | class TestCaseContext[I1:ClassTag,I2:ClassTag,O1:ClassTag,O2:ClassTag, U]( 327 | @transient private val testCase1: SSeq[I1], 328 | @transient private val testCaseOpt2: Option[SSeq[I2]], 329 | gt1: (Option[DStream[I1]], Option[DStream[I2]]) => DStream[O1], 330 | gtOpt2: Option[(Option[DStream[I1]], Option[DStream[I2]]) => DStream[O2]], 331 | @transient private val formulaNext: NextFormula[U], 332 | @transient private val atomsAdapter: (Option[RDD[I1]], Option[RDD[I2]], Option[RDD[O1]], Option[RDD[O2]]) => U) 333 | (@transient private val ssc : StreamingContext, 334 | @transient private val parallelism : Parallelism, @transient private val catchRDDs: Boolean, 335 | @transient private val maxNumberBatches: Int) 336 | extends Serializable { 337 | /* 338 | * With the constructor types we enforce having at least a non empty test case and a 339 | * non empty transformation. That assumption simplifies the code are 340 | * it is not limiting, because you need at least one input and one transformation 341 | * in order to be testing something! 342 | */ 343 | // TODO add assertions on Option compatibilities 344 | 345 | import TestCaseContext.{logger,printDStream,touchDStream,ec,getBatchForNow} 346 | 347 | /** Current value of the formula we are evaluating in this test case contexts 348 | */ 349 | @transient @volatile private var currFormula: NextFormula[U] = formulaNext 350 | // Number of batches in the test case that are waiting to be executed 351 | @transient @volatile private var numRemaningBatches: Int = 352 | (testCase1.length) max (testCaseOpt2.fold(0){_.length}) 353 | 354 | /** Whether the streaming context has started or not 355 | * */ 356 | private var started = false 357 | 358 | // have to create this here instead of in init() because otherwise I don't know how 359 | // to access the batch interval of the streaming context 360 | @transient private val inputDStream1 = 361 | new TestInputStream[I1](ssc.sparkContext, ssc, testCase1, parallelism.numSlices) 362 | // batch interval of the streaming context 363 | @transient private val batchInterval = inputDStream1.slideDuration.milliseconds 364 | 365 | def init(): Unit = this.synchronized { 366 | // ----------------------------------- 367 | // create input and output DStreams 368 | @transient val inputDStreamOpt1 = Some(inputDStream1) 369 | @transient val inputDStreamOpt2 = testCaseOpt2.map{ 370 | new TestInputStream[I2](ssc.sparkContext, ssc, _, parallelism.numSlices)} 371 | printDStream(inputDStream1, "InputDStream1") 372 | inputDStreamOpt2.foreach{printDStream(_, "InputDStream2")} 373 | 374 | // note we although we do access these transformed DStream, we only access them in slice(), 375 | // so we need some trivial action on transformedStream1 or we get org.apache.spark.SparkException: 376 | // org.apache.spark.streaming.dstream.MappedDStream@459bd6af has not been initialized) 377 | @transient val transformedStream1 = gt1(inputDStreamOpt1, inputDStreamOpt2) 378 | @transient val transformedStreamOpt2 = gtOpt2.map{_.apply(inputDStreamOpt1, inputDStreamOpt2)} 379 | touchDStream(transformedStream1) 380 | transformedStreamOpt2.foreach{touchDStream _} 381 | 382 | // ----------------------------------- 383 | // Register actions to evaluate the formula 384 | // volatile as those are read both from the foreachRDD below and the Prop.forall below 385 | // - only foreachRDD writes to currFormula 386 | // - DStreamTLProperty.forAllDStreamOption reads currFormula 387 | // thus using "the cheap read-write lock trick" https://www.ibm.com/developerworks/java/library/j-jtp06197/ 388 | val testCaseStateLock = new Serializable{} 389 | inputDStream1.foreachRDD { (inputBatch1, time) => 390 | // NOTE: batch cannot be completed until this code finishes, use 391 | // future if needed to avoid blocking the batch completion 392 | testCaseStateLock.synchronized { 393 | if ((currFormula.result.isEmpty) && (numRemaningBatches > 0)) { 394 | /* Note a new TestCaseContext with its own currFormula initialized to formulaNext 395 | * is created for each test case, but for each TestCaseContext currFormula 396 | * only gets to solved state once. 397 | * */ 398 | Try { 399 | val inputBatchOpt1 = Some(if (catchRDDs) inputBatch1.cache else inputBatch1) 400 | val inputBatchOpt2 = inputDStreamOpt2.map(getBatchForNow(_, time, catchRDDs)) 401 | val outputBatchOpt1 = Some(getBatchForNow(transformedStream1, time, catchRDDs)) 402 | val outputBatchOpt2 = transformedStreamOpt2.map(getBatchForNow(_, time, catchRDDs)) 403 | val adaptedAtoms = atomsAdapter(inputBatchOpt1, inputBatchOpt2, outputBatchOpt1, outputBatchOpt2) 404 | currFormula = currFormula.consume(Time(time.milliseconds))(adaptedAtoms) 405 | numRemaningBatches = numRemaningBatches - 1 406 | } recover { case throwable => 407 | // if there was some problem solving the assertions 408 | // then we solve the formula with an exception 409 | currFormula = Solved(Prop.Exception(throwable)) 410 | } 411 | if (currFormula.result.isDefined || numRemaningBatches <= 0) { 412 | Future { this.stop() } 413 | } 414 | } 415 | } 416 | } 417 | 418 | // ----------------------------------- 419 | // now that everything is ready we start the streaming context 420 | ssc.start() 421 | started = true 422 | } 423 | 424 | /** Blocks the caller until this test case context execution completes. 425 | * The test case stops itself as soon as the formula is resolved, 426 | * in order to fail fast in the first micro-batch that finds a counterexample 427 | * the next batch for the DStreams associated 428 | * to this object is completed. This way we can fail faster if a single 429 | * batch is too slow, instead of waiting for a long time for the whole 430 | * streaming context with ssc.awaitTerminationOrTimeout() 431 | * 432 | * @return true iff the formula for this test case context is resolved, 433 | * otherwise return false 434 | */ 435 | def waitForCompletion(): Unit = { 436 | this.ssc.awaitTerminationOrTimeout(batchInterval * (maxNumberBatches*1.3).ceil.toInt) 437 | } 438 | 439 | /** @return The result of the execution of this test case, or Prop.Undecided 440 | * if the result is inconclusive (e.g. because the test case it's not 441 | * long enough), or yet unknown. 442 | * */ 443 | def result: Prop.Status = currFormula.result.getOrElse(Prop.Undecided) 444 | 445 | /** Stops the internal streaming context, if it is running 446 | */ 447 | def stop() : Unit = this.synchronized { 448 | if (started) { 449 | Try { 450 | logger.warn("stopping test Spark Streaming context") 451 | ssc.stop(stopSparkContext = false, stopGracefully = true) 452 | Thread.sleep(1000L) 453 | ssc.stop(stopSparkContext = false, stopGracefully = false) 454 | } recover { 455 | case _ => { 456 | logger.warn("second attempt forcing stop of test Spark Streaming context") 457 | ssc.stop(stopSparkContext = false, stopGracefully = false) 458 | } 459 | } 460 | started = false 461 | } 462 | } 463 | } 464 | -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/prop/tl/Formula.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.prop.tl 2 | 3 | import org.specs2.execute.Result 4 | import org.scalacheck.Prop 5 | 6 | import scala.collection.parallel.ParSeq 7 | 8 | import scalaz.syntax.std.boolean._ 9 | import scalaz.syntax.traverse._ 10 | import scalaz.std.list._ 11 | import scalaz.std.option._ 12 | 13 | import scala.annotation.tailrec 14 | import scala.language.{postfixOps,implicitConversions} 15 | object Formula { 16 | /** More succinct notation for timeouts when combined with TimeoutMissingFormula.on() 17 | */ 18 | implicit def intToTimeout(t : Int) : Timeout = Timeout(t) 19 | /** Completes a TimeoutMissingFormula when a timeout is implicitly available 20 | */ 21 | implicit def timeoutMissingFormulaToFormula[T](f : TimeoutMissingFormula[T]) 22 | (implicit t : Timeout) : Formula[T] = f.on(t) 23 | /** @return a formula where the result of applying letterToResult to the 24 | * current letter must hold now 25 | */ 26 | implicit def resultFunToNow[T, R <% Result](letterToResult : T => R): BindNext[T] = 27 | now(letterToResult andThen implicitly[Function[R, Result]]) 28 | 29 | /** @return a formula where the result of applying letterToStatus to the 30 | * current letter must hold now 31 | */ 32 | implicit def statusFunToNow[T](letterToStatus: T => Prop.Status): BindNext[T] = 33 | nowS(letterToStatus) 34 | 35 | /** More succinct notation for BindNext formulas 36 | */ 37 | implicit def atomsConsumerToBindNext[T](atomsConsumer: T => Formula[T]): BindNext[T] = 38 | BindNext.fromAtomsConsumer(atomsConsumer) 39 | 40 | /** Convert a Specs2 Result into a ScalaCheck Prop.Status 41 | * See 42 | * - trait Status: https://github.com/rickynils/scalacheck/blob/1.12.2/src/main/scala/org/scalacheck/Prop.scala 43 | * - subclasses of Result: https://etorreborre.github.io/specs2/api/SPECS2-3.6.2/index.html#org.specs2.execute.Result 44 | * and https://etorreborre.github.io/specs2/guide/SPECS2-3.6.2/org.specs2.guide.StandardResults.html 45 | * */ 46 | @tailrec 47 | def resultToPropStatus(result : Result) : Prop.Status = result match { 48 | case err : org.specs2.execute.Error => Prop.Exception(err.t) 49 | case _ : org.specs2.execute.Failure => Prop.False 50 | case _ : org.specs2.execute.Pending => Prop.Undecided 51 | case _ : org.specs2.execute.Skipped => Prop.Undecided 52 | /* Prop.True is the same as passed, see lazy val passed 53 | * at https://github.com/rickynils/scalacheck/blob/1.12.2/src/main/scala/org/scalacheck/Prop.scala 54 | * TOOD: use Prop.Proof instead? 55 | */ 56 | case _ : org.specs2.execute.Success => Prop.True 57 | case dec : org.specs2.execute.DecoratedResult[_] => resultToPropStatus(dec.result) 58 | case _ => Prop.Undecided 59 | } 60 | 61 | /* For overloads the criteria is: 62 | * - avoid overload for Function, because Scala forbids them 63 | * - I haven't been able to make the techniques in http://stackoverflow.com/questions/3307427/scala-double-definition-2-methods-have-the-same-type-erasure 64 | * and http://stackoverflow.com/questions/17841993/double-definition-error-despite-different-parameter-types to work. Neither Scalaz type 65 | * unions. The main goal was defining an overload like Formula.always for and argument of type (T) => Result vs argument Formula[T] when using with 66 | * formulas defined with the case syntax https://groups.google.com/forum/#!topic/scala-user/rkau5IcuH48 67 | * - Shapeless could be an option to explore in the future, although it introduces a complex 68 | * dependency, specially for Scala 2.10 that doesn't have macros by default. Could explore this in the future 69 | * - When a overload is required we'll use the following strategy to avoid it: 70 | * * For overloads for functions to Result, Prop.Status, and Formula, replace overloads for pattern f 71 | * for T => Result, fS for T => Prop.Status, and fF for T => Formula[T], see example for at() 72 | * * For overloads for the 3 functions in the previous item, plus for Formula[T], use f for the Formula[T] 73 | * and T => Formula[T], and then for fR for T => R <% Result, and fS for T => Prop.Status, and fF for T => Formula[T]. 74 | * The result version has two version for T => Formula[T] because the overload doesn't works ok for functions defined 75 | * with the case syntax (see https://groups.google.com/forum/#!topic/scala-user/rkau5IcuH48). See example for always() 76 | * */ 77 | 78 | /** @return a formula where the result of applying to the current letter 79 | * the projection proj and then assertion must hold now 80 | */ 81 | def at[T, A, R <% Result](proj : (T) => A)(assertion : A => R): Formula[T] = 82 | now(proj andThen assertion andThen implicitly[Function[R, Result]]) 83 | 84 | /** @return a formula where the result of applying to the current tletter 85 | * the projection proj and then assertion must hold now 86 | */ 87 | def atS[T, A](proj : (T) => A)(assertion: A => Prop.Status): Formula[T] = 88 | nowS(assertion compose proj) 89 | 90 | /** @return a formula where the result of applying to the current tletter 91 | * the projection proj and then assertion must hold in the next 92 | * instant 93 | */ 94 | def atF[T, A](proj : (T) => A)(atomsConsumer : A => Formula[T]): Formula[T] = 95 | next(atomsConsumer compose proj) 96 | 97 | // Factories for non temporal connectives: note these act as clever constructors 98 | // for Or and And 99 | def or[T](phis: Formula[T]*): Formula[T] = 100 | if (phis.length == 0) Solved(Prop.False) 101 | else if (phis.length == 1) phis(0) 102 | else Or(phis:_*) 103 | def and[T](phis: Formula[T]*): Formula[T] = 104 | if (phis.length == 0) Solved(Prop.True) 105 | else if (phis.length == 1) phis(0) 106 | else And(phis:_*) 107 | 108 | // Factories for temporal connectives 109 | def next[T](phi: Formula[T]): Formula[T] = Next(phi) 110 | /** @return an application of BindNext to letterToFormula: the result of applying 111 | * letterToFormula to the current letter must hold in the next instant 112 | * */ 113 | def next[T](letterToFormula: T => Formula[T]): Formula[T] = BindNext.fromAtomsConsumer(letterToFormula) 114 | /** @return an application of BindNext to letterToFormula: the result of applying 115 | * letterToFormula to the current letter must hold in the next instant 116 | * */ 117 | def nextF[T](letterToFormula: T => Formula[T]): Formula[T] = next(letterToFormula) 118 | /** @return an application of BindNext to letterToFormula: the result of applying 119 | * letterToFormula to the current letter must hold in the next instant 120 | * */ 121 | def nextTime[T](letterToFormula: (T, Time) => Formula[T]): Formula[T] = 122 | BindNext.fromAtomsTimeConsumer(letterToFormula) 123 | 124 | /** @return the result of applying next to phi the number of 125 | * times specified 126 | * */ 127 | def next[T](times: Int)(phi: Formula[T]): Formula[T] = { 128 | require(times >= 0, s"times should be >=0, found $times") 129 | (1 to times).foldLeft(phi) { (f, _) => new Next[T](f) } 130 | } 131 | 132 | /* NOTE the Scaladoc description for the variants of now() are true without a 133 | next because Result corresponds to a timeless formula, and because NextFormula.consume() 134 | leaves the formula in a solved state without a need to consume any additional letter 135 | after the first one 136 | */ 137 | /** @return a formula where the result of applying letterToResult to the 138 | * current letter must hold now 139 | */ 140 | def now[T](letterToResult: T => Result): BindNext[T] = BindNext(letterToResult) 141 | /** @return a formula where the result of applying letterToResult to the 142 | * current letter must hold now 143 | */ 144 | def nowTime[T](letterToResult: (T, Time) => Result): BindNext[T] = BindNext(letterToResult) 145 | 146 | /** @return a formula where the result of applying letterToStatus to the 147 | * current letter must hold now 148 | */ 149 | def nowS[T](letterToStatus: T => Prop.Status): BindNext[T] = BindNext.fromStatusFun(letterToStatus) 150 | /** @return a formula where the result of applying letterToStatus to the 151 | * current letter must hold now 152 | */ 153 | def nowTimeS[T](letterToStatus: (T, Time) => Prop.Status): BindNext[T] = BindNext.fromStatusTimeFun(letterToStatus) 154 | 155 | def eventually[T](phi: Formula[T]): TimeoutMissingFormula[T] = 156 | new TimeoutMissingFormula[T](Eventually(phi, _)) 157 | /** @return a formula where eventually the result of applying letterToFormula to the 158 | * current letter must hold in the next instant 159 | */ 160 | def eventually[T](letterToFormula: T => Formula[T]): TimeoutMissingFormula[T] = 161 | eventuallyF(letterToFormula) 162 | /** @return a formula where eventually the result of applying letterToResult to the 163 | * current letter must hold now 164 | */ 165 | def eventuallyR[T](letterToResult: T => Result): TimeoutMissingFormula[T] = 166 | eventually(now(letterToResult)) 167 | 168 | /** @return a formula where eventually the result of applying letterToStatus to the 169 | * current letter must hold now 170 | */ 171 | def eventuallyS[T](letterToStatus: T => Prop.Status): TimeoutMissingFormula[T] = 172 | eventually(nowS(letterToStatus)) 173 | /** @return a formula where eventually the result of applying letterToFormula to the 174 | * current letter must hold in the next instant 175 | */ 176 | def eventuallyF[T](letterToFormula: T => Formula[T]): TimeoutMissingFormula[T] = 177 | eventually(next(letterToFormula)) 178 | 179 | /** Alias of eventually that can be used when there is a name class, for example 180 | * with EventuallyMatchers.eventually 181 | * */ 182 | def later[T](phi: Formula[T]) = eventually(phi) 183 | /** Alias of eventually that can be used when there is a name class, for example 184 | * with EventuallyMatchers.eventually 185 | * */ 186 | def later[T](assertion: T => Formula[T]) = eventually(assertion) 187 | /** Alias of eventually that can be used when there is a name class, for example 188 | * with EventuallyMatchers.eventually 189 | * */ 190 | def laterR[T](assertion: T => Result) = eventuallyR(assertion) 191 | /** Alias of eventually that can be used when there is a name class, for example 192 | * with EventuallyMatchers.eventually 193 | * */ 194 | def laterS[T](assertion: T => Prop.Status) = eventuallyS(assertion) 195 | /** Alias of eventually that can be used when there is a name class, for example 196 | * with EventuallyMatchers.eventually 197 | * */ 198 | def laterF[T](assertion: T => Formula[T]) = eventuallyF(assertion) 199 | 200 | def always[T](phi: Formula[T]) = new TimeoutMissingFormula[T](Always(phi, _)) 201 | /** @return a formula where always the result of applying letterToFormula to the 202 | * current letter must hold in the next instant 203 | */ 204 | def always[T](letterToFormula: T => Formula[T]): TimeoutMissingFormula[T] = 205 | alwaysF[T](letterToFormula) 206 | /** @return a formula where always the result of applying letterToResult to the 207 | * current letter must hold now 208 | */ 209 | def alwaysR[T](letterToResult: T => Result): TimeoutMissingFormula[T] = 210 | always(now(letterToResult)) 211 | /** @return a formula where always the result of applying letterToStatus to the 212 | * current letter must hold now 213 | */ 214 | def alwaysS[T](letterToStatus: T => Prop.Status): TimeoutMissingFormula[T] = 215 | always(nowS(letterToStatus)) 216 | /** @return a formula where always the result of applying letterToFormula to the 217 | * current letter must hold in the next instant 218 | */ 219 | def alwaysF[T](letterToFormula: T => Formula[T]): TimeoutMissingFormula[T] = 220 | always(next(letterToFormula)) 221 | } 222 | 223 | // using trait for the root of the AGT as recommended in http://twitter.github.io/effectivescala/ 224 | sealed trait Formula[T] 225 | extends Serializable { 226 | import Formula._ 227 | 228 | def safeWordLength: Option[Timeout] 229 | def nextFormula: NextFormula[T] 230 | 231 | // non temporal builder methods 232 | def unary_! = Not(this) 233 | def or(phi2 : Formula[T]) = Or(this, phi2) 234 | def and(phi2 : Formula[T]) = And(this, phi2) 235 | def ==>(phi2 : Formula[T]) = Implies(this, phi2) 236 | 237 | // temporal builder methods: next, eventually and always are methods of the Formula companion object 238 | def until(phi2 : Formula[T]): TimeoutMissingFormula[T] = new TimeoutMissingFormula[T](Until(this, phi2, _)) 239 | /** @return a formula where this formula happens until the result 240 | * of applying letterToFormula to the current letter holds 241 | * in the next instant 242 | */ 243 | def until(letterToFormula : T => Formula[T]): TimeoutMissingFormula[T] = this.untilF(letterToFormula) 244 | /** @return a formula where this formula happens until the result 245 | * of applying letterToResult to the current letter holds 246 | */ 247 | def untilR(letterToResult : T => Result): TimeoutMissingFormula[T] = this.until(now(letterToResult)) 248 | /** @return a formula where this formula happens until the result 249 | * of applying letterToStatus to the current letter holds 250 | */ 251 | def untilS(letterToStatus : T => Prop.Status): TimeoutMissingFormula[T] = 252 | this.until(nowS(letterToStatus)) 253 | /** @return a formula where this formula happens until the result 254 | * of applying letterToFormula to the current letter holds 255 | * in the next instant 256 | */ 257 | def untilF(letterToFormula : T => Formula[T]): TimeoutMissingFormula[T] = 258 | this.until(next(letterToFormula)) 259 | 260 | def release(phi2 : Formula[T]): TimeoutMissingFormula[T] = new TimeoutMissingFormula[T](Release(this, phi2, _)) 261 | /** @return a formula where this formula releases the result 262 | * of applying letterToFormula to the current letter from 263 | * holding in the next instant 264 | */ 265 | def release(letterToFormula : T => Formula[T]): TimeoutMissingFormula[T] = this.releaseF(letterToFormula) 266 | /** @return a formula where this formula releases the result 267 | * of applying letterToResult to the current letter from 268 | * holding now 269 | */ 270 | def releaseR(letterToResult : T => Result): TimeoutMissingFormula[T] = this.release(now(letterToResult)) 271 | /** @return a formula where this formula releases the result 272 | * of applying letterToStatus to the current letter from 273 | * holding now 274 | */ 275 | def releaseS(letterToStatus : T => Prop.Status): TimeoutMissingFormula[T] = 276 | this.release(nowS(letterToStatus)) 277 | /** @return a formula where this formula releases the result 278 | * of applying letterToFormula to the current letter from 279 | * holding in the next instant 280 | */ 281 | def releaseF(letterToFormula : T => Formula[T]): TimeoutMissingFormula[T] = 282 | this.release(next(letterToFormula)) 283 | } 284 | 285 | /** Restricted class of formulas that are in a form suitable for the 286 | * formula evaluation procedure 287 | */ 288 | sealed trait NextFormula[T] 289 | extends Formula[T] { 290 | override def nextFormula = this 291 | 292 | /** @return Option.Some if this formula is resolved, and Option.None 293 | * if it is still pending resolution when some additional values 294 | * of type T corresponding to more time instants are provided with 295 | * a call to consume() 296 | * */ 297 | def result : Option[Prop.Status] 298 | 299 | /** @return a new formula resulting from progressing in the evaluation 300 | * of this formula by consuming the new values atoms for the atomic 301 | * propositions corresponding to the values of the element of the universe 302 | * at a new instant of time time. This corresponds to the notion of "letter simplification" 303 | * in the paper 304 | */ 305 | def consume(time: Time)(atoms: T): NextFormula[T] 306 | } 307 | 308 | /** Resolved formulas 309 | * */ 310 | object Solved { 311 | def apply[T](result: Result) = ofResult[T](result) 312 | // these are needed to resolve ambiguities with apply 313 | def ofResult[T](result: Result): Solved[T] = new Solved[T](Formula.resultToPropStatus(result)) 314 | def ofStatus[T](status : Prop.Status): Solved[T] = Solved(status) 315 | } 316 | // see https://github.com/rickynils/scalacheck/blob/1.12.2/src/main/scala/org/scalacheck/Prop.scala 317 | case class Solved[T](status : Prop.Status) extends NextFormula[T] { 318 | override def safeWordLength = Some(Timeout(0)) 319 | override def result = Some(status) 320 | // do no raise an exception in call to consume, because with NextOr we will 321 | // keep undecided prop values until the rest of the formula in unraveled 322 | override def consume(time: Time)(atoms : T) = this 323 | } 324 | 325 | /** This class adds information to the time and atom consumption functions 326 | * used in Now, to enable a partial implementation of safeWordLength 327 | * */ 328 | abstract class TimedAtomsConsumer[T](fun: Time => Function[T, Formula[T]]) 329 | extends Function[Time, Function[T, Formula[T]]]{ 330 | 331 | override def apply(time: Time): (T => Formula[T]) = fun(time) 332 | /** @return false iff fun always returns a Solved formula 333 | * */ 334 | def returnsDynamicFormula: Boolean 335 | } 336 | class StaticTimedAtomsConsumer[T](fun: Time => Function[T, Formula[T]]) extends TimedAtomsConsumer(fun) { 337 | override def returnsDynamicFormula = false 338 | } 339 | class DynamicTimedAtomsConsumer[T](fun: Time => Function[T, Formula[T]]) extends TimedAtomsConsumer(fun) { 340 | override def returnsDynamicFormula = true 341 | } 342 | /** Formulas that have to be resolved now, which correspond to atomic proposition 343 | * as functions from the current state of the system to Specs2 assertions. 344 | * Note this also includes top / true and bottom / false as constant functions 345 | */ 346 | object BindNext { 347 | /* For now going for avoiding the type erasure problem, TODO check more sophisticated 348 | * solutions like http://hacking-scala.org/post/73854628325/advanced-type-constraints-with-type-classes 349 | * or http://hacking-scala.org/post/73854628325/advanced-type-constraints-with-type-classes based or 350 | * using ClassTag. Also why is there no conflict with the companion apply? 351 | */ 352 | def fromAtomsConsumer[T](atomsConsumer: T => Formula[T]): BindNext[T] = 353 | new BindNext(new DynamicTimedAtomsConsumer(Function.const(atomsConsumer))) 354 | def fromAtomsTimeConsumer[T](atomsTimeConsumer: (T, Time) => Formula[T]): BindNext[T] = 355 | new BindNext(new DynamicTimedAtomsConsumer(time => atoms => atomsTimeConsumer(atoms, time))) 356 | def fromStatusFun[T](atomsToStatus: T => Prop.Status): BindNext[T] = 357 | new BindNext(new StaticTimedAtomsConsumer[T](Function.const(atomsToStatus andThen Solved.ofStatus _))) 358 | def fromStatusTimeFun[T](atomsTimeToStatus: (T, Time) => Prop.Status): BindNext[T] = 359 | new BindNext(new StaticTimedAtomsConsumer(time => atoms => Solved.ofStatus(atomsTimeToStatus(atoms, time)))) 360 | def apply[T, R <% Result](atomsToResult: T => R): BindNext[T] = 361 | new BindNext(new StaticTimedAtomsConsumer[T]( 362 | Function.const(atomsToResult andThen implicitly[Function[R,Result]] andThen Solved.ofResult _))) 363 | def apply[T, R <% Result](atomsTimeToResult: (T, Time) => R): BindNext[T] = 364 | new BindNext(new StaticTimedAtomsConsumer(time => atoms => Solved.ofResult(atomsTimeToResult(atoms, time)))) 365 | } 366 | 367 | case class BindNext[T](timedAtomsConsumer: TimedAtomsConsumer[T]) 368 | extends NextFormula[T] { 369 | /* Note the case class structural equality gives an implementation 370 | * for equals equivalent to the one below, as a Function is only 371 | * equal to references to the same function, which corresponds to 372 | * intensional function equality. That is the only thing that makes 373 | * sense if we don't have a deduction system that is able to check 374 | * extensional function equality 375 | * 376 | override def equals(other : Any) : Boolean = 377 | other match { 378 | case that : BindNext[T] => that eq this 379 | case _ => false 380 | } 381 | * 382 | */ 383 | // we cannot fully compute this statically, because the returned 384 | // formula depends on the input word, but TimedAtomsConsumer.returnsDynamicFormula 385 | // allows us to build a safe approximation 386 | override def safeWordLength = 387 | (!timedAtomsConsumer.returnsDynamicFormula) option Timeout(1) 388 | override def result = None 389 | override def consume(time: Time)(atoms: T) = 390 | timedAtomsConsumer(time)(atoms).nextFormula 391 | } 392 | case class Not[T](phi : Formula[T]) extends Formula[T] { 393 | override def safeWordLength = phi safeWordLength 394 | override def nextFormula: NextFormula[T] = new NextNot(phi.nextFormula) 395 | } 396 | 397 | class NextNot[T](phi : NextFormula[T]) extends Not[T](phi) with NextFormula[T] { 398 | override def result = None 399 | /** Note in the implementation of or we add Exception to the truth lattice, 400 | * which always absorbs other values to signal a test evaluation error 401 | * */ 402 | override def consume(time: Time)(atoms : T) = { 403 | val phiConsumed = phi.consume(time)(atoms) 404 | phiConsumed.result match { 405 | case Some(res) => 406 | Solved (res match { 407 | /* Prop.True is the same as passed, see lazy val passed 408 | * at https://github.com/rickynils/scalacheck/blob/1.12.2/src/main/scala/org/scalacheck/Prop.scala 409 | * TODO: use Prop.Proof instead? 410 | */ 411 | case Prop.True => Prop.False 412 | case Prop.Proof => Prop.False 413 | case Prop.False => Prop.True 414 | case Prop.Exception(_) => res 415 | case Prop.Undecided => Prop.Undecided 416 | }) 417 | case None => new NextNot(phiConsumed) 418 | } 419 | } 420 | } 421 | 422 | case class Or[T](phis : Formula[T]*) extends Formula[T] { 423 | override def safeWordLength = 424 | phis.map(_.safeWordLength) 425 | .toList.sequence 426 | .map(_.maxBy(_.instants)) 427 | 428 | override def nextFormula = NextOr(phis.map(_.nextFormula):_*) 429 | } 430 | object NextOr { 431 | def apply[T](phis: NextFormula[T]*): NextFormula[T] = if (phis.length == 1) phis(0) else new NextOr(phis:_*) 432 | } 433 | class NextOr[T](phis: ParSeq[NextFormula[T]]) extends NextBinaryOp[T](phis) { 434 | def this(seqPhis: NextFormula[T]*) { 435 | this(NextBinaryOp.parArgs[T](seqPhis:_*)) 436 | } 437 | /** @return the result of computing the or of s1 and s2 in 438 | * the lattice of truth values, adding Exception which always 439 | * absorbs other values to signal a test evaluation error 440 | */ 441 | override def apply(s1: Prop.Status, s2: Prop.Status): Prop.Status = 442 | (s1, s2) match { 443 | case (Prop.Exception(_), _) => s1 444 | case (_, Prop.Exception(_)) => s2 445 | case (Prop.True, _) => Prop.True 446 | case (Prop.Proof, _) => Prop.Proof 447 | case (Prop.Undecided, Prop.False) => Prop.Undecided 448 | case _ => s2 449 | } 450 | override protected def build(phis: ParSeq[NextFormula[T]]) = 451 | new NextOr(phis) 452 | override protected def isSolverStatus(status: Prop.Status) = 453 | (status == Prop.True) || (status == Prop.Proof) 454 | } 455 | 456 | case class And[T](phis : Formula[T]*) extends Formula[T] { 457 | override def safeWordLength = 458 | phis.map(_.safeWordLength) 459 | .toList.sequence 460 | .map(_.maxBy(_.instants)) 461 | override def nextFormula = NextAnd(phis.map(_.nextFormula):_*) 462 | } 463 | object NextAnd { 464 | def apply[T](phis: NextFormula[T]*): NextFormula[T] = if (phis.length == 1) phis(0) else new NextAnd(phis:_*) 465 | } 466 | class NextAnd[T](phis: ParSeq[NextFormula[T]]) extends NextBinaryOp[T](phis) { 467 | def this(seqPhis: NextFormula[T]*) { 468 | this(NextBinaryOp.parArgs[T](seqPhis:_*)) 469 | } 470 | /** @return the result of computing the and of s1 and s2 in 471 | * the lattice of truth values 472 | */ 473 | override def apply(s1: Prop.Status, s2: Prop.Status): Prop.Status = 474 | (s1, s2) match { 475 | case (Prop.Exception(_), _) => s1 476 | case (_, Prop.Exception(_)) => s2 477 | case (Prop.False, _) => Prop.False 478 | case (Prop.Undecided, Prop.False) => Prop.False 479 | case (Prop.Undecided, _) => Prop.Undecided 480 | case (Prop.True, _) => s2 481 | case (Prop.Proof, _) => s2 482 | } 483 | override protected def build(phis: ParSeq[NextFormula[T]]) = 484 | new NextAnd(phis) 485 | override protected def isSolverStatus(status: Prop.Status) = 486 | status == Prop.False 487 | } 488 | 489 | object NextBinaryOp { 490 | def parArgs[T](seqPhis: NextFormula[T]*): ParSeq[NextFormula[T]] = { 491 | // TODO could use seqPhis.par.tasksupport here 492 | // to configure parallelization details, see 493 | // http://docs.scala-lang.org/overviews/parallel-collections/configuration 494 | seqPhis.par 495 | } 496 | } 497 | /** Abstract the functionality of NextAnd and NextOr, which are binary 498 | * boolean operators that apply to a collection of formulas with a reduce() 499 | * */ 500 | abstract class NextBinaryOp[T](phis: ParSeq[NextFormula[T]]) 501 | extends Function2[Prop.Status, Prop.Status, Prop.Status] 502 | with NextFormula[T] { 503 | 504 | // TODO: consider replacing by getting the companion of the concrete subclass 505 | // following http://stackoverflow.com/questions/9172775/get-companion-object-of-class-by-given-generic-type-scala, a 506 | // or something in the line of scala.collection.generic.GenericCompanion (used e.g. in Seq.companion()), 507 | // and then calling apply to build 508 | protected def build(phis: ParSeq[NextFormula[T]]): NextFormula[T] 509 | 510 | /* return true if status solves this operator: e.g. Prop.True 511 | * or Prop.Proof resolve and or without evaluating anything else, 512 | * otherwise return false */ 513 | protected def isSolverStatus(status: Prop.Status): Boolean 514 | 515 | override def safeWordLength = 516 | phis.map(_.safeWordLength) 517 | .toList.sequence 518 | .map(_.maxBy(_.instants)) 519 | override def result = None 520 | override def consume(time: Time)(atoms : T) = { 521 | val (phisDefined, phisUndefined) = phis 522 | .map { _.consume(time)(atoms) } 523 | .partition { _.result.isDefined } 524 | val definedStatus = (! phisDefined.isEmpty) option { 525 | phisDefined 526 | .map { _.result.get } 527 | .reduce { apply(_, _) } 528 | } 529 | // short-circuit operator if possible. Note an edge case when all the phis 530 | // are defined after consuming the input, but we might still not have a 531 | // positive (true of proof) result 532 | if (definedStatus.isDefined && definedStatus.get.isInstanceOf[Prop.Exception]) 533 | Solved(definedStatus.get) 534 | else if ((definedStatus.isDefined && isSolverStatus(definedStatus.get)) 535 | || phisUndefined.size == 0) { 536 | Solved(definedStatus.getOrElse(Prop.Undecided)) 537 | } else { 538 | // if definedStatus is undecided keep it in case 539 | // the rest of the and is reduced to true later on 540 | val newPhis = definedStatus match { 541 | case Some(Prop.Undecided) => Solved[T](Prop.Undecided) +: phisUndefined 542 | case _ => phisUndefined 543 | } 544 | build(newPhis) 545 | } 546 | } 547 | } 548 | 549 | case class Implies[T](phi1 : Formula[T], phi2 : Formula[T]) extends Formula[T] { 550 | override def safeWordLength = for { 551 | safeLength1 <- phi1.safeWordLength 552 | safeLength2 <- phi2.safeWordLength 553 | } yield safeLength1 max safeLength2 554 | 555 | override def nextFormula = NextOr(new NextNot(phi1.nextFormula), phi2.nextFormula) 556 | } 557 | 558 | case class Next[T](phi : Formula[T]) extends Formula[T] { 559 | import Formula.intToTimeout 560 | override def safeWordLength = phi.safeWordLength.map(_ + 1) 561 | override def nextFormula = NextNext(phi.nextFormula) 562 | } 563 | object NextNext { 564 | def apply[T](phi: => NextFormula[T]): NextFormula[T] = new NextNext(phi) 565 | } 566 | class NextNext[T](_phi: => NextFormula[T]) extends NextFormula[T] { 567 | import Formula.intToTimeout 568 | 569 | private lazy val phi = _phi 570 | override def safeWordLength = phi.safeWordLength.map(_ + 1) 571 | 572 | override def result = None 573 | override def consume(time: Time)(atoms : T) = phi 574 | } 575 | 576 | case class Eventually[T](phi : Formula[T], t : Timeout) extends Formula[T] { 577 | require(t.instants >=1, s"timeout must be greater or equal than 1, found ${t}") 578 | 579 | import Formula.intToTimeout 580 | override def safeWordLength = phi.safeWordLength.map(_ + t - 1) 581 | 582 | override def nextFormula = { 583 | val nextPhi = phi.nextFormula 584 | if (t.instants <= 1) nextPhi 585 | // equivalent to paper formula assuming nt(C[phi]) = nt(C[nt(phi)]) 586 | else NextOr(nextPhi, NextNext(Eventually(nextPhi, t-1).nextFormula)) 587 | } 588 | } 589 | case class Always[T](phi : Formula[T], t : Timeout) extends Formula[T] { 590 | require(t.instants >=1, s"timeout must be greater or equal than 1, found ${t}") 591 | 592 | import Formula.intToTimeout 593 | override def safeWordLength = phi.safeWordLength.map(_ + t - 1) 594 | override def nextFormula = { 595 | val nextPhi = phi.nextFormula 596 | if (t.instants <= 1) nextPhi 597 | // equivalent to paper formula assuming nt(C[phi]) = nt(C[nt(phi)]) 598 | else NextAnd(nextPhi, NextNext(Always(nextPhi, t-1).nextFormula)) 599 | } 600 | } 601 | case class Until[T](phi1 : Formula[T], phi2 : Formula[T], t : Timeout) extends Formula[T] { 602 | require(t.instants >=1, s"timeout must be greater or equal than 1, found ${t}") 603 | 604 | import Formula.intToTimeout 605 | override def safeWordLength = for { 606 | safeLength1 <- phi1.safeWordLength 607 | safeLength2 <- phi2.safeWordLength 608 | } yield (safeLength1 max safeLength2) + t -1 609 | 610 | override def nextFormula = { 611 | val (nextPhi1, nextPhi2) = (phi1.nextFormula, phi2.nextFormula) 612 | if (t.instants <= 1) nextPhi2 613 | // equivalent to paper formula assuming nt(C[phi]) = nt(C[nt(phi)]) 614 | else NextOr(nextPhi2, 615 | NextAnd(nextPhi1, NextNext(Until(nextPhi1, nextPhi2, t-1).nextFormula))) 616 | } 617 | } 618 | case class Release[T](phi1 : Formula[T], phi2 : Formula[T], t : Timeout) extends Formula[T] { 619 | require(t.instants >=1, s"timeout must be greater or equal than 1, found ${t}") 620 | 621 | import Formula.intToTimeout 622 | override def safeWordLength = for { 623 | safeLength1 <- phi1.safeWordLength 624 | safeLength2 <- phi2.safeWordLength 625 | } yield (safeLength1 max safeLength2) + t -1 626 | 627 | override def nextFormula = { 628 | val (nextPhi1, nextPhi2) = (phi1.nextFormula, phi2.nextFormula) 629 | if (t.instants <= 1) NextAnd(nextPhi1, nextPhi2) 630 | // equivalent to paper formula assuming nt(C[phi]) = nt(C[nt(phi)]) 631 | else NextOr(NextAnd(nextPhi1, nextPhi2), 632 | NextAnd(nextPhi2, NextNext(Release(nextPhi1, nextPhi2, t-1).nextFormula))) 633 | 634 | } 635 | } 636 | 637 | /** Case class wrapping the number of instants / turns a temporal 638 | * operator has to be resolved (e.g. an the formula in an 639 | * eventually happening). 640 | * - A timeout of 1 means the operator has to be resolved now. 641 | * - A timeout of 0 means the operator fails to be resolved 642 | * 643 | * A type different to Int allows us to be more specified when requiring 644 | * implicit parameters in functions 645 | * */ 646 | case class Timeout(val instants : Int) extends Serializable { 647 | def +[T <% Timeout](t : T) = Timeout { instants + t.instants } 648 | def -[T <% Timeout](t : T) = Timeout { instants - t.instants } 649 | def max[T <% Timeout](t : T) = Timeout { math.max(instants, t.instants) } 650 | } 651 | 652 | /** This class is used in the builder methods in Formula and companion, 653 | * to express formulas with a timeout operator at root that is missing 654 | * its timeout, wich can be provided to complete the formula with the on() method 655 | */ 656 | class TimeoutMissingFormula[T](val toFormula : Timeout => Formula[T]) 657 | extends Serializable { 658 | 659 | def on(t : Timeout) = toFormula(t) 660 | /** Alias of on that can be used for obtaining a more readable spec, for 661 | * example combined with Formula.always() 662 | */ 663 | def during(t : Timeout) = on(t) 664 | } 665 | 666 | /** @param millis: number of milliseconds since January 1, 1970 UTC 667 | * */ 668 | case class Time(millis: Long) extends Serializable 669 | -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/spark/SharedSparkContext.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark 2 | 3 | import org.apache.spark._ 4 | 5 | import org.slf4j.LoggerFactory 6 | 7 | /** This trait can be used to share a Spark Context. The context is created 8 | * the first time sc() is called, and stopped when close() is called 9 | * */ 10 | trait SharedSparkContext 11 | extends Serializable 12 | with java.io.Closeable { 13 | 14 | // cannot use private[this] due to https://issues.scala-lang.org/browse/SI-8087 15 | //@transient private[this] val logger = Logger(LoggerFactory.getLogger("SharedSparkContext")) 16 | @transient private val logger = LoggerFactory.getLogger("SharedSparkContext") 17 | 18 | /** Override for custom config 19 | * */ 20 | def sparkMaster : String 21 | 22 | /** Override for custom config 23 | * */ 24 | def sparkAppName : String = "ScalaCheck Spark test" 25 | 26 | // lazy val so early definitions are not needed for subtyping 27 | @transient lazy val conf = new SparkConf().setMaster(sparkMaster).setAppName(sparkAppName) 28 | 29 | @transient protected[this] var _sc : Option[SparkContext] = None 30 | def sc() : SparkContext = { 31 | _sc.getOrElse { 32 | logger.warn("creating test Spark context") 33 | _sc = Some(new SparkContext(conf)) 34 | _sc.get 35 | } 36 | } 37 | 38 | override def close() : Unit = { 39 | _sc.foreach { sc => 40 | logger.warn("stopping test Spark context") 41 | sc.stop() 42 | // To avoid Akka rebinding to the same port, since it doesn't unbind immediately on shutdown 43 | System.clearProperty("spark.driver.port") 44 | } 45 | _sc = None 46 | } 47 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/spark/SharedSparkContextBeforeAfterAll.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark 2 | 3 | import org.specs2.specification.BeforeAfterAll 4 | 5 | import org.scalacheck.Gen 6 | import org.scalacheck.util.Buildable 7 | 8 | import org.apache.spark._ 9 | import org.apache.spark.rdd.RDD 10 | 11 | import scala.reflect.ClassTag 12 | 13 | /** Shares a Spark Context among all the test in a Specs2 suite, this trait is also eases the use 14 | * of Scalacheck with Spark, as if some tests are ScalaCheck properties, then the Spark context 15 | * is shared among all the generated test cases. 16 | * 17 | * Life cycle: a SparkContext is created beforeAll and closed afterAll 18 | * 19 | * The Spark context can be made explicitly available to all the tests, passing this.sc(), 20 | * or implicitly via an implicit val 21 | * 22 | * Also implicits are provided to convert ScalaCheck generators of Seq into generators of RDD, 23 | * and Seq into RDD 24 | * */ 25 | trait SharedSparkContextBeforeAfterAll 26 | extends BeforeAfterAll 27 | with SharedSparkContext { 28 | 29 | /** Force the creation of the Spark Context before any test 30 | */ 31 | override def beforeAll : Unit = { this.sc() } 32 | 33 | /** Close Spark Context after all tests 34 | */ 35 | override def afterAll : Unit = { super[SharedSparkContext].close() } 36 | 37 | /* Another option would be replacing BeforeAfterAll with an eager creation 38 | * of this.sc through making impSC not lazy, and cleaning up in AfterAll, 39 | * but I think the current solution has a more clear intent 40 | * */ 41 | /** Make implicitly available the SparkContext 42 | * */ 43 | @transient implicit lazy val impSC : SparkContext = this.sc() 44 | 45 | /** Number of partitions used to parallelize Seq values into RDDs, 46 | * override in subclass for custom parallelism 47 | */ 48 | def defaultParallelism: Int = 2 49 | 50 | /** Make implicitly available the value of parallelism finally set for this object 51 | * */ 52 | implicit lazy val parallelism = Parallelism(defaultParallelism) 53 | } 54 | 55 | /** Case class wrapping the number of partitions to use when parallelizing sequences 56 | * to Spark, a type different from Int allows us to be more specified when requiring 57 | * implicit parameters in functions 58 | * */ 59 | case class Parallelism(val numSlices : Int) extends Serializable 60 | -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/spark/streaming/SharedStreamingContext.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.streaming 2 | 3 | import org.apache.spark.streaming.{StreamingContext,Duration} 4 | 5 | import org.slf4j.LoggerFactory 6 | 7 | import scala.util.Try 8 | 9 | import es.ucm.fdi.sscheck.spark.SharedSparkContext 10 | 11 | trait SharedStreamingContext 12 | extends SharedSparkContext { 13 | 14 | // cannot use private[this] due to https://issues.scala-lang.org/browse/SI-8087 15 | // @transient private[this] val logger = Logger(LoggerFactory.getLogger("SharedStreamingContext")) 16 | @transient private val logger = LoggerFactory.getLogger("SharedStreamingContext") 17 | 18 | /** Override for custom config 19 | * */ 20 | def batchDuration : Duration 21 | 22 | // disabled by default because it is quite costly 23 | def enableCheckpointing : Boolean = false 24 | 25 | @transient protected[this] var _ssc : Option[StreamingContext] = None 26 | def ssc() : StreamingContext = { 27 | _ssc.getOrElse { 28 | // first force the creation of a SparkContext, if needed 29 | val __sc = sc() 30 | logger.warn(s"creating test Spark Streaming context") 31 | _ssc = Some(new StreamingContext(__sc, batchDuration)) 32 | if (enableCheckpointing) { 33 | val checkpointDir = Utils.createTempDir().toString 34 | logger.warn(s"configuring Spark Streaming checkpoint directory ${checkpointDir}") 35 | _ssc.get.checkpoint(checkpointDir) 36 | } 37 | _ssc.get 38 | } 39 | } 40 | 41 | /** Close the shared StreamingContext. NOTE the inherited SparkContext 42 | * is NOT closed, as often we would like to have different life cycles 43 | * for these two kinds of contexts: SparkContext is heavyweight and 44 | * StreamingContext is lightweight, and we cannot add additional computations 45 | * to a StreamingContext once it has started 46 | * */ 47 | override def close() : Unit = { 48 | close(stopSparkContext=false) 49 | } 50 | 51 | /** Close the shared StreamingContext, and optionally the inherited 52 | * SparkContext. Note if stopSparkContext is true then the inherited 53 | * SparkContext is closed, even if the shared StreamingContext was already closed 54 | * */ 55 | def close(stopSparkContext : Boolean = false) : Unit = { 56 | // stop _ssc with stopSparkContext=false and then stop _sc 57 | // with super[SharedSparkContext].close() if stopSparkContext 58 | // to respect logging and other additional actions 59 | _ssc.foreach { ssc => 60 | logger.warn("stopping test Spark Streaming context") 61 | Try { 62 | /* need to use stopGracefully=false for DStreamTLProperty.forAllDStream 63 | * to work ok in an spec with several props 64 | */ 65 | ssc.stop(stopSparkContext=false, stopGracefully=false) 66 | } recover { 67 | case _ => { 68 | logger.warn("second attempt forcing stop of test Spark Streaming context") 69 | ssc.stop(stopSparkContext=false, stopGracefully=false) 70 | } 71 | } 72 | _ssc = None 73 | } 74 | if (stopSparkContext) { 75 | super[SharedSparkContext].close() 76 | } 77 | } 78 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/spark/streaming/SharedStreamingContextBeforeAfterEach.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.streaming 2 | 3 | import org.specs2.specification.BeforeAfterEach 4 | 5 | import org.apache.spark.streaming.StreamingContext 6 | 7 | import es.ucm.fdi.sscheck.spark.SharedSparkContextBeforeAfterAll 8 | 9 | /** Shares a Spark Context among all the test in a Specs2 suite, and provides a StreamingContext 10 | * for each example a suite. Each example has its own StreamingContext because no transformations 11 | * nor actions can be added to a StreamingContext after it has started (see 12 | * [[https://spark.apache.org/docs/latest/streaming-programming-guide.html#initializing-streamingcontext points to remember]]). 13 | * On the other hand, a StreamingContext can be shared among all the test cases of a ScalaCheck property, 14 | * as the same property (hence the same Spark transformations and actions) is applied to all the test cases. 15 | * 16 | * Life cycle: a SparkContext is created beforeAll and closed afterAll. A StreamingContext 17 | * is created before each Specs2 example and closed after each example 18 | * 19 | * WARNING: when using in Specs2 specifications, [[https://etorreborre.github.io/specs2/guide/SPECS2-3.6.2/org.specs2.guide.Execution.html sequential execution]] 20 | * must be enabled to avoid having more than a single StreamingContext started in the same JVM 21 | * 22 | * Note the shared StreamingContext is not made implicitly available, as it happens 23 | * with SparkContext in SharedSparkContextBeforeAfterAll. This is due to the more 24 | * complex life cycle of streaming contexts, and the fact that they doesn't accept 25 | * registering more computations after they have started 26 | */ 27 | trait SharedStreamingContextBeforeAfterEach 28 | extends BeforeAfterEach 29 | with SharedSparkContextBeforeAfterAll 30 | with SharedStreamingContext { 31 | 32 | /** Force the creation of the StreamingContext before the test 33 | * */ 34 | override def before : Unit = { 35 | require(_ssc.isEmpty) 36 | this.ssc() 37 | } 38 | 39 | /** Close the StreamingContext after the test 40 | */ 41 | override def after : Unit = { 42 | // We don't require ! _ssc.isEmpty, as it 43 | // might have been already closed by a failing ScalaCheck Prop 44 | super[SharedStreamingContext].close() 45 | } 46 | 47 | /** Make implicitly available the StreamingContext 48 | * */ 49 | implicit def impSSC : StreamingContext = this.ssc() 50 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/spark/streaming/StreamingContextUtils.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.streaming 2 | 3 | import scala.concurrent._ 4 | import scala.concurrent.duration._ 5 | import scala.language.postfixOps 6 | import ExecutionContext.Implicits.global 7 | import org.apache.spark.streaming.StreamingContext 8 | import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverStarted, StreamingListenerBatchCompleted} 9 | 10 | // Inline alternative implementation based on SyncVar 11 | object StreamingContextUtils { 12 | /** This is a blocking call that awaits (with scala.concurrent.Await.result) for the receiver 13 | * of the input Streaming Context to complete start. This can be used to avoid sending data 14 | * to a receiver before it is ready. 15 | * */ 16 | def awaitUntilReceiverStarted(atMost : scala.concurrent.duration.Duration = 2 seconds) 17 | (ssc : StreamingContext): Unit = { 18 | val receiverStartedPromise = Promise[Unit] 19 | // val sv = new SyncVar[Unit] 20 | ssc.addStreamingListener(new StreamingListener { 21 | override def onReceiverStarted(receiverStarted: StreamingListenerReceiverStarted) : Unit = { 22 | receiverStartedPromise success Unit 23 | // sv.put(()) 24 | } 25 | }) 26 | // sv.get(atMost toMillis) 27 | Await.result(receiverStartedPromise.future, atMost) 28 | } 29 | 30 | /** This is a blocking call that awaits for completion of numBatches in ssc. 31 | * NOTE if a receiver is used awaitUntilReceiverStarted() should be called before 32 | * 33 | * @param atMost maximum amount of time to spend waiting for all the batches to complete 34 | * */ 35 | def awaitForNBatchesCompleted(numBatches : Int, atMost : scala.concurrent.duration.Duration = 30 seconds) 36 | (ssc : StreamingContext) : Unit = { 37 | val onBatchCompletedSyncVar = new SyncVar[Unit] 38 | ssc.addStreamingListener(new StreamingListener { 39 | override def onBatchCompleted(batchCompleted: StreamingListenerBatchCompleted) : Unit = { 40 | if (! onBatchCompletedSyncVar.isSet) { 41 | // note only this threads makes puts, so no problem with concurrency 42 | onBatchCompletedSyncVar.put(()) 43 | } 44 | } 45 | }) 46 | val waitingForBatches = Future { 47 | for (_ <- 1 to numBatches) { 48 | onBatchCompletedSyncVar.take() 49 | } 50 | } 51 | Await.result(waitingForBatches, atMost) 52 | } 53 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/spark/streaming/TestInputStream.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package es.ucm.fdi.sscheck.spark.streaming 18 | 19 | import org.apache.spark.streaming._ 20 | import org.apache.spark._ 21 | import org.apache.spark.rdd.RDD 22 | import org.apache.spark.SparkContext._ 23 | 24 | import scala.language.implicitConversions 25 | import scala.reflect.ClassTag 26 | import org.apache.spark.streaming.dstream.SscheckFriendlyInputDStream 27 | // import org.apache.spark.streaming.dstream.FriendlyInputDStream 28 | 29 | /** 30 | * Copied from spark-testing-base https://github.com/holdenk/spark-testing-base/blob/f66fd14b6a87efc09cd26c302740f3b0fb2585ae/src/main/1.3/scala/com/holdenkarau/spark/testing/TestInputStream.scala 31 | * Copied here to avoid conflicting version problems with spark-testing-base, see https://github.com/juanrh/sscheck/issues/36 32 | * This is an almost verbatim copy, just extending SscheckFriendlyInputDStream instead of FriendlyInputDStream, 33 | * to avoid naming conflicts 34 | * 35 | * This is a input stream just for the testsuites. This is equivalent to a checkpointable, 36 | * replayable, reliable message queue like Kafka. It requires a sequence as input, and 37 | * returns the i_th element at the i_th batch unde manual clock. 38 | * Based on TestInputStream class from TestSuiteBase in the Apache Spark project. 39 | */ 40 | class TestInputStream[T: ClassTag](@transient var sc: SparkContext, 41 | ssc_ : StreamingContext, input: Seq[Seq[T]], numPartitions: Int) 42 | extends SscheckFriendlyInputDStream[T](ssc_) { 43 | 44 | def start() {} 45 | 46 | def stop() {} 47 | 48 | def compute(validTime: Time): Option[RDD[T]] = { 49 | logInfo("Computing RDD for time " + validTime) 50 | val index = ((validTime - ourZeroTime) / slideDuration - 1).toInt 51 | val selectedInput = if (index < input.size) input(index) else Seq[T]() 52 | 53 | // lets us test cases where RDDs are not created 54 | if (selectedInput == null) { 55 | return None 56 | } 57 | 58 | val rdd = sc.makeRDD(selectedInput, numPartitions) 59 | logInfo("Created RDD " + rdd.id + " with " + selectedInput) 60 | Some(rdd) 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/scala/es/ucm/fdi/sscheck/spark/streaming/Utils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package es.ucm.fdi.sscheck.spark.streaming 19 | 20 | import java.io._ 21 | import java.util.UUID 22 | import java.net.BindException 23 | 24 | //import org.eclipse.jetty.util.MultiException 25 | 26 | import org.apache.spark.{Logging, SparkConf, SparkException} 27 | 28 | /* 29 | * Copied from spark-testing-base https://github.com/holdenk/spark-testing-base/blob/f66fd14b6a87efc09cd26c302740f3b0fb2585ae/src/main/1.3/scala/com/holdenkarau/spark/testing/Utils.scala 30 | * 31 | * Copied here to avoid conflicting version problems with spark-testing-base, see https://github.com/juanrh/sscheck/issues/36 32 | * This is an almost verbatim copy, just commenting unused methods that lead to more dependencies 33 | * 34 | * This is a subset of the Spark Utils object that we use for testing 35 | */ 36 | object Utils extends Logging { 37 | private val shutdownDeletePaths = new scala.collection.mutable.HashSet[String]() 38 | 39 | // Add a shutdown hook to delete the temp dirs when the JVM exits 40 | Runtime.getRuntime.addShutdownHook(new Thread("delete Spark temp dirs") { 41 | override def run(): Unit = { 42 | shutDownCleanUp() 43 | } 44 | }) 45 | 46 | def shutDownCleanUp(): Unit = { 47 | shutdownDeletePaths.foreach { dirPath => 48 | try { 49 | Utils.deleteRecursively(new File(dirPath)) 50 | } catch { 51 | // Doesn't really matter if we fail. 52 | // scalastyle:off println 53 | case e: Exception => println("Exception during cleanup") 54 | // scalastyle:on println 55 | } 56 | } 57 | } 58 | 59 | /** 60 | * Check to see if file is a symbolic link. 61 | */ 62 | def isSymlink(file: File): Boolean = { 63 | if (file == null) throw new NullPointerException("File must not be null") 64 | val fileInCanonicalDir = if (file.getParent() == null) { 65 | file 66 | } else { 67 | new File(file.getParentFile().getCanonicalFile(), file.getName()) 68 | } 69 | 70 | !fileInCanonicalDir.getCanonicalFile().equals(fileInCanonicalDir.getAbsoluteFile()) 71 | } 72 | 73 | private def listFilesSafely(file: File): Seq[File] = { 74 | if (file.exists()) { 75 | val files = file.listFiles() 76 | if (files == null) { 77 | throw new IOException("Failed to list files for dir: " + file) 78 | } 79 | files 80 | } else { 81 | List() 82 | } 83 | } 84 | 85 | 86 | /** 87 | * Delete a file or directory and its contents recursively. 88 | * Don't follow directories if they are symlinks. 89 | * Throws an exception if deletion is unsuccessful. 90 | */ 91 | def deleteRecursively(file: File) { 92 | if (file != null) { 93 | try { 94 | if (file.isDirectory && !isSymlink(file)) { 95 | var savedIOException: IOException = null 96 | for (child <- listFilesSafely(file)) { 97 | try { 98 | deleteRecursively(child) 99 | } catch { 100 | // In case of multiple exceptions, only last one will be thrown 101 | case ioe: IOException => savedIOException = ioe 102 | } 103 | } 104 | if (savedIOException != null) { 105 | throw savedIOException 106 | } 107 | shutdownDeletePaths.synchronized { 108 | shutdownDeletePaths.remove(file.getAbsolutePath) 109 | } 110 | } 111 | } finally { 112 | if (!file.delete()) { 113 | // Delete can also fail if the file simply did not exist 114 | if (file.exists()) { 115 | throw new IOException("Failed to delete: " + file.getAbsolutePath) 116 | } 117 | } 118 | } 119 | } 120 | } 121 | 122 | 123 | // Register the path to be deleted via shutdown hook 124 | def registerShutdownDeleteDir(file: File) { 125 | val absolutePath = file.getAbsolutePath() 126 | shutdownDeletePaths.synchronized { 127 | shutdownDeletePaths += absolutePath 128 | } 129 | } 130 | 131 | 132 | /** 133 | * Create a directory inside the given parent directory. The directory is guaranteed to be 134 | * newly created, and is not marked for automatic deletion. 135 | */ 136 | def createDirectory(root: String): File = { 137 | var attempts = 0 138 | val maxAttempts = 10 139 | var dir: File = null 140 | while (dir == null) { 141 | attempts += 1 142 | if (attempts > maxAttempts) { 143 | throw new IOException("Failed to create a temp directory (under " + root + ") after " + 144 | maxAttempts + " attempts!") 145 | } 146 | try { 147 | dir = new File(root, "spark-" + UUID.randomUUID.toString) 148 | if (dir.exists() || !dir.mkdirs()) { 149 | dir = null 150 | } 151 | } catch { case e: SecurityException => dir = null; } 152 | } 153 | 154 | dir 155 | } 156 | 157 | /** 158 | * Create a temporary directory inside the given parent directory. The directory will be 159 | * automatically deleted when the VM shuts down. 160 | */ 161 | def createTempDir(root: String = System.getProperty("java.io.tmpdir")): File = { 162 | val dir = createDirectory(root) 163 | registerShutdownDeleteDir(dir) 164 | dir 165 | } 166 | 167 | 168 | // /** 169 | // * Attempt to start a service on the given port, or fail after a number of attempts. 170 | // * Each subsequent attempt uses 1 + the port used in the previous attempt (unless the port is 0). 171 | // * 172 | // * @param startPort The initial port to start the service on. 173 | // * @param startService Function to start service on a given port. 174 | // * This is expected to throw java.net.BindException on port collision. 175 | // * @param conf A SparkConf used to get the maximum number of retries when binding to a port. 176 | // * @param serviceName Name of the service. 177 | // */ 178 | // def startServiceOnPort[T]( 179 | // startPort: Int, 180 | // startService: Int => (T, Int), 181 | // conf: SparkConf, 182 | // serviceName: String = ""): (T, Int) = { 183 | // 184 | // require(startPort == 0 || (1024 <= startPort && startPort < 65536), 185 | // "startPort should be between 1024 and 65535 (inclusive), or 0 for a random free port.") 186 | // 187 | // val serviceString = if (serviceName.isEmpty) "" else s" '$serviceName'" 188 | // val maxRetries = 100 189 | // for (offset <- 0 to maxRetries) { 190 | // // Do not increment port if startPort is 0, which is treated as a special port 191 | // val tryPort = if (startPort == 0) { 192 | // startPort 193 | // } else { 194 | // // If the new port wraps around, do not try a privilege port 195 | // ((startPort + offset - 1024) % (65536 - 1024)) + 1024 196 | // } 197 | // try { 198 | // val (service, port) = startService(tryPort) 199 | // logInfo(s"Successfully started service$serviceString on port $port.") 200 | // return (service, port) 201 | // } catch { 202 | // case e: Exception if isBindCollision(e) => 203 | // if (offset >= maxRetries) { 204 | // val exceptionMessage = 205 | // s"${e.getMessage}: Service$serviceString failed after $maxRetries retries!" 206 | // val exception = new BindException(exceptionMessage) 207 | // // restore original stack trace 208 | // exception.setStackTrace(e.getStackTrace) 209 | // throw exception 210 | // } 211 | // logWarning(s"Service$serviceString could not bind on port $tryPort. " + 212 | // s"Attempting port ${tryPort + 1}.") 213 | // } 214 | // } 215 | // // Should never happen 216 | // throw new SparkException(s"Failed to start service$serviceString on port $startPort") 217 | // } 218 | // 219 | // /** 220 | // * Return whether the exception is caused by an address-port collision when binding. 221 | // */ 222 | // def isBindCollision(exception: Throwable): Boolean = { 223 | // import scala.collection.JavaConversions._ 224 | // 225 | // exception match { 226 | // case e: BindException => 227 | // if (e.getMessage != null) { 228 | // return true 229 | // } 230 | // isBindCollision(e.getCause) 231 | // case e: MultiException => e.getThrowables.exists(isBindCollision) 232 | // case e: Exception => isBindCollision(e.getCause) 233 | // case _ => false 234 | // } 235 | // } 236 | } -------------------------------------------------------------------------------- /src/main/scala/org/apache/spark/streaming/dstream/SscheckFriendlyInputDStream.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.spark.streaming.dstream 19 | 20 | import scala.language.implicitConversions 21 | import scala.reflect.ClassTag 22 | 23 | import org.apache.spark.streaming._ 24 | /** 25 | * Copied from spark-testing-base https://github.com/holdenk/spark-testing-base/blob/fedc276a312c408e5a9adad0c0c29119f8b41c86/src/main/1.3/scala/org/apache/spark/streaming/dstream/FriendlyInputDStream.scala 26 | * Copied here to avoid conflicting version problems with spark-testing-base, see https://github.com/juanrh/sscheck/issues/36 27 | * This is a verbatim copy just renaming the class, to avoid naming conflicts 28 | * 29 | * This is a class to provide access to zerotime information 30 | */ 31 | abstract class SscheckFriendlyInputDStream [T: ClassTag](@transient var ssc_ : StreamingContext) 32 | extends InputDStream[T](ssc_) { 33 | 34 | var ourZeroTime: Time = null 35 | 36 | override def initialize(time: Time) { 37 | ourZeroTime = time 38 | super.initialize(time) 39 | } 40 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/gen/PDStreamGenTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.{Properties, Gen} 4 | import org.scalacheck.Arbitrary.arbitrary 5 | import org.scalacheck.Prop.{forAll, exists, AnyOperators, collect} 6 | // import Buildables.buildableSeq 7 | import Buildables._ 8 | import org.scalatest._ 9 | import org.scalatest.Matchers._ 10 | import org.scalatest.prop.PropertyChecks._ 11 | import org.scalatest.Inspectors.{forAll => testForAll} 12 | import PDStreamGenConversions._ 13 | 14 | /* 15 | * NOTE the use of the import alias org.scalatest.Inspectors.{forAll => testForAll} to 16 | * distinguish between ScalaTest forAll inspector used in matchers, and ScalaTest 17 | * forAll adapter for ScalaCheck 18 | * */ 19 | 20 | object PDStreamGenTest extends Properties("Properties class PDStreamGen") { 21 | property("""dstreamUnion() should respect expected batch sizes""") = { 22 | // using small lists as we'll use Prop.exists 23 | val (batchMaxSize, dstreamMaxSize) = (100, 20) 24 | forAll ("batchSize1" |: Gen.choose(0, batchMaxSize), 25 | "batchSize2" |: Gen.choose(0, batchMaxSize), 26 | "dstreamSize1" |: Gen.choose(0, dstreamMaxSize), 27 | "dstreamSize2" |: Gen.choose(0, dstreamMaxSize)) 28 | { (batchSize1 : Int, batchSize2 : Int, dstreamSize1: Int, dstreamSize2: Int) => 29 | def batchGen1 : Gen[Batch[Int]] = BatchGen.ofN(batchSize1, arbitrary[Int]) 30 | def dstreamGen1 : Gen[PDStream[Int]] = PDStreamGen.ofN(dstreamSize1, batchGen1) 31 | def batchGen2 : Gen[Batch[Int]] = BatchGen.ofN(batchSize2, arbitrary[Int]) 32 | def dstreamGen2 : Gen[PDStream[Int]] = PDStreamGen.ofN(dstreamSize2, batchGen2) 33 | val (gs1, gs2) = (dstreamGen1, dstreamGen2) 34 | forAll ("dsUnion" |: gs1 + gs2) { (dsUnion : PDStream[Int]) => 35 | collect (s"batchSize1=${batchSize1}, batchSize2=${batchSize2}, dstreamSize1=${dstreamSize1}, dstreamSize2=${dstreamSize2}") { 36 | dsUnion should have length (math.max(dstreamSize1, dstreamSize2)) 37 | // if no batch is generated then the effective batch size is 0 38 | def effectiveBatchSize(dstreamSize : Int, batchSize : Int) = 39 | if (dstreamSize <= 0) 0 else batchSize 40 | val (effectiveBatchSize1, effectiveBatchSize2) = 41 | (effectiveBatchSize(dstreamSize1, batchSize1), effectiveBatchSize(dstreamSize2, batchSize2)) 42 | // in general batches don't have the effectiveBatchSize1 + effectiveBatchSize2 size 43 | // because the smallest dstream is filled with empty batches 44 | testForAll (dsUnion : Seq[Batch[Int]]) { 45 | (batch : Batch[Int]) => batch.length should be <= (effectiveBatchSize1 + effectiveBatchSize2) 46 | } 47 | // but the batches in the prefix where both dstreams have not empty batches should 48 | // have a size of exactly effectiveBatchSize1 + effectiveBatchSize2 49 | testForAll (dsUnion.slice(0, math.min(dstreamSize1, dstreamSize2)) : Seq[Batch[Int]]) { 50 | _ should have length (effectiveBatchSize1 + effectiveBatchSize2) 51 | } 52 | true // need to finish like that when using ScalaCheck matchers 53 | } 54 | } 55 | } 56 | } 57 | 58 | /* This property is removed because the example proving it 59 | * is almost never found, and this causes a failure of 60 | * the automatic tests 61 | * 62 | * property("""dstreamUnion() generates batches as the union of the batches generated by its inputs 63 | (weak, existential)""") = { 64 | // using small lists as we'll use Prop.exists 65 | val (batchMaxSize, dstreamMaxSize) = (2, 1) 66 | def batchGen : Gen[Batch[Int]] = BatchGen.ofNtoM(0, batchMaxSize, arbitrary[Int]) 67 | def dstreamGen : Gen[DStream[Int]] = DStreamGen.ofNtoM(0, dstreamMaxSize, batchGen) 68 | val (gs1, gs2) = (dstreamGen, dstreamGen) 69 | 70 | forAll ("dsUnion" |: dstreamUnion(gs1, gs2)) { (dsUnion : DStream[Int]) => 71 | // dsUnion.length should be (1) 72 | exists ("gsi zip" |: Gen.zip(gs1, gs2)) { (ds12 : (DStream[Int], DStream[Int])) => 73 | // dsUnion should be (ds12._1 ++ ds12._2): this raises an exception on the first failure 74 | // => not the existential behaviour. TODO: experiment wrapping in Try and map to Boolean 75 | dsUnion ?= ds12._1 ++ ds12._2 76 | } 77 | } 78 | } */ 79 | 80 | } 81 | 82 | -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/gen/ReGenTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.{Properties, Gen} 4 | import org.scalacheck.Arbitrary.arbitrary 5 | import org.scalacheck.Prop.{forAll, BooleanOperators, exists, atLeastOne} 6 | 7 | object ReGenTest extends Properties("ReGen regex generators properties") { 8 | 9 | property("epsilon generates empty sequences") = 10 | forAll (ReGen.epsilon) { (xs : Seq[Int]) => 11 | xs.length == 0 12 | } 13 | 14 | property("symbol generates a single element that contains the argument") = 15 | forAll { x : String => 16 | forAll (ReGen.symbol(x)) { xs : Seq[String] => 17 | xs.length == 1 && xs(0) == x 18 | } 19 | } 20 | 21 | // def alt[A](gs : Gen[Seq[A]]*) : Gen[Seq[A]] = { 22 | property("alt is equivalent to epsilon if zero generators are provided") = 23 | forAll (ReGen.alt()) { (xs : Seq[Int]) => 24 | forAll (ReGen.epsilon) { (ys : Seq[Int]) => 25 | xs == ys 26 | } 27 | } 28 | 29 | property("alt works for more than one argument, and generates values for some of the alternatives (weak, existential)") = { 30 | val (g1, g2, g3) = (ReGen.symbol(0), ReGen.symbol(1), ReGen.symbol(2)) 31 | forAll (ReGen.alt(g1, g2, g3)) { (xs : Seq[Int]) => 32 | atLeastOne( 33 | exists (g1) { (ys : Seq[Int]) => 34 | xs == ys 35 | }, 36 | exists (g2) { (ys : Seq[Int]) => 37 | xs == ys 38 | }, 39 | exists (g3) { (ys : Seq[Int]) => 40 | xs == ys 41 | } 42 | ) 43 | } 44 | } 45 | 46 | // conc and star only have similar weak existential properties 47 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/gen/TLGenTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.{Properties, Gen} 4 | import org.scalacheck.Arbitrary.arbitrary 5 | import org.scalacheck.Prop.{forAll, exists, AnyOperators, collect} 6 | import org.scalatest._ 7 | import org.scalatest.Matchers._ 8 | import org.scalatest.prop.PropertyChecks._ 9 | import org.scalatest.Inspectors.{forAll => testForAll} 10 | import Batch.seq2batch 11 | import BatchGenConversions._ 12 | import PDStreamGen._ 13 | import Buildables.{buildableBatch, buildablePDStreamFromBatch} 14 | import DStreamMatchers._ 15 | import es.ucm.fdi.sscheck.prop.UtilsProp 16 | 17 | /** Tests for the LTL inspired HO generators defined at BatchGen and DStreamGen 18 | * 19 | * WARNING: due to using nested forall, shrinking might generate wrong counterexamples, 20 | * so the _ORIGINAL versions should be used in case of test failure 21 | * 22 | * Note most tests check completeness of the generators, but not correctness, i.e. that 23 | * all the generated data corresponds to some input data. We could do that but logically 24 | * it makes sense the interpreting the TL formulas as requiring something to happen, but 25 | * not requiring that something doesn't happen, has we don't have negation. 26 | * */ 27 | object TLGenTest extends Properties("TLGen temporal logic generators properties") { 28 | /* 29 | * NOTE the use of the import alias org.scalatest.Inspectors.{forAll => testForAll} to 30 | * distinguish between ScalaTest forAll inspector used in matchers, and ScalaTest 31 | * forAll adapter for ScalaCheck 32 | * */ 33 | // 34 | // Tests for BatchGen LTL inspired HO generators 35 | // 36 | property("BatchGen.now() applied to a batch generator returns a dstream generator " + 37 | "with exactly one batch, and doesn't introduce new elements") = 38 | forAll ("batch" |: arbitrary[Batch[Int]]) { batch : Batch[Int] => 39 | // note here we are dropping some elements 40 | val g = BatchGen.now(Gen.someOf(batch)) 41 | forAll ("dstream" |: g) { dstream : PDStream[Int] => 42 | dstream should have length (1) 43 | dstream(0).length should be <= (batch.length) 44 | testForAll (dstream(0)) { batch should contain (_)} 45 | true 46 | } 47 | } 48 | 49 | property("BatchGen.now() applied to a constant batch generator returns a dstream generator " + 50 | "with exactly that batch as the only batch") = 51 | forAll ("batch" |: arbitrary[Batch[Int]]) { batch : Batch[Int] => 52 | // using a constant generator 53 | val g = BatchGen.now(batch : Batch[Int]) 54 | forAll ("dstream" |: g) { dstream : PDStream[Int] => 55 | dstream should have length (1) 56 | dstream(0) should be (batch) 57 | true 58 | } 59 | } 60 | 61 | property("BatchGen.next() returns a dstream generator that has exactly two batches, " + 62 | "the first one is emptyBatch, and the second one is its argument") = 63 | forAll ("batch" |: arbitrary[Batch[Int]]) { batch : Batch[Int] => 64 | // using a constant generator 65 | val g = BatchGen.next(batch : Batch[Int]) 66 | forAll ("dstream" |: g) { dstream : PDStream[Int] => 67 | dstream should have length (2) 68 | dstream(0) should be (Batch.empty) 69 | dstream(1) should be (batch) 70 | true 71 | } 72 | } 73 | 74 | property("BatchGen.laterN(n, bg) generates n batches and then bg" ) = 75 | forAll ("batch" |: arbitrary[Batch[Int]], "n" |: Gen.choose(-10, 30)) { 76 | (batch : Batch[Int], n : Int) => 77 | // using a constant generator 78 | val g = BatchGen.laterN(n, batch) 79 | forAll ("nextDStream" |: g) { nextDStream : PDStream[Int] => 80 | nextDStream should have length (math.max(0, n) + 1) 81 | testForAll (nextDStream.slice(0, n)) {_ should be (Batch.empty)} 82 | nextDStream(nextDStream.length-1) should be (batch) 83 | true 84 | } 85 | } 86 | 87 | property("BatchGen.until is a strong until, i.e. the second generator always occurs, and " + 88 | "the first one occours before") = 89 | forAll ("batch1" |: arbitrary[Batch[Int]], "batch2" |: arbitrary[Batch[Int]]) { 90 | (batch1 : Batch[Int], batch2 : Batch[Int]) => 91 | // using constant generators 92 | val g = BatchGen.until(batch1, batch2) 93 | forAll ("untilDStream" |: g) { untilDStream : PDStream[Int] => 94 | testForAll (untilDStream.slice(0, untilDStream.length-1)) { _ should be (batch1)} 95 | if (untilDStream.length >0) 96 | untilDStream(untilDStream.length-1) should be (batch2) 97 | true 98 | } 99 | } 100 | 101 | property("BatchGen.eventually eventually produces data from the " + 102 | "argument batch generator") = 103 | forAll ("batch" |: arbitrary[Batch[Int]]) { batch : Batch[Int] => 104 | forAll ("eventuallyDStream" |: BatchGen.eventually(batch)) { eventuallyDStream : PDStream[Int] => 105 | eventuallyDStream(eventuallyDStream.length-1) should be (batch) 106 | true 107 | } 108 | } 109 | 110 | property("BatchGen.always always produces data from the argument batch generator") = 111 | forAll ("batch" |: arbitrary[Batch[Int]]) { batch : Batch[Int] => 112 | forAll ("alwaysDStream" |: BatchGen.always(batch)) { alwaysDStream : PDStream[Int] => 113 | testForAll (alwaysDStream.toList) {_ should be (batch)} 114 | true 115 | } 116 | } 117 | 118 | property("BatchGen.release is a weak relase, i.e either bg2 happens forever, " + 119 | "or it happens until bg1 happens, including the moment when bg1 happens") = 120 | forAll ("batch1" |: arbitrary[Batch[Int]], "batch2" |: arbitrary[Batch[Int]]) { 121 | (batch1 : Batch[Int], batch2 : Batch[Int]) => 122 | // using constant generators 123 | val g = BatchGen.release(batch1, batch2) 124 | forAll ("releaseDStream" |: g) { releaseDStream : PDStream[Int] => 125 | testForAll (releaseDStream.slice(0, releaseDStream.length-2)) { _ should be (batch2)} 126 | if (releaseDStream.length >0) 127 | releaseDStream(releaseDStream.length-1) should 128 | (be (batch1 ++ batch2) or be (batch2)) 129 | true 130 | } 131 | } 132 | 133 | // 134 | // Tests for DStreamGen LTL inspired HO generators 135 | // 136 | // small DStream generator for costly tests 137 | val smallDsg = PDStreamGen.ofNtoM(0, 10, BatchGen.ofNtoM(0, 5, arbitrary[Int])) 138 | 139 | property("DStreamGen.next() returns a dstream generator that has exactly 1 + the " + 140 | "number of batches of its argument, the first one is emptyBatch, and the rest " + 141 | "are the batches generated by its argument") = 142 | forAll ("dstream" |: arbitrary[PDStream[Int]]) { dstream : PDStream[Int] => 143 | // using a constant generator 144 | val g = PDStreamGen.next(dstream) 145 | forAll ("nextDStream" |: g) { nextDStream : PDStream[Int] => 146 | nextDStream should have length (1 + dstream.length) 147 | nextDStream(0) should be (Batch.empty) 148 | nextDStream.slice(1, nextDStream.size) should be (dstream) 149 | true 150 | } 151 | } 152 | 153 | property("DStreamGen.laterN(n, dsg) generates n batches and then dsg" ) = 154 | forAll ("dstream" |: arbitrary[PDStream[Int]], "n" |: Gen.choose(-10, 30)) { 155 | (dstream : PDStream[Int], n : Int) => 156 | // using a constant generator 157 | val g = PDStreamGen.laterN(n, dstream) 158 | forAll ("nextDStream" |: g) { nextDStream : PDStream[Int] => 159 | nextDStream should have length (math.max(0, n) + dstream.length) 160 | testForAll (nextDStream.slice(0, n)) {_ should be (Batch.empty)} 161 | nextDStream.slice(n, nextDStream.length) should be (dstream.toList) 162 | true 163 | } 164 | } 165 | 166 | property("DStreamGen.until is a strong until, i.e. the second generator always occurs, " + 167 | "and the first one occours before") = { 168 | // explicitly limiting generator sizes to avoid too slow tests 169 | forAll ("dstream1" |: smallDsg, "dstream2" |: smallDsg) { (dstream1 : PDStream[Int], dstream2 : PDStream[Int]) => 170 | // using constant generators 171 | val (dstream1Len, dstream2Len) = (dstream1.length, dstream2.length) 172 | val g = PDStreamGen.until(dstream1, dstream2) 173 | forAll ("untilDStream" |: g) { untilDStream : PDStream[Int] => 174 | for {i <- 0 until untilDStream.length - dstream1Len - dstream2Len} { 175 | dstream1 should beSubsetOf (untilDStream.slice(i, i + dstream1Len)) 176 | } 177 | // FIXME esto falla si el phi es mas largo que phi2, pq se descuenta mal 178 | if (dstream1Len <= dstream2Len) { 179 | val tail = untilDStream.slice(untilDStream.length - dstream2Len, untilDStream.length) 180 | dstream2 should beSubsetOf(tail) 181 | } 182 | /* else there is no reference for the start of dstream2 in untilDStream, 183 | * for example: 184 | * 185 | > dstream1: PDStream(Batch(-1, -1, 285357432, -2147483648, 1268745804), Bat 186 | ch(), Batch(), Batch(-13643088), Batch(591481111, -1926229852, 2147483647 187 | , 1, -732424421), Batch(0, -1), Batch(0, -1936654354, -1, 44866036), Batc 188 | h(1983936151)) 189 | > dstream2: PDStream(Batch(1, 1), Batch()) 190 | > untilDStream: PDStream(Batch(-1, -1, 285357432, -2147483648, 1268745804), 191 | Batch(1, 1), Batch(), Batch(-13643088), Batch(591481111, -1926229852, 21 192 | 47483647, 1, -732424421), Batch(0, -1), Batch(0, -1936654354, -1, 4486603 193 | 6), Batch(1983936151)) 194 | */ 195 | true 196 | } 197 | } 198 | } 199 | 200 | property("DStreamGen.eventually eventually produces data from the argument generator") = 201 | forAll ("dstream" |: arbitrary[PDStream[Int]]) { dstream : PDStream[Int] => 202 | forAll ("eventuallyDStream" |: PDStreamGen.eventually(dstream)) { eventuallyDStream : PDStream[Int] => 203 | val eventuallyDStreamLen = eventuallyDStream.length 204 | val ending = eventuallyDStream.slice(eventuallyDStreamLen - dstream.length, eventuallyDStreamLen) 205 | ending should be (dstream) 206 | true 207 | } 208 | } 209 | 210 | property("DStreamGen.always always produces data from the argument generator") = 211 | // explicitly limiting generator sizes to avoid too slow tests 212 | forAll (smallDsg) { dstream : PDStream[Int] => 213 | val dstreamLen = dstream.length 214 | forAll ("alwaysDStream" |: PDStreamGen.always(dstream)) { alwaysDStream : PDStream[Int] => 215 | for {i <- 0 until alwaysDStream.length - dstreamLen} { 216 | dstream should beSubsetOf (alwaysDStream.slice(i, i + dstreamLen)) 217 | } 218 | true 219 | } 220 | } 221 | 222 | property("DStreamGen.release is a weak relase, i.e either the second generator happens forever, " + 223 | "or it happens until the first generator happens, including the moment when the first generator happens") = 224 | { 225 | // explicitly limiting generator sizes to avoid too slow tests 226 | forAll ("dstream1" |: smallDsg, "dstream2" |: smallDsg) { 227 | (dstream1 : PDStream[Int], dstream2 : PDStream[Int]) => 228 | // using constant generators 229 | val (dstream1Len, dstream2Len) = (dstream1.length, dstream2.length) 230 | val g = PDStreamGen.release(dstream1, dstream2) 231 | forAll ("releaseDStream" |: g) { releaseDStream : PDStream[Int] => 232 | // this is similar to always, but note the use of max to account for 233 | // the case when dstream1 happens and dstream1 is longer than dstream2 234 | // We don't check if dstream1 happens, because it might not 235 | for {i <- 0 until releaseDStream.length - math.max(dstream1Len, dstream2Len)} { 236 | dstream2 should beSubsetOf (releaseDStream.slice(i, i + dstream2Len)) 237 | } 238 | true 239 | } 240 | } 241 | } 242 | 243 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/gen/UtilsGenTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.gen 2 | 3 | import org.scalacheck.{Properties, Gen} 4 | import org.scalacheck.Arbitrary.arbitrary 5 | import org.scalacheck.Prop.{forAll, exists, BooleanOperators} 6 | import Buildables.buildableSeq 7 | 8 | object UtilsGenTest extends Properties("UtilsGenTest test") { 9 | property("""containerOfNtoM is able to generate sequences of 10 | string with size between N and M strings for both N and M >= 0""") = 11 | forAll (Gen.choose(0, 10), Gen.choose(0, 10)) { (n : Int, m : Int) => 12 | val g = UtilsGen.containerOfNtoM(n, m, arbitrary[String]) : Gen[Seq[String]] 13 | forAll (g) { ( xs : Seq[String]) => 14 | xs.length >= n && xs.length <= m 15 | } 16 | } 17 | 18 | property("repN respects its lenght constraints") = 19 | forAll (Gen.choose(0, 10), Gen.choose(0, 10)) { (n : Int, xsLen : Int) => 20 | val g = UtilsGen.repN(n, Gen.listOfN(xsLen, Gen.alphaStr)) 21 | forAll (g) { (xs : Seq[String]) => 22 | xs.length == xsLen * n 23 | } 24 | } 25 | 26 | property("repNtoM respects its lenght constraints") = 27 | forAll (Gen.choose(0, 10), Gen.choose(0, 10), Gen.choose(0, 10)) { (n : Int, m : Int, xsLen : Int) => 28 | val g = UtilsGen.repNtoM(n, m, Gen.listOfN(xsLen, arbitrary[String])) 29 | forAll (g) { (xs : Seq[String]) => 30 | val xsLenObs = xs.length 31 | xsLenObs >= xsLen * n && xsLen * n <= xsLen * m 32 | } 33 | } 34 | 35 | property("""concSeq returns the result of concatenating the sequences 36 | generated by its arguments""") = { 37 | // In order for Prop.exists to be effective, we use a small domain. 38 | // For all the lengths considered: this is very weak because very little lengths 39 | // are considered but it's better than nothing. 40 | forAll (Gen.choose(0, 2)) { (xsLen : Int) => 41 | // we consider two generators for lists of elements with that size 42 | val (gxs1, gxs2) = (Gen.listOfN(xsLen, Gen.choose(1, 3)), Gen.listOfN(xsLen, Gen.choose(4, 6))) 43 | // val g = UtilsGen.concSeq(gxs1, gxs2) 44 | forAll (UtilsGen.concSeq(gxs1, gxs2)) { (xs : Seq[Int]) => 45 | // Prop.exists is not overloaded to support several generators 46 | // so we have to use zip 47 | exists (Gen.zip(gxs1, gxs2)) { (xs12 : (List[Int], List[Int])) => 48 | xs == xs12._1 ++ xs12._2 49 | } 50 | } 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/prop/tl/FormulaTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.prop.tl 2 | 3 | import org.scalacheck.{Gen,Prop} 4 | import org.scalacheck.Arbitrary.arbitrary 5 | 6 | import org.junit.runner.RunWith 7 | import org.specs2.runner.JUnitRunner 8 | import org.specs2.ScalaCheck 9 | import org.specs2.Specification 10 | import org.specs2.execute.Result 11 | 12 | import Formula._ 13 | 14 | /* TODO tests for formulas with quantifiers */ 15 | @RunWith(classOf[JUnitRunner]) 16 | class FormulaTest 17 | extends Specification { 18 | 19 | def is = sequential ^ s2""" 20 | Basic test for temporal logic formulas representation 21 | - where some example formulas are correctly built $exampleFormulas 22 | - where nextFormula is defined correctly $nextFormulaOk 23 | - where examples from the paper for nextFormula work as expected $pending nextFormulaPaper 24 | - where evaluation with consume works correclty $consumeOk 25 | - where Formula.next works ok when used for several times $nextTimes 26 | - where safeWordLength is ok $pending 27 | """ 28 | 29 | // Consider an universe with an Int i and a String s 30 | type U = (Int, String) 31 | type Form = Formula[U] 32 | val (i, s) = ((_ : U)._1, (_ : U)._2) 33 | // some atomic propositions 34 | val aP : Form = at(i)(_ must be_>(2)) 35 | val aQ : Form = at(s)(_ contains "hola") 36 | 37 | def exampleFormulas = { 38 | val notP = ! aP 39 | val pImpliesQ = aP ==> aQ 40 | val nextP = next (aP) 41 | val alwaysP = always (aP) during 6 42 | 43 | // note there are no exceptions due to suspended evaluation 44 | // Now ((x : Any) => 1 === 0) 45 | 46 | val pUntilQ = aP until aQ on 4 47 | val pUntilQExplicitTimeout = aP until aQ on Timeout(4) 48 | val nestedUntil = (aP until aQ on 4) until aP on 3 49 | val pUntilQImplicitTimeout : Form = { 50 | implicit val t = Timeout(3) 51 | aP until aQ 52 | } 53 | 54 | aP must not be_==(aQ) 55 | // TODO: add examples for each of the case classes 56 | } 57 | 58 | // TODO: adapt to new lazy next form 59 | // TODO: adapt to NextAnd and NextOr extending NextBinaryOp 60 | def nextFormulaOk = { 61 | // now 62 | { aP. nextFormula === aP } and 63 | { aP. nextFormula must not be_==(aQ. nextFormula) } and 64 | // solved 65 | { 66 | val solvedP = aP. nextFormula.consume(Time(0L))((3, "hola")) 67 | solvedP. nextFormula === solvedP 68 | } and 69 | // 70 | // not 71 | { (! aP). nextFormula === !(aP. nextFormula) } and 72 | // // or 73 | // { (aP or aQ). nextFormula === (aP. nextFormula or aQ. nextFormula) } and 74 | // // and 75 | // { (aP and aQ). nextFormula === (aP. nextFormula and aQ. nextFormula) } and 76 | // // implies: note it is reduced to an or 77 | // { (aP ==> aQ). nextFormula === (! aP. nextFormula or aQ. nextFormula) } and 78 | // 79 | // next 80 | { NextNext(???) 81 | true 82 | } // and 83 | //{ next(aP). nextFormula === next(aP. nextFormula) } //and 84 | // 85 | // eventually 86 | // { (later(aP) on 1). nextFormula === aP } and 87 | // { (later(aP) on 2). nextFormula === (aP or next(aP)) } and 88 | // { (later(aP) on 3). nextFormula === or(aP, next(aP), next(next(aP))) } // and 89 | // 90 | // // always 91 | // { (always(aP) during 1). nextFormula === aP } and 92 | // { (always(aP) during 2). nextFormula === (aP and next(aP)) } and 93 | // { (always(aP) during 3). nextFormula === and(aP, next(aP), next(next(aP))) } and 94 | // // 95 | // // until 96 | // { (aP until aQ on 1). nextFormula === aQ } and 97 | // { (aP until aQ on 2). nextFormula === or(aQ, and(aP, next(aQ))) } and 98 | // { (aP until aQ on 3). nextFormula === 99 | // or(aQ, and(aP, next(aQ)), and(aP, next(aP), next(next(aQ)))) } and 100 | // // 101 | // // release 102 | // { (aP release aQ on 1).nextFormula === or(aQ, and(aP, aQ)) } and 103 | // { (aP release aQ on 2).nextFormula === 104 | // or(and(aQ, next(aQ)), 105 | // and(aP, aQ), 106 | // and(aQ, next(aP), next(aQ)) 107 | // ) } and 108 | // { (aP release aQ on 3).nextFormula === 109 | // or(and(aQ, next(aQ), next(next(aQ))), 110 | // and(aP, aQ), 111 | // and(aQ, next(aP), next(aQ)), 112 | // and(aQ, next(aQ), next(next(aP)), next(next(aQ))) 113 | // ) } 114 | } 115 | 116 | // TODO: adapt to new lazy next form 117 | def nextFormulaPaper = { 118 | val phi = always (aQ ==> (later(aP) on 2)) during 2 119 | phi.nextFormula === 120 | ( (!aQ or (aP or next(aP))) and 121 | next(!aQ or (aP or next(aP))) ) 122 | } 123 | 124 | def consumeOk = { 125 | type U = (Int, Int) 126 | type Form = Formula[U] 127 | // some atomic propositions 128 | val a : Form = (u : U) => u._1 must be_> (0) 129 | val b : Form = (u : U) => u._2 must be_> (0) 130 | // some letters 131 | val aL : U = (1, 0) // only a holds 132 | val bL : U = (0, 1) // only b holds 133 | val abL : U = (1, 1) // both a and b hold 134 | val phi = always (b ==> (later(a) on 2)) during 2 135 | val psi = b until next(a and next(a)) on 2 136 | 137 | { 138 | val phiNF = phi.nextFormula 139 | println(s"phi: $phi") 140 | println(s"phiNF: $phiNF") 141 | val phiNF1 = phiNF.consume(Time(0L))(bL) 142 | println(s"phiNF1: $phiNF1") 143 | val phiNF2 = phiNF1.consume(Time(1L))(bL) 144 | println(s"phiNF2: $phiNF2") 145 | // consuming a letter with a solution is ok and doesn't change the result 146 | val phiNF3 = phiNF2.consume(Time(3L))(bL) 147 | println(s"phiNF3: $phiNF3") 148 | ( phiNF.result must beNone ) and 149 | ( phiNF1.result must beNone ) and 150 | ( phiNF2.result must beSome(Prop.False) ) and 151 | ( phiNF2 must haveClass[Solved[_]] ) and 152 | ( phiNF3 === phiNF2 ) 153 | } and { 154 | val psiNF = psi.nextFormula 155 | println 156 | println(s"psi: $psi") 157 | println(s"psiNF: $psiNF") 158 | val psiNF1 = psiNF.consume(Time(4L))(bL) 159 | println(s"psiNF1: $psiNF1") 160 | val psiNF2 = psiNF1.consume(Time(5L))(bL) 161 | println(s"psiNF2: $psiNF2") 162 | val psiNF3 = psiNF2.consume(Time(6L))(abL) 163 | println(s"psiNF3: $psiNF3") 164 | val psiNF4 = psiNF3.consume(Time(7L))(aL) 165 | println(s"psiNF4: $psiNF4") 166 | ( psiNF.result must beNone ) and 167 | ( psiNF1.result must beNone ) and 168 | ( psiNF2.result must beNone ) and 169 | ( psiNF3.result must beNone ) and 170 | ( psiNF4.result must beSome(Prop.True) ) and 171 | ( psiNF4 must haveClass[Solved[_]] ) and 172 | ( psiNF4.consume(Time(8L))(aL) === psiNF4 ) 173 | } 174 | } 175 | 176 | def nextTimes = { 177 | (next(1)(aP) === next(aP)) and 178 | (next(0)(aP) === aP) and 179 | (next(2)(aP) === next(next(aP))) 180 | } 181 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/SharedSparkContextBeforeAfterAllTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | 6 | import org.specs2.mutable.Specification 7 | import org.specs2.ScalaCheck 8 | import org.specs2.scalacheck.Parameters 9 | 10 | import org.scalacheck.{Prop, Gen} 11 | import org.scalacheck.Arbitrary.arbitrary 12 | import org.scalacheck.Prop.AnyOperators 13 | 14 | import org.apache.spark._ 15 | import org.apache.spark.rdd.RDD 16 | 17 | import es.ucm.fdi.sscheck.gen.RDDGen 18 | import es.ucm.fdi.sscheck.gen.RDDGen._ 19 | 20 | import scala.reflect.ClassTag 21 | 22 | @RunWith(classOf[JUnitRunner]) 23 | class SharedSparkContextBeforeAfterAllTest 24 | extends Specification 25 | with SharedSparkContextBeforeAfterAll 26 | with ScalaCheck { 27 | 28 | override def defaultParallelism: Int = 3 29 | override def sparkMaster : String = "local[5]" 30 | override def sparkAppName = this.getClass().getName() 31 | 32 | implicit def defaultScalacheckParams = Parameters(minTestsOk = 101).verbose 33 | 34 | "Sharing a Spark Context between several ScalaCheck properties and test cases, and closing it properly".title ^ 35 | "forall that ignores the Spark context" ! forallOkIgnoreSparkContext 36 | "simple test that uses the Spark context explicitly" ! simpleExplicitUseSparkContext(sc) 37 | "forall that uses the Spark context explicitly, and parallelizes a Seq explicitly" ! forallSparkContextAndParallelizeExplicit(sc) 38 | "forall that uses the Spark context from this, and parallelizes a Seq explicitly" ! forallSparkContextFromThisAndParallelize 39 | "forall that parallelizes a Seq with an implicit" ! forallSparkContextAndParallelizeImplicit 40 | "forall with implicit conversion of Seq generator to RDD generator" ! forallImplicitToRDDGen 41 | "forall that uses RDDGen.of" ! forallRDDGen 42 | "forall that uses RDDGen.of with local overload of parallelism" ! forallRDDGenOverloadPar 43 | "forall that uses RDDGen.ofNtoM" ! forallRDDGenOfNtoM 44 | "forall that uses RDDGen.ofN, testing frequency generator" ! forallRDDGenOfNFreqMean 45 | 46 | def forallOkIgnoreSparkContext = Prop.forAll { x : Int => 47 | x + x === 2 * x 48 | } 49 | 50 | def simpleExplicitUseSparkContext(sc : SparkContext) = { 51 | sc.parallelize(1 to 10).count === 10 52 | } 53 | 54 | def fooCheckOnSeqAndRDD(batch : Seq[Double], rdd : RDD[Double]) = { 55 | val plusOneRdd = rdd.map(_+1) 56 | if (plusOneRdd.count > 0) 57 | plusOneRdd.sum must be_>= (batch.sum - 0.0001) // substraction to account for rounding 58 | rdd.count() ?= batch.length 59 | } 60 | 61 | def forallSparkContextAndParallelizeExplicit(sc : SparkContext) = 62 | Prop.forAll ("batch" |: Gen.listOf(Gen.choose(-1000.0, 1000.0))) { batch : List[Double] => 63 | // take Spark Context from argument and parallelize explicitly 64 | val rdd = sc.parallelize(batch) 65 | fooCheckOnSeqAndRDD(batch, rdd) 66 | }. set(minTestsOk = 50).verbose 67 | 68 | def forallSparkContextFromThisAndParallelize = 69 | Prop.forAll ("batch" |: Gen.listOf(Gen.choose(-1000.0, 1000.0))) { batch : List[Double] => 70 | // take Spark context from this and parallelize explicitly 71 | val rdd = sc.parallelize(batch) 72 | fooCheckOnSeqAndRDD(batch, rdd) 73 | }. set(minTestsOk = 50).verbose 74 | 75 | def forallSparkContextAndParallelizeImplicit = 76 | Prop.forAll ("batch" |: Gen.listOf(Gen.choose(-1000.0, 1000.0))) { batch : List[Double] => 77 | // parallelize implicitly 78 | val rdd : RDD[Double] = batch 79 | fooCheckOnSeqAndRDD(batch, rdd) 80 | }. set(minTestsOk = 50).verbose 81 | 82 | def forallImplicitToRDDGen = 83 | /* NOTE the context below doesn't force the generator to be a Gen[RDD[Int]], so 84 | we have to force it explicitly 85 | Prop.forAll (Gen.listOf(0) ) { xs : RDD[Int] => 86 | */ 87 | Prop.forAll ("rdd" |: Gen.listOf(Gen.choose(-100, 100)) : Gen[RDD[Int]]) { xs : RDD[Int] => 88 | val xsSum = xs.sum 89 | xsSum ?= xs.map(_+1).map(_-1).sum 90 | xsSum + xs.count must beEqualTo(xs.map(_+1).sum) 91 | }. set(minTestsOk = 50).verbose 92 | 93 | def forallRDDGen = 94 | Prop.forAll("rdd" |: RDDGen.of(Gen.choose(-100, 100))) { rdd : RDD[Int] => 95 | rdd.count === rdd.map(_+1).count 96 | }. set(minTestsOk = 10).verbose 97 | 98 | def forallRDDGenOverloadPar = { 99 | // note here we have to use "parallelism" as the name of the val 100 | // in order to shadow this.parallelism, to achieve its overloading 101 | val parLevel = 5 102 | implicit val parallelism = Parallelism(numSlices=parLevel) 103 | Prop.forAll("rdd" |: RDDGen.of(Gen.choose(-100, 100))){ rdd : RDD[Int] => 104 | parLevel === rdd.partitions.length 105 | }. set(minTestsOk = 10).verbose 106 | } 107 | 108 | def forallRDDGenOfNtoM = { 109 | val minWords, maxWords = (50, 100) 110 | Prop.forAll(RDDGen.ofNtoM(50, 100, arbitrary[String])) { rdd : RDD[String] => 111 | rdd.map(_.length()).sum must be_>=(0.0) 112 | } 113 | } 114 | 115 | def forallRDDGenOfNFreqMean = { 116 | val freqs = Map(1 -> 0, 4 -> 1) 117 | val rddSize = 300 118 | val gRDDFreq = RDDGen.ofN(rddSize, Gen.frequency(freqs.mapValues(Gen.const(_)).toSeq:_*)) 119 | val expectedMean = { 120 | val freqS = freqs.toSeq 121 | val num = freqS .map({case (f, v) => v * f}). sum 122 | val den = freqS .map(_._1). sum 123 | num / den.toDouble 124 | } 125 | Prop.forAll("rdd" |: gRDDFreq){ rdd : RDD[Int] => 126 | rdd.mean must be ~(expectedMean +/- 0.1) 127 | } 128 | }. set(minTestsOk = 50).verbose 129 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/demo/StreamingFormulaDemo1.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.demo 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | import org.specs2.ScalaCheck 6 | import org.specs2.Specification 7 | import org.specs2.matcher.ResultMatchers 8 | import org.scalacheck.Arbitrary.arbitrary 9 | 10 | import org.apache.spark.rdd.RDD 11 | import org.apache.spark.streaming.Duration 12 | import org.apache.spark.streaming.dstream.DStream 13 | 14 | import es.ucm.fdi.sscheck.spark.streaming.SharedStreamingContextBeforeAfterEach 15 | import es.ucm.fdi.sscheck.prop.tl.{Formula,DStreamTLProperty} 16 | import es.ucm.fdi.sscheck.prop.tl.Formula._ 17 | import es.ucm.fdi.sscheck.gen.{PDStreamGen,BatchGen} 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class StreamingFormulaDemo1 21 | extends Specification 22 | with DStreamTLProperty 23 | with ResultMatchers 24 | with ScalaCheck { 25 | 26 | // Spark configuration 27 | override def sparkMaster : String = "local[*]" 28 | override def batchDuration = Duration(150) 29 | override def defaultParallelism = 4 30 | 31 | def is = 32 | sequential ^ s2""" 33 | Simple demo Specs2 example for ScalaCheck properties with temporal 34 | formulas on Spark Streaming programs 35 | - where a simple property for DStream.count is a success ${countForallAlwaysProp(_.count)} 36 | - where a faulty implementation of the DStream.count is detected ${countForallAlwaysProp(faultyCount) must beFailing} 37 | """ 38 | 39 | def faultyCount(ds : DStream[Double]) : DStream[Long] = 40 | ds.count.transform(_.map(_ - 1)) 41 | 42 | def countForallAlwaysProp(testSubject : DStream[Double] => DStream[Long]) = { 43 | type U = (RDD[Double], RDD[Long]) 44 | val (inBatch, transBatch) = ((_ : U)._1, (_ : U)._2) 45 | val numBatches = 10 46 | val formula : Formula[U] = always { (u : U) => 47 | transBatch(u).count === 1 and 48 | inBatch(u).count === transBatch(u).first 49 | } during numBatches 50 | 51 | val gen = BatchGen.always(BatchGen.ofNtoM(10, 50, arbitrary[Double]), numBatches) 52 | 53 | forAllDStream( 54 | gen)( 55 | testSubject)( 56 | formula) 57 | }.set(minTestsOk = 10).verbose 58 | 59 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/demo/StreamingFormulaDemo2.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.demo 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | import org.specs2.ScalaCheck 6 | import org.specs2.Specification 7 | import org.specs2.matcher.ResultMatchers 8 | import org.scalacheck.Arbitrary.arbitrary 9 | import org.scalacheck.Gen 10 | 11 | import org.apache.spark.rdd.RDD 12 | import org.apache.spark.streaming.Duration 13 | import org.apache.spark.streaming.dstream.DStream 14 | import org.apache.spark.streaming.dstream.DStream._ 15 | 16 | import scalaz.syntax.std.boolean._ 17 | 18 | import es.ucm.fdi.sscheck.spark.streaming.SharedStreamingContextBeforeAfterEach 19 | import es.ucm.fdi.sscheck.prop.tl.{Formula,DStreamTLProperty} 20 | import es.ucm.fdi.sscheck.prop.tl.Formula._ 21 | import es.ucm.fdi.sscheck.gen.{PDStreamGen,BatchGen} 22 | import es.ucm.fdi.sscheck.gen.BatchGenConversions._ 23 | import es.ucm.fdi.sscheck.gen.PDStreamGenConversions._ 24 | import es.ucm.fdi.sscheck.matcher.specs2.RDDMatchers._ 25 | 26 | @RunWith(classOf[JUnitRunner]) 27 | class StreamingFormulaDemo2 28 | extends Specification 29 | with DStreamTLProperty 30 | with ResultMatchers 31 | with ScalaCheck { 32 | 33 | // Spark configuration 34 | override def sparkMaster : String = "local[*]" 35 | override def batchDuration = Duration(300) 36 | override def defaultParallelism = 3 37 | override def enableCheckpointing = true 38 | 39 | def is = 40 | sequential ^ s2""" 41 | Check process to persistently detect and ban bad users 42 | - where a stateful implementation extracts the banned users correctly ${checkExtractBannedUsersList(listBannedUsers)} 43 | - where a trivial implementation ${checkExtractBannedUsersList(statelessListBannedUsers) must beFailing} 44 | """ 45 | type UserId = Long 46 | 47 | def listBannedUsers(ds : DStream[(UserId, Boolean)]) : DStream[UserId] = 48 | ds.updateStateByKey((flags : Seq[Boolean], maybeFlagged : Option[Unit]) => 49 | maybeFlagged match { 50 | case Some(_) => maybeFlagged 51 | case None => flags.contains(false) option {()} 52 | } 53 | ).transform(_.keys) 54 | 55 | def statelessListBannedUsers(ds : DStream[(UserId, Boolean)]) : DStream[UserId] = 56 | ds.map(_._1) 57 | 58 | def checkExtractBannedUsersList(testSubject : DStream[(UserId, Boolean)] => DStream[UserId]) = { 59 | val batchSize = 20 60 | val (headTimeout, tailTimeout, nestedTimeout) = (10, 10, 5) 61 | val (badId, ids) = (15L, Gen.choose(1L, 50L)) 62 | val goodBatch = BatchGen.ofN(batchSize, ids.map((_, true))) 63 | val badBatch = goodBatch + BatchGen.ofN(1, (badId, false)) 64 | val gen = BatchGen.until(goodBatch, badBatch, headTimeout) ++ 65 | BatchGen.always(Gen.oneOf(goodBatch, badBatch), tailTimeout) 66 | 67 | type U = (RDD[(UserId, Boolean)], RDD[UserId]) 68 | val (inBatch, outBatch) = ((_ : U)._1, (_ : U)._2) 69 | 70 | val formula = { 71 | val badInput = at(inBatch)(_ should existsRecord(_ == (badId, false))) 72 | val allGoodInputs = at(inBatch)(_ should foreachRecord(_._2 == true)) 73 | val noIdBanned = at(outBatch)(_.isEmpty) 74 | val badIdBanned = at(outBatch)(_ should existsRecord(_ == badId)) 75 | 76 | ( ( allGoodInputs and noIdBanned ) until badIdBanned on headTimeout ) and 77 | ( always { badInput ==> (always(badIdBanned) during nestedTimeout) } during tailTimeout ) 78 | } 79 | 80 | forAllDStream( 81 | gen)( 82 | testSubject)( 83 | formula) 84 | }.set(minTestsOk = 10).verbose 85 | 86 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/demo/StreamingFormulaManyArgsDemo.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.demo 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | import org.specs2.ScalaCheck 6 | import org.specs2.Specification 7 | import org.specs2.matcher.ResultMatchers 8 | import org.scalacheck.Arbitrary.arbitrary 9 | import org.scalacheck.Gen 10 | 11 | import org.apache.spark.rdd.RDD 12 | import org.apache.spark.streaming.Duration 13 | import org.apache.spark.streaming.dstream.DStream 14 | 15 | import es.ucm.fdi.sscheck.spark.streaming.SharedStreamingContextBeforeAfterEach 16 | import es.ucm.fdi.sscheck.prop.tl.{Formula,DStreamTLProperty} 17 | import es.ucm.fdi.sscheck.prop.tl.Formula._ 18 | import es.ucm.fdi.sscheck.gen.{PDStreamGen,BatchGen} 19 | import es.ucm.fdi.sscheck.matcher.specs2.RDDMatchers._ 20 | 21 | import scala.math 22 | 23 | @RunWith(classOf[JUnitRunner]) 24 | class StreamingFormulaManyArgsDemo 25 | extends Specification 26 | with DStreamTLProperty 27 | with ResultMatchers 28 | with ScalaCheck { 29 | 30 | // Spark configuration 31 | override def sparkMaster : String = "local[*]" 32 | override def batchDuration = Duration(450) 33 | override def defaultParallelism = 4 34 | 35 | def is = 36 | sequential ^ s2"""Simple sample sscheck properties with more than 1 input and/or output 37 | - where we can have 2 inputs and 1 output defined as a combination of both 38 | inputs ${cartesianStreamProp(cartesianDStreamOk)} 39 | - where a combination of 2 with a faulty transformation ${cartesianStreamProp(cartesianDStreamWrong) must beFailing} 40 | - where we can have 1 input and 2 outputs defined as a transformation of the input 41 | ${dstreamCartesianProp1To2} 42 | - where we can have 2 input and 2 outputs defined as a transformation of the inputs 43 | ${dstreamCartesianProp2To2} 44 | """ 45 | 46 | val (maxBatchSize, numBatches) = (50, 10) 47 | def genAlwaysInts(min: Int, max: Int) = 48 | BatchGen.always(BatchGen.ofNtoM(1, maxBatchSize, Gen.choose(min, max)), numBatches) 49 | 50 | def cartesianDStreamOk(xss: DStream[Int], yss: DStream[Int]): DStream[(Int, Int)] = 51 | xss.transformWith(yss, (xs: RDD[Int], ys: RDD[Int]) => { 52 | xs.cartesian(ys) 53 | }) 54 | 55 | def cartesianDStreamWrong(xss: DStream[Int], yss: DStream[Int]): DStream[(Int, Int)] = 56 | xss.map{x => (x,x)} 57 | 58 | def cartesianStreamProp(dstreamsCartesian: (DStream[Int], DStream[Int]) => DStream[(Int, Int)]) = { 59 | type U = (RDD[Int], RDD[Int], RDD[(Int, Int)]) 60 | val (xsMin, xsMax, ysMin, ysMax) = (1, 10, 20, 30) 61 | 62 | val formula = alwaysR[U] { case (xs, ys, xsys) => 63 | xsys.map(_._1).subtract(xs).count === 0 and 64 | xsys.map(_._2).subtract(ys).count === 0 65 | } during numBatches 66 | forAllDStream21( 67 | genAlwaysInts(1, 10), 68 | genAlwaysInts(20, 30))( 69 | dstreamsCartesian)( 70 | formula) 71 | }.set(minTestsOk = 10).verbose 72 | 73 | def dstreamCartesianProp1To2 = { 74 | type U = (RDD[Int], RDD[(Int, Int)], RDD[(Int, Int, Int)]) 75 | 76 | val formula = alwaysR[U] { case (xs, pairs, triples) => 77 | pairs.map(_._1).subtract(xs).count === 0 and 78 | pairs.map(_._2).subtract(xs).count === 0 and 79 | triples.map(_._1).subtract(xs).count === 0 and 80 | triples.map(_._2).subtract(xs).count === 0 and 81 | triples.map(_._3).subtract(xs).count === 0 82 | } during numBatches 83 | forAllDStream12( 84 | genAlwaysInts(1, 10))( 85 | _.map{x => (x,x)}, 86 | _.map{x => (x,x, x)})( 87 | formula) 88 | }.set(minTestsOk = 10).verbose 89 | 90 | def dstreamCartesianProp2To2 = { 91 | type U = (RDD[Int], RDD[Int], RDD[(Int, Int)], RDD[(Int, Int)]) 92 | 93 | val formula = alwaysR[U] { case (xs, ys, xsys, ysxs) => 94 | xsys.map(_._1).subtract(xs).count === 0 and 95 | xsys.map(_._2).subtract(ys).count === 0 and 96 | ysxs.map(_._1).subtract(ys).count === 0 and 97 | ysxs.map(_._2).subtract(xs).count === 0 98 | } during numBatches 99 | forAllDStream22( 100 | genAlwaysInts(1, 10), 101 | genAlwaysInts(20, 30))( 102 | (xs, ys) => cartesianDStreamOk(xs, ys), 103 | (xs, ys) => cartesianDStreamOk(ys, xs))( 104 | formula) 105 | }.set(minTestsOk = 10).verbose 106 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/demo/StreamingFormulaQuantDemo.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.demo 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | import org.specs2.ScalaCheck 6 | import org.specs2.Specification 7 | import org.specs2.matcher.ResultMatchers 8 | import org.scalacheck.Gen 9 | 10 | import org.apache.spark.rdd.RDD 11 | import org.apache.spark.streaming.Duration 12 | import org.apache.spark.streaming.dstream.DStream 13 | import org.apache.spark.streaming.dstream.DStream._ 14 | 15 | import scalaz.syntax.std.boolean._ 16 | 17 | import es.ucm.fdi.sscheck.prop.tl.DStreamTLProperty 18 | import es.ucm.fdi.sscheck.prop.tl.Formula._ 19 | import es.ucm.fdi.sscheck.gen.BatchGen 20 | import es.ucm.fdi.sscheck.gen.BatchGenConversions._ 21 | import es.ucm.fdi.sscheck.gen.PDStreamGenConversions._ 22 | 23 | @RunWith(classOf[JUnitRunner]) 24 | class StreamingFormulaQuantDemo 25 | extends Specification 26 | with DStreamTLProperty 27 | with ResultMatchers 28 | with ScalaCheck { 29 | 30 | // Spark configuration 31 | override def sparkMaster : String = "local[*]" 32 | override def batchDuration = Duration(400) 33 | override def defaultParallelism = 3 34 | override def enableCheckpointing = true 35 | 36 | def is = 37 | sequential ^ s2""" 38 | Check process to persistently detect and ban bad users 39 | - where a stateful implementation extracts the banned users correctly ${checkExtractBannedUsersList(listBannedUsers)} 40 | - where a trivial implementation ${checkExtractBannedUsersList(statelessListBannedUsers) must beFailing} 41 | """ 42 | type UserId = Long 43 | 44 | def listBannedUsers(ds : DStream[(UserId, Boolean)]) : DStream[UserId] = { 45 | val bannedUsers = ds.updateStateByKey((flags : Seq[Boolean], maybeFlagged : Option[Unit]) => 46 | maybeFlagged match { 47 | case Some(_) => maybeFlagged 48 | case None => flags.contains(false) option {()} 49 | } 50 | ).transform(_.keys) 51 | bannedUsers.foreachRDD(rdd => println(s"banned = ${rdd.collect().mkString(",")}")) 52 | bannedUsers 53 | } 54 | 55 | def statelessListBannedUsers(ds : DStream[(UserId, Boolean)]) : DStream[UserId] = { 56 | val banned = ds.map(_._1) 57 | banned.foreachRDD(rdd => println(s"banned = ${rdd.collect().mkString(",")}")) 58 | banned 59 | } 60 | 61 | def checkExtractBannedUsersList(testSubject : DStream[(UserId, Boolean)] => DStream[UserId]) = { 62 | val batchSize = 10 //20 63 | val (headTimeout, tailTimeout, nestedTimeout) = (10, 10, 5) 64 | val (badId, ids) = (Gen.oneOf(-1L, -2L), Gen.choose(1L, 50L)) 65 | val goodBatch = BatchGen.ofN(batchSize, ids.map((_, true))) 66 | val badBatch = goodBatch + BatchGen.ofN(1, badId.map((_, false))) 67 | val gen = BatchGen.until(goodBatch, badBatch, headTimeout) ++ 68 | BatchGen.always(Gen.oneOf(goodBatch, badBatch), tailTimeout) 69 | 70 | type U = (RDD[(UserId, Boolean)], RDD[UserId]) 71 | val (inBatch, outBatch) = ((_ : U)._1, (_ : U)._2) 72 | 73 | val badIdsAreAlwaysBanned = alwaysF[U]{ case (inBatch, _) => 74 | val badIds = inBatch.filter{ case (_, isGood) => ! isGood }. keys 75 | println(s"found badIds = ${badIds.collect.mkString(",")}") 76 | alwaysR[U]{ case (_, outBatch) => 77 | badIds.subtract(outBatch) isEmpty 78 | } during nestedTimeout 79 | } during tailTimeout 80 | 81 | // Same as badIdsAreAlwaysBanned but using at(), just to check the syntax is ok 82 | // and we can always use at(), for which we also need the trick 83 | // to avoid override, so it ends up being similar to using several nows, 84 | // because at() is most effective when combined with _ for simple assertions 85 | val badIdsAreAlwaysBannedAt = always{ atF(inBatch){ inBatch => 86 | val badIds = inBatch.filter{ case (_, isGood) => ! isGood }. keys 87 | always{ at(outBatch){outBatch => 88 | badIds.subtract(outBatch) isEmpty 89 | }} during nestedTimeout 90 | }} during tailTimeout 91 | 92 | // Same as badIdsAreAlwaysBanned but using overloads of now(), just to check the syntax is ok 93 | val badIdsAreAlwaysBannedNow = { 94 | always { nextF[U] { case (inBatch, _) => 95 | val badIds = inBatch.filter{ case (_, isGood) => ! isGood }. keys 96 | always { now[U] { case (_, outBatch) => 97 | badIds.subtract(outBatch) isEmpty 98 | }} during nestedTimeout 99 | }} during tailTimeout 100 | } 101 | 102 | // trivial example just to check that we can use the time in formulas 103 | val timeAlwaysIncreases = always( nextTime[U]{ (_, t1) => 104 | always( nowTime[U]{ (_, t2) => 105 | println(s"time $t2 should be greater than time $t1") 106 | t2.millis must beGreaterThan(t1.millis) 107 | }) during nestedTimeout 108 | }) during tailTimeout 109 | 110 | forAllDStream( 111 | gen)( 112 | testSubject)( 113 | badIdsAreAlwaysBanned and 114 | badIdsAreAlwaysBannedAt and 115 | badIdsAreAlwaysBannedNow and 116 | timeAlwaysIncreases) 117 | }.set(minTestsOk = 15).verbose 118 | } 119 | -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/simple/SimpleStreamingFormulas.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.simple 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | import org.specs2.matcher.ResultMatchers 6 | import org.scalacheck.Arbitrary.arbitrary 7 | import org.apache.spark.rdd.RDD 8 | import org.apache.spark.streaming.Duration 9 | import org.apache.spark.streaming.dstream.DStream 10 | import es.ucm.fdi.sscheck.spark.streaming.SharedStreamingContextBeforeAfterEach 11 | import es.ucm.fdi.sscheck.prop.tl.{Formula,DStreamTLProperty} 12 | import es.ucm.fdi.sscheck.prop.tl.Formula._ 13 | import es.ucm.fdi.sscheck.matcher.specs2.RDDMatchers._ 14 | import es.ucm.fdi.sscheck.gen.{PDStreamGen,BatchGen} 15 | import org.scalacheck.Gen 16 | import es.ucm.fdi.sscheck.gen.PDStream 17 | import es.ucm.fdi.sscheck.gen.Batch 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class SimpleStreamingFormulas 21 | extends org.specs2.Specification 22 | with DStreamTLProperty 23 | with org.specs2.ScalaCheck { 24 | 25 | // Spark configuration 26 | override def sparkMaster : String = "local[*]" 27 | override def batchDuration = Duration(50) 28 | override def defaultParallelism = 4 29 | 30 | def is = 31 | sequential ^ s2""" 32 | Simple demo Specs2 example for ScalaCheck properties with temporal 33 | formulas on Spark Streaming programs 34 | - Given a stream of integers 35 | When we filter out negative numbers 36 | Then we get only numbers greater or equal to 37 | zero $filterOutNegativeGetGeqZero 38 | - where time increments for each batch $timeIncreasesMonotonically 39 | """ 40 | 41 | def filterOutNegativeGetGeqZero = { 42 | type U = (RDD[Int], RDD[Int]) 43 | val numBatches = 10 44 | val gen = BatchGen.always(BatchGen.ofNtoM(10, 50, arbitrary[Int]), 45 | numBatches) 46 | val formula = always(nowTime[U]{ (letter, time) => 47 | val (_input, output) = letter 48 | output should foreachRecord {_ >= 0} 49 | }) during numBatches 50 | 51 | forAllDStream( 52 | gen)( 53 | _.filter{ x => !(x < 0)})( 54 | formula) 55 | }.set(minTestsOk = 50).verbose 56 | 57 | def timeIncreasesMonotonically = { 58 | type U = (RDD[Int], RDD[Int]) 59 | val numBatches = 10 60 | val gen = BatchGen.always(BatchGen.ofNtoM(10, 50, arbitrary[Int])) 61 | 62 | val formula = always(nextTime[U]{ (letter, time) => 63 | nowTime[U]{ (nextLetter, nextTime) => 64 | time.millis <= nextTime.millis 65 | } 66 | }) during numBatches-1 67 | 68 | forAllDStream( 69 | gen)( 70 | identity[DStream[Int]])( 71 | formula) 72 | }.set(minTestsOk = 10).verbose 73 | } -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/streaming/ScalaCheckStreamingTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.streaming 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | import org.specs2.ScalaCheck 6 | import org.specs2.execute.{AsResult, Result} 7 | 8 | import org.scalacheck.{Prop, Gen} 9 | import org.scalacheck.Arbitrary.arbitrary 10 | 11 | import org.apache.spark._ 12 | import org.apache.spark.rdd.RDD 13 | import org.apache.spark.streaming.{Duration} 14 | import org.apache.spark.streaming.dstream.DStream 15 | 16 | import es.ucm.fdi.sscheck.prop.tl.Formula._ 17 | import es.ucm.fdi.sscheck.prop.tl.DStreamTLProperty 18 | import es.ucm.fdi.sscheck.matcher.specs2.RDDMatchers._ 19 | 20 | @RunWith(classOf[JUnitRunner]) 21 | class ScalaCheckStreamingTest 22 | extends org.specs2.Specification 23 | with DStreamTLProperty 24 | with org.specs2.matcher.ResultMatchers 25 | with ScalaCheck { 26 | 27 | override def sparkMaster : String = "local[5]" 28 | override def batchDuration = Duration(350) 29 | override def defaultParallelism = 4 30 | 31 | def is = 32 | sequential ^ s2""" 33 | Simple properties for Spark Streaming 34 | - where the first property is a success $prop1 35 | - where a simple property for DStream.count is a success ${countProp(_.count)} 36 | - where a faulty implementation of the DStream.count is detected ${countProp(faultyCount) must beFailing} 37 | """ 38 | 39 | def prop1 = { 40 | val batchSize = 30 41 | val numBatches = 10 42 | val dsgenSeqSeq1 = { 43 | val zeroSeqSeq = Gen.listOfN(numBatches, Gen.listOfN(batchSize, 0)) 44 | val oneSeqSeq = Gen.listOfN(numBatches, Gen.listOfN(batchSize, 1)) 45 | Gen.oneOf(zeroSeqSeq, oneSeqSeq) 46 | } 47 | type U = (RDD[Int], RDD[Int]) 48 | 49 | forAllDStream[Int, Int]( 50 | "inputDStream" |: dsgenSeqSeq1)( 51 | (inputDs : DStream[Int]) => { 52 | val transformedDs = inputDs.map(_+1) 53 | transformedDs 54 | })(always ((u : U) => { 55 | val (inputBatch, transBatch) = u 56 | inputBatch.count === batchSize and 57 | inputBatch.count === transBatch.count and 58 | (inputBatch.intersection(transBatch).isEmpty should beTrue) and 59 | ( inputBatch should foreachRecord(_ == 0) or 60 | (inputBatch should foreachRecord(_ == 1)) 61 | ) 62 | }) during numBatches 63 | )}.set(minTestsOk = 10).verbose 64 | 65 | def faultyCount(ds : DStream[Double]) : DStream[Long] = 66 | ds.count.transform(_.map(_ - 1)) 67 | 68 | def countProp(testSubject : DStream[Double] => DStream[Long]) = { 69 | type U = (RDD[Double], RDD[Long]) 70 | val numBatches = 10 71 | forAllDStream[Double, Long]( 72 | Gen.listOfN(numBatches, Gen.listOfN(30, arbitrary[Double])))( 73 | testSubject 74 | )(always ((u : U) => { 75 | val (inputBatch, transBatch) = u 76 | transBatch.count === 1 and 77 | inputBatch.count === transBatch.first 78 | }) during numBatches 79 | )}.set(minTestsOk = 10).verbose 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/test/scala/es/ucm/fdi/sscheck/spark/streaming/SharedStreamingContextBeforeAfterEachTest.scala: -------------------------------------------------------------------------------- 1 | package es.ucm.fdi.sscheck.spark.streaming 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.runner.JUnitRunner 5 | import org.specs2.execute.Result 6 | 7 | import org.apache.spark.streaming.Duration 8 | import org.apache.spark.rdd.RDD 9 | 10 | import scala.collection.mutable.Queue 11 | import scala.concurrent.duration._ 12 | 13 | import org.slf4j.LoggerFactory 14 | 15 | import es.ucm.fdi.sscheck.matcher.specs2.RDDMatchers._ 16 | 17 | // sbt "test-only es.ucm.fdi.sscheck.spark.streaming.SharedStreamingContextBeforeAfterEachTest" 18 | 19 | @RunWith(classOf[JUnitRunner]) 20 | class SharedStreamingContextBeforeAfterEachTest 21 | extends org.specs2.Specification 22 | with org.specs2.matcher.MustThrownExpectations 23 | with org.specs2.matcher.ResultMatchers 24 | with SharedStreamingContextBeforeAfterEach { 25 | 26 | // cannot use private[this] due to https://issues.scala-lang.org/browse/SI-8087 27 | @transient private val logger = LoggerFactory.getLogger("SharedStreamingContextBeforeAfterEachTest") 28 | 29 | // Spark configuration 30 | override def sparkMaster : String = "local[5]" 31 | override def batchDuration = Duration(250) 32 | override def defaultParallelism = 3 33 | override def enableCheckpointing = false // as queueStream doesn't support checkpointing 34 | 35 | def is = 36 | sequential ^ s2""" 37 | Simple test for SharedStreamingContextBeforeAfterEach 38 | where a simple queueStream test must be successful $successfulSimpleQueueStreamTest 39 | where a simple queueStream test can also fail $failingSimpleQueueStreamTest 40 | """ 41 | 42 | def successfulSimpleQueueStreamTest = simpleQueueStreamTest(expectedCount = 0) 43 | def failingSimpleQueueStreamTest = simpleQueueStreamTest(expectedCount = 1) must beFailing 44 | 45 | def simpleQueueStreamTest(expectedCount : Int) : Result = { 46 | val record = "hola" 47 | val batches = Seq.fill(5)(Seq.fill(10)(record)) 48 | val queue = new Queue[RDD[String]] 49 | queue ++= batches.map(batch => sc.parallelize(batch, numSlices = defaultParallelism)) 50 | val inputDStream = ssc.queueStream(queue, oneAtATime = true) 51 | val sizesDStream = inputDStream.map(_.length) 52 | 53 | var batchCount = 0 54 | // NOTE wrapping assertions with a Result object is needed 55 | // to avoid the Spark Streaming runtime capturing the exceptions 56 | // from failing assertions 57 | var result : Result = ok 58 | inputDStream.foreachRDD { rdd => 59 | batchCount += 1 60 | println(s"completed batch number $batchCount: ${rdd.collect.mkString(",")}") 61 | result = result and { 62 | rdd.filter(_!= record).count() === expectedCount 63 | rdd should existsRecord(_ == "hola") 64 | } 65 | } 66 | sizesDStream.foreachRDD { rdd => 67 | result = result and { 68 | rdd should foreachRecord(record.length)(len => _ == len) 69 | } 70 | } 71 | 72 | // should only start the dstream after all the transformations and actions have been defined 73 | ssc.start() 74 | 75 | // wait for completion of batches.length batches 76 | StreamingContextUtils.awaitForNBatchesCompleted(batches.length, atMost = 10 seconds)(ssc) 77 | 78 | result 79 | } 80 | } 81 | 82 | 83 | 84 | 85 | --------------------------------------------------------------------------------