├── portals-core ├── js │ └── src │ │ └── main │ │ └── scala │ │ ├── .gitkeep │ │ └── portals │ │ └── util │ │ └── Logger.scala ├── jvm │ └── src │ │ ├── main │ │ └── scala │ │ │ ├── .gitkeep │ │ │ └── portals │ │ │ └── util │ │ │ └── Logger.scala │ │ └── test │ │ └── scala │ │ └── portals │ │ └── api │ │ └── builder │ │ ├── SequencerTest.scala │ │ └── GeneratorTest.scala └── shared │ └── src │ └── main │ └── scala │ └── portals │ ├── runtime │ ├── state │ │ ├── Snapshot.scala │ │ ├── Snapshottable.scala │ │ ├── StateBackend.scala │ │ └── MapStateBackendImpl.scala │ ├── GarbageCollectingRuntime.scala │ ├── interpreter │ │ ├── processors │ │ │ ├── InterpreterConnection.scala │ │ │ ├── Stepper.scala │ │ │ ├── InterpreterStream.scala │ │ │ ├── InterpreterGenerator.scala │ │ │ ├── InterpreterSplitter.scala │ │ │ └── InterpreterPortal.scala │ │ ├── trackers │ │ │ ├── SuspendingTracker.scala │ │ │ ├── GraphTracker.scala │ │ │ ├── ProgressTracker.scala │ │ │ └── StreamTracker.scala │ │ └── InterpreterContext.scala │ ├── PortalsRuntime.scala │ ├── distributed │ │ └── DistributedRuntime.scala │ ├── SteppingRuntime.scala │ ├── WrappedEvents.scala │ ├── BatchedEvents.scala │ ├── SuspendingRuntime.scala │ └── test │ │ └── TestRuntime.scala │ ├── util │ ├── Common.scala │ ├── Key.scala │ ├── Serializer.scala │ ├── JavaSerializer.scala │ └── Future.scala │ ├── application │ ├── task │ │ ├── Continuation.scala │ │ ├── ContinuationMeta.scala │ │ ├── TaskStateImpl.scala │ │ ├── OutputCollector.scala │ │ ├── TaskState.scala │ │ └── OutputCollectorImpl.scala │ ├── ASTPrinter.scala │ └── sequencer │ │ └── Sequencer.scala │ ├── compiler │ ├── CompilerContext.scala │ ├── phases │ │ ├── CompilerPhases.scala │ │ ├── CodeGeneration.scala │ │ ├── portal │ │ │ ├── RewritePortalPhase.scala │ │ │ └── RewritePortalEvents.scala │ │ ├── RuntimeCompilerPhases.scala │ │ └── WellFormedCheck.scala │ ├── physicalplan │ │ ├── PhysicalPlanPrinter.scala │ │ ├── PlanConfig.scala │ │ ├── PhysicalPlan.scala │ │ └── PhysicalPlanBuilder.scala │ ├── Compiler.scala │ ├── compilers │ │ ├── PreCheckCompiler.scala │ │ ├── PortalRewriteCompiler.scala │ │ └── BaseCompiler.scala │ ├── CompilerBuilder.scala │ ├── CompilerSubPhase.scala │ └── CompilerPhase.scala │ ├── system │ ├── PortalsSystem.scala │ ├── Systems.scala │ ├── ParallelSystem.scala │ └── TestSystem.scala │ └── api │ └── builder │ ├── FlowBuilderContext.scala │ ├── WorkflowBuilderImpl.scala │ ├── WorkflowBuilderContext.scala │ ├── ConnectionBuilder.scala │ ├── SplitBuilder.scala │ ├── SplitterBuilder.scala │ ├── ApplicationBuilderImpl.scala │ ├── SequencerBuilder.scala │ ├── WorkflowBuilder.scala │ ├── ApplicationBuilderContext.scala │ └── PortalBuilder.scala ├── portals-portalsjs ├── jvm │ └── src │ │ └── main │ │ └── scala │ │ └── .gitkeep ├── shared │ └── src │ │ └── main │ │ └── scala │ │ └── .gitkeep ├── js │ └── src │ │ └── main │ │ └── resources │ │ ├── examples │ │ ├── playground │ │ │ ├── helloVLDB.js.out │ │ │ ├── pingPong.js.out │ │ │ ├── simpleRecursive.js.out │ │ │ ├── wordCount.js.out │ │ │ ├── registry.js.out │ │ │ ├── helloVLDB.js │ │ │ ├── map.js │ │ │ ├── rangeFilter.js │ │ │ ├── billionEvents.js │ │ │ ├── sleepingBeauty.js │ │ │ ├── stateful.js │ │ │ ├── simpleRecursive.js │ │ │ ├── registry.js │ │ │ ├── multiDataflow.js │ │ │ ├── pingPong.js │ │ │ ├── wordCount.js │ │ │ ├── portalService.js │ │ │ ├── stateful.js.out │ │ │ ├── map.js.out │ │ │ └── portalAggregation.js │ │ └── basic │ │ │ ├── flowbuilder │ │ │ ├── map.js │ │ │ ├── filter.js │ │ │ ├── identity.js │ │ │ ├── flatMap.js │ │ │ ├── processor.js │ │ │ ├── task.js │ │ │ ├── union.js │ │ │ ├── filter.js.out │ │ │ ├── key.js │ │ │ ├── withStar.js │ │ │ ├── init.js │ │ │ ├── portalTask.js │ │ │ ├── portalStar.js │ │ │ ├── key.js.out │ │ │ ├── identity.js.out │ │ │ ├── processor.js.out │ │ │ ├── portalStar.js.out │ │ │ ├── portalTask.js.out │ │ │ ├── init.js.out │ │ │ ├── map.js.out │ │ │ ├── task.js.out │ │ │ ├── withStar.js.out │ │ │ └── flatMap.js.out │ │ │ ├── taskState.js │ │ │ ├── registry.js │ │ │ ├── sequencer.js │ │ │ ├── taskTest.js │ │ │ ├── splitter.js │ │ │ ├── taskState.js.out │ │ │ ├── generator.js │ │ │ ├── taskTest.js.out │ │ │ └── registry.js.out │ │ ├── portalsjs-run-testFile.sh │ │ ├── testFile.js │ │ ├── test-run-testFile.js │ │ └── obfuscator.js └── README.md ├── project ├── build.properties └── plugins.sbt ├── scripts ├── portalsjs │ ├── portalsjs-test-runner.sh │ ├── portalsjs-compile.sh │ ├── portalsjs-test-run-testFile.sh │ └── portalsjs-test-generate-outputs.sh └── deployment │ ├── docker │ └── Dockerfile │ └── kubernetes │ └── Deployment.yaml ├── .gitignore ├── portals-examples └── src │ ├── main │ └── scala │ │ └── portals │ │ └── examples │ │ ├── shoppingcart │ │ ├── ShoppingCartConfig.scala │ │ ├── ShoppingCartMain.scala │ │ ├── tasks │ │ │ ├── AggregatingTask.scala │ │ │ ├── OrdersTask.scala │ │ │ ├── InventoryTask.scala │ │ │ ├── AnalyticsTask.scala │ │ │ └── CartTask.scala │ │ ├── ShoppingCartData.scala │ │ ├── ThrottledGenerator.scala │ │ └── ShoppingCart.scala │ │ ├── bankaccount │ │ ├── BankAccountConfig.scala │ │ ├── BankAccountMain.scala │ │ ├── BankAccountData.scala │ │ ├── BankAccount.scala │ │ ├── BankAccountEvents.scala │ │ └── tasks │ │ │ └── TriggerTask.scala │ │ └── tutorial │ │ ├── EventFiltering.scala │ │ ├── HelloWorld.scala │ │ ├── PortalPingPong.scala │ │ ├── StepByStep.scala │ │ └── DynamicQuery.scala │ └── test │ └── scala │ └── portals │ └── examples │ ├── ExamplesTest.scala │ └── tutorial │ └── PingPongTest.scala ├── portals-benchmark └── src │ └── main │ └── scala │ └── portals │ └── benchmark │ ├── Benchmark.scala │ ├── BenchmarkConfig.scala │ ├── BenchmarkGrid.scala │ └── BenchmarkRunner.scala ├── portals-distributed └── src │ └── main │ └── scala │ └── portals │ └── distributed │ ├── SubmittableApplication.scala │ ├── Events.scala │ ├── examples │ ├── shoppingcart │ │ ├── Orders.scala │ │ ├── Cart.scala │ │ ├── Inventory.scala │ │ ├── Analytics.scala │ │ └── ShoppingCart.scala │ └── HelloWorld.scala │ ├── SBTRunServer.scala │ ├── Server.scala │ ├── ServerRuntime.scala │ └── Util.scala ├── .scalafmt.conf ├── NOTICE ├── .github └── workflows │ ├── js-build-test.yaml │ ├── build-test.yaml │ ├── docs.yml │ └── docs-versioned.yml └── portals-libraries └── src ├── test └── scala │ └── portals │ └── libraries │ └── actor │ ├── ExamplesTest.scala │ ├── ActorTest.scala │ └── ActorTestUtils.scala └── main └── scala └── portals └── libraries └── actor └── examples ├── PingPong.scala └── CountingActor.scala /portals-core/js/src/main/scala/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /portals-core/jvm/src/main/scala/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /portals-portalsjs/jvm/src/main/scala/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /portals-portalsjs/shared/src/main/scala/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.8.2 2 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/helloVLDB.js.out: -------------------------------------------------------------------------------- 1 | Hello VLDB! 2 | -------------------------------------------------------------------------------- /scripts/portalsjs/portalsjs-test-runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node portals-portalsjs/js/src/main/resources/test-runner.js; 3 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/state/Snapshot.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.state 2 | 3 | trait Snapshot 4 | -------------------------------------------------------------------------------- /scripts/portalsjs/portalsjs-compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sbt compile; 3 | sbt fastLinkJS; 4 | sbt fastOptJS; 5 | sbt fullOptJS; 6 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/portalsjs-run-testFile.sh: -------------------------------------------------------------------------------- 1 | node portals-portalsjs/js/src/main/resources/test-run-testFile.js 2 | -------------------------------------------------------------------------------- /scripts/portalsjs/portalsjs-test-run-testFile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node portals-portalsjs/js/src/main/resources/test-run-testFile.js; 3 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/pingPong.js.out: -------------------------------------------------------------------------------- 1 | 8 2 | 7 3 | 6 4 | 5 5 | 4 6 | 3 7 | 2 8 | 1 9 | Done! 10 | -------------------------------------------------------------------------------- /scripts/portalsjs/portalsjs-test-generate-outputs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node portals-portalsjs/js/src/main/resources/test-generate-outputs.js; 3 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/simpleRecursive.js.out: -------------------------------------------------------------------------------- 1 | 9 2 | 8 3 | 7 4 | 6 5 | 5 6 | 4 7 | 3 8 | 2 9 | 1 10 | 0 11 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/state/Snapshottable.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.state 2 | 3 | trait Snapshottable[S <: Snapshot]: 4 | def snapshot(): S 5 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/wordCount.js.out: -------------------------------------------------------------------------------- 1 | the,1 2 | quick,1 3 | brown,1 4 | fox,1 5 | jumps,1 6 | over,1 7 | the,2 8 | lazy,1 9 | dog,1 10 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/util/Common.scala: -------------------------------------------------------------------------------- 1 | package portals.util 2 | 3 | object Common: 4 | object Types: 5 | type Path = String 6 | type Filter[T] = T => Boolean 7 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/util/Key.scala: -------------------------------------------------------------------------------- 1 | package portals.util 2 | 3 | trait Key: 4 | val x: Long 5 | 6 | object Key: 7 | def apply(_x: Long): Key = new Key { val x = _x } 8 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/task/Continuation.scala: -------------------------------------------------------------------------------- 1 | package portals.application.task 2 | 3 | type Continuation[T, U, Req, Rep] = AskerTaskContext[T, U, Req, Rep] ?=> Unit 4 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/util/Serializer.scala: -------------------------------------------------------------------------------- 1 | package portals.util 2 | 3 | trait Serializer: 4 | def serialize[T](obj: T): Array[Byte] 5 | 6 | def deserialize[T](bytes: Array[Byte]): T 7 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") 2 | addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.13.2") 3 | addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.0.0") 4 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/GarbageCollectingRuntime.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime 2 | 3 | trait GarbageCollectingRuntime: 4 | /** Perform GC on the runtime objects. */ 5 | def garbageCollect(): Unit 6 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/registry.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 16 4 | 81 5 | 256 6 | 625 7 | 1296 8 | 2401 9 | 4096 10 | 6561 11 | 10000 12 | 14641 13 | 20736 14 | 28561 15 | 38416 16 | 50625 17 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/ASTPrinter.scala: -------------------------------------------------------------------------------- 1 | package portals.application 2 | 3 | object ASTPrinter: 4 | def println(ast: AST): Unit = pprint.pprintln(ast) 5 | 6 | def toString(ast: AST): String = pprint.apply(ast).plainText 7 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/CompilerContext.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler 2 | 3 | /** Compiler context, to be used within the compiler. */ 4 | private[portals] class CompilerContext: 5 | /** Log a message. */ 6 | def log(msg: String): Unit = println(msg) 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # sbt 2 | project/target 3 | project/project/target 4 | target/ 5 | 6 | # macOS 7 | *.DS_Store 8 | 9 | # VS Code 10 | .idea/ 11 | .bsp/ 12 | .bloop/ 13 | .metals/ 14 | .vscode/ 15 | *metals.sbt 16 | 17 | # Node 18 | node_modules/ 19 | package-lock.json 20 | package.json -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/ShoppingCartConfig.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart 2 | 3 | object ShoppingCartConfig: 4 | inline val N_EVENTS = 1024 * 128 5 | inline val N_USERS = 128 6 | inline val N_ITEMS = 1024 7 | inline val LOGGING = false 8 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/task/ContinuationMeta.scala: -------------------------------------------------------------------------------- 1 | package portals.application.task 2 | 3 | import portals.util.Key 4 | 5 | case class ContinuationMeta( 6 | id: Int, 7 | asker: String, 8 | portal: String, 9 | portalAsker: String, 10 | askerKey: Key, 11 | ) 12 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/bankaccount/BankAccountConfig.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.bankaccount 2 | 3 | object BankAccountConfig: 4 | inline val N_EVENTS = 1024 5 | inline val N_ACCOUNTS = 128 6 | inline val N_OPS_PER_SAGA = 4 7 | inline val LOGGING = false 8 | inline val STARTING_BALANCE = 100 9 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/phases/CompilerPhases.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.phases 2 | 3 | import portals.compiler.phases.portal.* 4 | 5 | private[portals] object CompilerPhases: 6 | def wellFormedCheck = WellFormedCheck 7 | def codeGeneration = CodeGeneration 8 | def rewritePortal = RewritePortalPhase 9 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/state/StateBackend.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.state 2 | 3 | trait StateBackend extends Snapshottable[Snapshot]: 4 | def get(key: Any): Option[Any] 5 | def set(key: Any, value: Any): Unit 6 | def del(key: Any): Unit 7 | def clear(): Unit 8 | def iterator: Iterator[(Any, Any)] 9 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/processors/InterpreterConnection.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.processors 2 | 3 | import portals.application.AtomicConnection 4 | 5 | /** Internal API. Wraps around an atomic connection. */ 6 | private[portals] class InterpreterConnection(val connection: AtomicConnection[_]) 7 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/system/PortalsSystem.scala: -------------------------------------------------------------------------------- 1 | package portals.system 2 | 3 | import portals.application.Application 4 | 5 | trait PortalsSystem: 6 | /** Launch a Portals application. */ 7 | def launch(application: Application): Unit 8 | 9 | /** Terminate the system and cleanup. */ 10 | def shutdown(): Unit 11 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/PortalsRuntime.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime 2 | 3 | import portals.application.Application 4 | 5 | trait PortalsRuntime: 6 | /** Launch an application. */ 7 | def launch(application: Application): Unit 8 | 9 | /** Terminate the runtime. */ 10 | def shutdown(): Unit 11 | 12 | end PortalsRuntime 13 | -------------------------------------------------------------------------------- /portals-benchmark/src/main/scala/portals/benchmark/Benchmark.scala: -------------------------------------------------------------------------------- 1 | package portals.benchmark 2 | 3 | // see https://github.com/shamsimam/savina/blob/master/src/main/java/edu/rice/habanero/benchmarks/Benchmark.java 4 | trait Benchmark: 5 | val name: String 6 | def initialize(args: List[String]): Unit 7 | def runOneIteration(): Unit 8 | def cleanupOneIteration(): Unit 9 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/distributed/DistributedRuntime.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.distributed 2 | 3 | import portals.application.Application 4 | import portals.runtime.PortalsRuntime 5 | 6 | private[portals] class DistributedRuntime extends PortalsRuntime: 7 | def launch(application: Application): Unit = ??? 8 | def shutdown(): Unit = ??? 9 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/testFile.js: -------------------------------------------------------------------------------- 1 | var builder = PortalsJS.ApplicationBuilder("helloWorld") 2 | var _ = builder.workflows 3 | .source(builder.generators.fromArray(["Hello World!"]).stream) 4 | .logger() 5 | .sink() 6 | .freeze() 7 | var helloWorld = builder.build() 8 | var system = PortalsJS.System() 9 | system.launch(helloWorld) 10 | system.stepUntilComplete() 11 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/helloVLDB.js: -------------------------------------------------------------------------------- 1 | var builder = PortalsJS.ApplicationBuilder("helloVLDB") 2 | var _ = builder.workflows 3 | .source(builder.generators.fromArray(["Hello VLDB!"]).stream) 4 | .logger() 5 | .sink() 6 | .freeze() 7 | var helloVLDB = builder.build() 8 | var system = PortalsJS.System() 9 | system.launch(helloVLDB) 10 | system.stepUntilComplete() -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/map.js: -------------------------------------------------------------------------------- 1 | var builder = PortalsJS.ApplicationBuilder("map") 2 | var _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .map(ctx => x => [x, x.toString().length]) 5 | .logger() 6 | .sink() 7 | .freeze() 8 | var map = builder.build() 9 | var system = PortalsJS.System() 10 | system.launch(map) 11 | system.stepUntilComplete() -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/physicalplan/PhysicalPlanPrinter.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.physicalplan 2 | 3 | private[portals] object PhysicalPlanPrinter: 4 | /** Print the physical plan to the console. */ 5 | def println(ast: PhysicalPlan[_]): Unit = pprint.pprintln(ast) 6 | 7 | /** Print the physical plan to a string. */ 8 | def toString(ast: PhysicalPlan[_]): String = pprint.apply(ast).plainText 9 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/rangeFilter.js: -------------------------------------------------------------------------------- 1 | var builder = PortalsJS.ApplicationBuilder("rangeFilter") 2 | var _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 1024, 8).stream) 4 | .filter(x => x % 2 == 0) 5 | .logger() 6 | .sink() 7 | .freeze() 8 | var rangeFilter = builder.build() 9 | var system = PortalsJS.System() 10 | system.launch(rangeFilter) 11 | system.stepUntilComplete() -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/map.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("map"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .map(ctx => x => x * x) 5 | .logger() 6 | .sink() 7 | .freeze(); 8 | let map = builder.build(); 9 | let system = PortalsJS.System(); 10 | system.launch(map); 11 | system.stepUntilComplete(); 12 | system.shutdown(); 13 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/SubmittableApplication.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed 2 | 3 | import portals.application.Application 4 | 5 | /** Submittable portals application. Extend this trait from an object in order 6 | * to submit an application to the server. 7 | * 8 | * @see 9 | * [[portals.distributed.examples.HelloWorld]] 10 | */ 11 | trait SubmittableApplication: 12 | def apply(): Application 13 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/filter.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("filter"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .filter(x => x % 2 == 0) 5 | .logger() 6 | .sink() 7 | .freeze(); 8 | let filter = builder.build(); 9 | let system = PortalsJS.System(); 10 | system.launch(filter); 11 | system.stepUntilComplete(); 12 | system.shutdown(); 13 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/trackers/SuspendingTracker.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.trackers 2 | 3 | import portals.util.Common.Types.Path 4 | 5 | class SuspendingTracker: 6 | private var _suspended: Set[Path] = Set.empty 7 | 8 | def isSuspended(path: Path): Boolean = _suspended.contains(path) 9 | 10 | def add(path: Path): Unit = _suspended += path 11 | 12 | def remove(path: Path): Unit = _suspended -= path 13 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/bankaccount/BankAccountMain.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.bankaccount 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.* 6 | import portals.application.ASTPrinter 7 | import portals.system.Systems 8 | 9 | object BankAccountMain extends App: 10 | val app = BankAccount.app 11 | val system = Systems.test() 12 | val _ = system.launch(app) 13 | system.stepUntilComplete() 14 | system.shutdown() 15 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/identity.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("identity"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .key(x => 0) 5 | .identity() 6 | .logger() 7 | .sink() 8 | .freeze(); 9 | let identity = builder.build(); 10 | let system = PortalsJS.System(); 11 | system.launch(identity); 12 | system.stepUntilComplete(); 13 | system.shutdown(); 14 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/billionEvents.js: -------------------------------------------------------------------------------- 1 | var builder = PortalsJS.ApplicationBuilder("billionEvents") 2 | var _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 1024*1024*1024, 1024*1024).stream) 4 | .filter(x => x % (1024*1024) == 0) 5 | .logger() 6 | .sink() 7 | .freeze() 8 | var billionEvents = builder.build() 9 | var system = PortalsJS.System() 10 | system.launch(billionEvents) 11 | system.stepUntilComplete() -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/flatMap.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("flatMap"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .flatMap(ctx => x => { return [x, x * x]; }) 5 | .logger() 6 | .sink() 7 | .freeze(); 8 | let flatMap = builder.build(); 9 | let system = PortalsJS.System(); 10 | system.launch(flatMap); 11 | system.stepUntilComplete(); 12 | system.shutdown(); 13 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/processor.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("processor"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .processor(ctx => x => ctx.emit(x * 2)) 5 | .logger() 6 | .sink() 7 | .freeze(); 8 | let processor = builder.build(); 9 | let system = PortalsJS.System(); 10 | system.launch(processor); 11 | system.stepUntilComplete(); 12 | system.shutdown(); 13 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/task.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("task"); 2 | let t = builder.tasks.map(ctx => x => x * x); 3 | let _ = builder.workflows 4 | .source(builder.generators.fromRange(0, 128, 8).stream) 5 | .task(t) 6 | .logger() 7 | .sink() 8 | .freeze(); 9 | let task = builder.build(); 10 | let system = PortalsJS.System(); 11 | system.launch(task); 12 | system.stepUntilComplete(); 13 | system.shutdown(); 14 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 3.7.4 2 | runner.dialect = scala3 3 | 4 | preset=default 5 | maxColumn = 120 6 | docstrings.wrapMaxColumn = 80 7 | 8 | rewrite.trailingCommas.style = "keep" 9 | 10 | align.preset = none 11 | 12 | rewrite.rules = [Imports] 13 | rewrite.imports.sort = original 14 | rewrite.imports.expand = true 15 | rewrite.imports.groups = [ 16 | ["java.*"], 17 | ["scala.*"], 18 | ["pekko.*"], 19 | ["org.*"], 20 | ["com.*"], 21 | ["ch.*"], 22 | ["portals.*"], 23 | ] -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/test-run-testFile.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const portalsJS = 'portals-portalsjs/js/target/scala-3.3.0/portals-portalsjs-fastopt.js'; // the PortalsJS library 4 | const testFile = 'portals-portalsjs/js/src/main/resources/testFile.js'; // testFile location 5 | const portalsJSCode = fs.readFileSync(portalsJS, 'utf8'); 6 | const example = fs.readFileSync(testFile, 'utf8'); 7 | const exampleCode = portalsJSCode + "\n" + example; 8 | eval(exampleCode); 9 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/phases/CodeGeneration.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.phases 2 | 3 | import portals.application.Application 4 | import portals.compiler.* 5 | import portals.compiler.physicalplan.* 6 | 7 | private[portals] object CodeGeneration extends CompilerPhase[Application, PhysicalPlan[_]]: 8 | override def run(application: Application)(using ctx: CompilerContext): PhysicalPlan[_] = 9 | PhysicalPlanBuilder.fromApplication(application) 10 | 11 | end CodeGeneration // object 12 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/union.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("union"); 2 | let flow = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | let flow1 = flow 5 | .identity() 6 | let flow2 = flow 7 | .identity() 8 | let _ = flow1 9 | .union([flow2]) 10 | .logger() 11 | .sink() 12 | .freeze(); 13 | let union = builder.build(); 14 | let system = PortalsJS.System(); 15 | system.launch(union); 16 | system.stepUntilComplete(); 17 | system.shutdown(); 18 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/Compiler.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler 2 | 3 | /** Compiler that transforms an input of type `T` into an output of type `U`. 4 | * 5 | * @tparam T 6 | * input type 7 | * @tparam U 8 | * output type 9 | */ 10 | private[portals] trait Compiler[T, U]: 11 | 12 | /** Compile the input into an output. 13 | * 14 | * @param t 15 | * input to be transformed 16 | * @return 17 | * the transformed output 18 | */ 19 | def compile(t: T): U 20 | end Compiler // trait 21 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/filter.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 2 3 | 4 4 | 6 5 | 8 6 | 10 7 | 12 8 | 14 9 | 16 10 | 18 11 | 20 12 | 22 13 | 24 14 | 26 15 | 28 16 | 30 17 | 32 18 | 34 19 | 36 20 | 38 21 | 40 22 | 42 23 | 44 24 | 46 25 | 48 26 | 50 27 | 52 28 | 54 29 | 56 30 | 58 31 | 60 32 | 62 33 | 64 34 | 66 35 | 68 36 | 70 37 | 72 38 | 74 39 | 76 40 | 78 41 | 80 42 | 82 43 | 84 44 | 86 45 | 88 46 | 90 47 | 92 48 | 94 49 | 96 50 | 98 51 | 100 52 | 102 53 | 104 54 | 106 55 | 108 56 | 110 57 | 112 58 | 114 59 | 116 60 | 118 61 | 120 62 | 122 63 | 124 64 | 126 65 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/obfuscator.js: -------------------------------------------------------------------------------- 1 | const JavaScriptObfuscator = require('javascript-obfuscator'); 2 | const fs = require('fs'); 3 | 4 | // Read the original JavaScript file 5 | const originalCode = fs.readFileSync('portals-portalsjs/js/target/scala-3.3.0/portals-portalsjs-opt.js', 'utf8'); 6 | // Obfuscate the code 7 | const obfuscatedCode = JavaScriptObfuscator.obfuscate(originalCode).getObfuscatedCode(); 8 | // Write the obfuscated code to a new file 9 | fs.writeFileSync('portals-portalsjs/js/target/scala-3.3.0/portals-portalsjs.js', obfuscatedCode, 'utf8'); 10 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/FlowBuilderContext.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | private[portals] trait FlowBuilderContext[T, U](using val wbctx: WorkflowBuilderContext[T, U]): 4 | // latest task that has been created 5 | val latest: Option[String] = None // immutable 6 | 7 | object FlowBuilderContext: 8 | // creates a context with the name of the latest created task 9 | def apply[T, U](name: Option[String])(using WorkflowBuilderContext[T, U]) = 10 | new FlowBuilderContext[T, U] { 11 | override val latest: Option[String] = name 12 | } 13 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/key.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("key"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .key(x => x % 2) 5 | .map(ctx => x => { 6 | let state = PortalsJS.PerKeyState("state", 0, ctx); 7 | state.set(state.get() + 1); 8 | return state.get(); 9 | }) 10 | .logger() 11 | .sink() 12 | .freeze(); 13 | let key = builder.build(); 14 | let system = PortalsJS.System(); 15 | system.launch(key); 16 | system.stepUntilComplete(); 17 | system.shutdown(); 18 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/system/Systems.scala: -------------------------------------------------------------------------------- 1 | package portals.system 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.application.Application 6 | 7 | trait Systems 8 | 9 | import portals.system.PortalsSystem 10 | import portals.system.TestSystem 11 | 12 | object Systems extends Systems: 13 | def default(): PortalsSystem = test() 14 | 15 | def test(): TestSystem = new TestSystem() 16 | 17 | def test(seed: Int): TestSystem = new TestSystem(Some(seed)) 18 | 19 | @experimental 20 | def parallel(nThreads: Int): PortalsSystem = new ParallelSystem(nThreads) 21 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/ShoppingCartMain.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.system.Systems 6 | 7 | object ShoppingCartMain extends App: 8 | val system = Systems.test() 9 | val _ = system.launch(ShoppingCart.app) 10 | // -- to execute for longer time, uncomment the following two lines: -- 11 | // val now = System.currentTimeMillis() 12 | // while System.currentTimeMillis() - now < 10_000_000 do system.stepUntilComplete() 13 | system.stepUntilComplete() 14 | system.shutdown() 15 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/withStar.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("withStar"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .identity() 5 | .withName("withStar") 6 | .withOnNext(ctx => x => ctx.emit(-x)) 7 | .withOnAtomComplete(ctx => ctx.emit(1)) 8 | .withOnComplete(ctx => ctx.emit(2)) 9 | .logger() 10 | .sink() 11 | .freeze(); 12 | let withStar = builder.build(); 13 | let system = PortalsJS.System(); 14 | system.launch(withStar); 15 | system.stepUntilComplete(); 16 | system.shutdown(); 17 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/system/ParallelSystem.scala: -------------------------------------------------------------------------------- 1 | package portals.system 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.application.Application 6 | import portals.runtime.parallel.ParallelRuntime 7 | 8 | @experimental 9 | class ParallelSystem(nThreads: Int) extends PortalsSystem: 10 | val runtime = ParallelRuntime(nThreads) 11 | 12 | /** Launch a Portals application. */ 13 | def launch(application: Application): Unit = 14 | runtime.launch(application) 15 | 16 | /** Terminate the system and cleanup. */ 17 | def shutdown(): Unit = 18 | runtime.shutdown() 19 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/sleepingBeauty.js: -------------------------------------------------------------------------------- 1 | function sleep(milliseconds) { 2 | const start = Date.now(); 3 | while (Date.now() - start < milliseconds) {} 4 | } 5 | let builder = PortalsJS.ApplicationBuilder("sleepingBeauty") 6 | let _ = builder.workflows 7 | .source(builder.generators.fromRange(0, 1024, 2).stream) 8 | .map(ctx => x => { 9 | sleep(500); 10 | return x; 11 | }) 12 | .logger() 13 | .sink() 14 | .freeze() 15 | let sleepingBeauty = builder.build() 16 | let system = PortalsJS.System() 17 | system.launch(sleepingBeauty) 18 | system.stepUntilComplete() -------------------------------------------------------------------------------- /scripts/deployment/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sbtscala/scala-sbt:openjdk-oraclelinux8-11.0.16_1.8.1_3.2.1 2 | 3 | COPY ./portals-benchmark/ /app/portals-benchmark 4 | COPY ./portals-core/ /app/portals-core 5 | COPY ./portals-distributed/ /app/portals-distributed 6 | COPY ./portals-examples/ /app/portals-examples 7 | COPY ./portals-libraries/ /app/portals-libraries 8 | COPY ./project/ /app/project 9 | COPY ./build.sbt /app/build.sbt 10 | 11 | WORKDIR /app 12 | 13 | RUN sbt compile 14 | 15 | RUN sbt publishLocal 16 | 17 | EXPOSE 8080 18 | 19 | CMD sbt "distributed/runMain portals.distributed.SBTRunServer 0.0.0.0 8080" 20 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/InterpreterContext.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter 2 | 3 | import scala.util.Random 4 | 5 | import portals.application.Application 6 | import portals.runtime.interpreter.trackers.* 7 | 8 | private[portals] class InterpreterContext(seed: Option[Int] = None): 9 | val rctx = new ApplicationTracker() 10 | val progressTracker = new ProgressTracker() 11 | val streamTracker = new StreamTracker() 12 | val graphTracker = new GraphTracker() 13 | val suspendingTracker = new SuspendingTracker() 14 | val rnd = seed.map(new Random(_)).getOrElse(new Random()) 15 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Portals 2 | 3 | Copyright 2022-2023 The Portals Project Committee 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/processors/Stepper.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.processors 2 | 3 | import portals.runtime.BatchedEvents.* 4 | 5 | enum StepperType: 6 | case Processing 7 | case Generating 8 | 9 | private[portals] sealed trait Stepper: 10 | val tpe: StepperType 11 | 12 | private[portals] trait ProcessingStepper extends Stepper: 13 | override val tpe = StepperType.Processing 14 | def step(atom: EventBatch): List[EventBatch] 15 | 16 | private[portals] trait GeneratingStepper extends Stepper: 17 | override val tpe = StepperType.Generating 18 | def step(): List[EventBatch] 19 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/init.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("init"); 2 | let _ = builder.workflows 3 | .source(builder.generators.fromRange(0, 128, 8).stream) 4 | .key(x => 0) 5 | .init(ctx => { 6 | let state = PortalsJS.PerKeyState("state", 0, ctx); 7 | return builder.tasks.map(c => x => { 8 | state.set(state.get() + x); 9 | return state.get(); 10 | }); 11 | }) 12 | .logger() 13 | .sink() 14 | .freeze(); 15 | let init = builder.build(); 16 | let system = PortalsJS.System(); 17 | system.launch(init); 18 | system.stepUntilComplete(); 19 | system.shutdown(); 20 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/taskState.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("taskStateTest"); 2 | let generator = builder.generators.fromRange(0, 128, 8); 3 | builder.workflows 4 | .source(generator.stream) 5 | .key(x => x % 16) 6 | .processor(ctx => x => { 7 | let state = PortalsJS.PerKeyState("state", 0, ctx); 8 | state.set(state.get() + 1); 9 | ctx.emit(state.get()); 10 | }) 11 | .logger() 12 | .sink() 13 | .freeze(); 14 | let taskStateTest = builder.build(); 15 | let system = PortalsJS.System(); 16 | system.launch(taskStateTest); 17 | system.stepUntilComplete(); 18 | system.shutdown(); 19 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/state/MapStateBackendImpl.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.state 2 | 3 | private[portals] class MapStateBackendImpl extends StateBackend: 4 | private val map = scala.collection.mutable.Map[Any, Any]() 5 | override def get(key: Any): Option[Any] = map.get(key) 6 | override def set(key: Any, value: Any): Unit = map.put(key, value) 7 | override def del(key: Any): Unit = map.remove(key) 8 | override def clear(): Unit = map.clear() 9 | override def iterator: Iterator[(Any, Any)] = map.iterator 10 | override def snapshot(): Snapshot = { val c = map.clone(); new Snapshot { def iterator = c.iterator } } 11 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/stateful.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("stateful") 2 | 3 | let generator = builder.generators.fromRange(0, 128, 1) 4 | 5 | let workflow = builder.workflows 6 | .source(generator.stream) 7 | .key(x => (x % 4)) 8 | .processor(ctx => x => { 9 | let state = PortalsJS.PerKeyState("state", 0, ctx); 10 | state.set(state.get() + x); 11 | ctx.emit(state.get()); 12 | return; 13 | }) 14 | .logger() 15 | .sink() 16 | .freeze() 17 | 18 | let stateful = builder.build() 19 | let system = PortalsJS.System() 20 | system.launch(stateful) 21 | system.stepUntilComplete() 22 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/physicalplan/PlanConfig.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.physicalplan 2 | 3 | /** Configuration for a physical plan. */ 4 | private[portals] sealed trait PlanConfig 5 | 6 | /** Base configuration of a physical plan with a keyrange. 7 | * 8 | * @param keyrange 9 | * The keyrange of this plan, right-open, i.e. [start, end), (exclusive end) 10 | * (except for Long.MaxValue). 11 | */ 12 | private[portals] case class BaseConfig(keyrange: (Long, Long)) extends PlanConfig 13 | 14 | /** Empty configuration of a physical plan. To be used for building a plan. */ 15 | private[portals] object EmptyConfig extends PlanConfig 16 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/portalTask.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("portalTask"); 2 | let portal = builder.portal.portal("portal"); 3 | builder.workflows 4 | .source(builder.generators.fromRange(0, 128, 8).stream) 5 | .askerreplier(portal, portal, ctx => x => { 6 | let f = ctx.ask(portal, x); 7 | ctx.await(f, c => { 8 | ctx.emit(f.value(ctx)); 9 | }); 10 | }, ctx => x => ctx.reply(-x)) 11 | .logger() 12 | .sink() 13 | .freeze(); 14 | let portalTask = builder.build(); 15 | let system = PortalsJS.System(); 16 | system.launch(portalTask); 17 | system.stepUntilComplete(); 18 | system.shutdown(); 19 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/registry.js: -------------------------------------------------------------------------------- 1 | let builder1 = PortalsJS.ApplicationBuilder("registryTest1"); 2 | let generator = builder1.generatorsWithName("name").fromRange(0, 128, 8); 3 | let app1 = builder1.build(); 4 | let builder2 = PortalsJS.ApplicationBuilder("registryTest2"); 5 | let foreignStream = builder2.registry.streams.get("/registryTest1/generators/name/stream"); 6 | builder2.workflows 7 | .source(foreignStream) 8 | .logger("from app2: ") 9 | .sink() 10 | .freeze(); 11 | let app2 = builder2.build(); 12 | let system = PortalsJS.System(); 13 | system.launch(app1); 14 | system.launch(app2); 15 | system.stepUntilComplete(); 16 | system.shutdown(); 17 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/WorkflowBuilderImpl.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | 5 | private[portals] class WorkflowBuilderImpl[T, U](using wbctx: WorkflowBuilderContext[T, U]) 6 | extends WorkflowBuilder[T, U]: 7 | 8 | // internal, do not use, completes anything that wasn't frozen 9 | private[portals] override def complete(): Unit = 10 | try wbctx.freeze() 11 | catch e => () 12 | 13 | override def freeze(): Workflow[T, U] = wbctx.freeze() 14 | 15 | override def source[TT >: T <: T](ref: AtomicStreamRefKind[T]): FlowBuilder[T, U, TT, TT] = 16 | FlowBuilder(None).source(ref) 17 | end WorkflowBuilderImpl 18 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/sequencer.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("sequencerTest"); 2 | let sequencer = builder.sequencers.random(); 3 | let generator1 = builder.generators.fromRange(0, 128, 8); 4 | let generator2 = builder.generators.fromRange(0, 128, 8); 5 | builder.connections.connect(generator1.stream, sequencer); 6 | builder.connections.connect(generator2.stream, sequencer); 7 | builder.workflows 8 | .source(sequencer.stream) 9 | .identity() 10 | .logger() 11 | .sink() 12 | .freeze(); 13 | let sequencerTest = builder.build(); 14 | let system = PortalsJS.System(); 15 | system.launch(sequencerTest); 16 | system.stepUntilComplete(); 17 | system.shutdown(); 18 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/simpleRecursive.js: -------------------------------------------------------------------------------- 1 | var builder = PortalsJS.ApplicationBuilder("simpleRecursive") 2 | var gen = builder.generators.fromArray([10]) 3 | var seq = builder.sequencers.random() 4 | var recursiveWorkflow = builder.workflows 5 | .source(seq.stream) 6 | .processor(ctx => x => { 7 | if (x > 0) { 8 | ctx.emit(x - 1) 9 | } 10 | }) 11 | .logger() 12 | .sink() 13 | .freeze() 14 | var _ = builder.connections.connect(gen.stream, seq) 15 | var _ = builder.connections.connect(recursiveWorkflow.stream, seq) 16 | var simpleRecursive = builder.build() 17 | var system = PortalsJS.System() 18 | system.launch(simpleRecursive) 19 | system.stepUntilComplete() -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/Events.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed 2 | 3 | import upickle.default._ 4 | 5 | /** `Events` handled by the `Server`. */ 6 | object Events: 7 | /** `Launch` the `application` specified by its Java Path. 8 | * 9 | * Note: the corresponding classfiles will need to have been uploaded in 10 | * advance. 11 | */ 12 | case class Launch(application: String) derives ReadWriter 13 | 14 | /** The `path` and `bytes` of a classfile to be uploaded. */ 15 | case class ClassFileInfo(path: String, bytes: Array[Byte]) derives ReadWriter 16 | 17 | /** Submit the `classFiles` to the server. */ 18 | case class SubmitClassFiles(classFiles: Seq[ClassFileInfo]) derives ReadWriter 19 | -------------------------------------------------------------------------------- /.github/workflows/js-build-test.yaml: -------------------------------------------------------------------------------- 1 | name: JS build and test 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v3 15 | - name: Setup JDK 16 | uses: actions/setup-java@v3 17 | with: 18 | distribution: temurin 19 | java-version: 17 20 | - name: Install Node.js 21 | uses: actions/setup-node@v3 22 | with: 23 | node-version: 19 24 | - name: Build 25 | run: sh scripts/portalsjs/portalsjs-compile.sh 26 | - name: Test 27 | run: sh scripts/portalsjs/portalsjs-test-runner.sh 28 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/taskTest.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("taskTest"); 2 | let task1 = builder.tasks.map(ctx => x => x + 1); 3 | let task2 = builder.tasks.processor(ctx => x => ctx.emit(x - 1)); 4 | let task3 = builder.tasks.filter(x => x % 2 == 0); 5 | let task4 = builder.tasks.flatMap(ctx => x => [x, -x]); 6 | let generator = builder.generators.fromRange(0, 128, 8); 7 | builder.workflows 8 | .source(generator.stream) 9 | .task(task1) 10 | .task(task2) 11 | .task(task3) 12 | .task(task4) 13 | .logger() 14 | .sink() 15 | .freeze(); 16 | let taskTest = builder.build(); 17 | let system = PortalsJS.System(); 18 | system.launch(taskTest); 19 | system.stepUntilComplete(); 20 | system.shutdown(); 21 | -------------------------------------------------------------------------------- /portals-portalsjs/README.md: -------------------------------------------------------------------------------- 1 | # PortalsJS 2 | 3 | ## Compile, Test, and Run 4 | The final compiled file is: `portals-portalsjs/js/target/scala-X/portals-portalsjs-opt.js`. 5 | 6 | Compile: 7 | ```bash 8 | sh scripts/portalsjs/portalsjs-compile.sh 9 | ``` 10 | 11 | Test: 12 | ```bash 13 | sh scripts/portalsjs/portalsjs-test-runner.sh 14 | ``` 15 | 16 | Generate new test outputs: 17 | ```bash 18 | sh scripts/portalsjs/portalsjs-test-generate-outputs.sh 19 | ``` 20 | 21 | To obfuscate, run: 22 | ```bash 23 | # install packages 24 | npm install javascript-obfuscator fs 25 | # run obfuscator script 26 | node portals-portalsjs/js/src/main/resources/obfuscator.js; 27 | ``` 28 | 29 | To run: 30 | Load the JS file, see https://github.com/portals-project/playground. 31 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/splitter.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("splitterTest"); 2 | let generator = builder.generators.fromRange(0, 128, 8); 3 | let splitter = builder.splitters.empty(generator.stream); 4 | let splitEven = builder.splits.split(splitter, x => x % 2 == 0); 5 | let splitOdd = builder.splits.split(splitter, x => x % 2 == 1); 6 | builder.workflows 7 | .source(splitEven) 8 | .logger("splitEven: ") 9 | .sink() 10 | .freeze(); 11 | builder.workflows 12 | .source(splitOdd) 13 | .logger("splitOdd: ") 14 | .sink() 15 | .freeze(); 16 | let splitterTest = builder.build(); 17 | let system = PortalsJS.System(); 18 | system.launch(splitterTest); 19 | system.stepUntilComplete(); 20 | system.shutdown(); 21 | -------------------------------------------------------------------------------- /scripts/deployment/kubernetes/Deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: portals-deployment 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: portals 10 | template: 11 | metadata: 12 | labels: 13 | app: portals 14 | spec: 15 | containers: 16 | - name: portals-server 17 | image: "portals" 18 | imagePullPolicy: Never 19 | ports: 20 | - containerPort: 8080 21 | 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | name: portals-service 27 | spec: 28 | type: NodePort 29 | selector: 30 | app: portals 31 | ports: 32 | - protocol: TCP 33 | port: 8080 34 | targetPort: 8080 35 | 36 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yaml: -------------------------------------------------------------------------------- 1 | # https://www.scala-sbt.org/1.x/docs/GitHub-Actions-with-sbt.html 2 | 3 | name: Build and test 4 | on: 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | branches: 10 | - main 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout 16 | uses: actions/checkout@v3 17 | - name: Setup JDK 18 | uses: actions/setup-java@v3 19 | with: 20 | distribution: temurin 21 | java-version: 17 22 | - name: Build and Test 23 | run: sbt -v test 24 | - name: Scalafmt scala 25 | run: sbt scalafmtCheckAll 26 | - name: Scalafmt sbt 27 | run: sbt scalafmtSbtCheck 28 | - name: Build docs 29 | run: sbt doc 30 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/util/JavaSerializer.scala: -------------------------------------------------------------------------------- 1 | package portals.util 2 | 3 | import java.io.ByteArrayInputStream 4 | import java.io.ByteArrayOutputStream 5 | import java.io.ObjectInputStream 6 | import java.io.ObjectOutputStream 7 | 8 | object JavaSerializer extends Serializer { 9 | // TODO: consider using `spores3` instead 10 | 11 | override def serialize[T](obj: T): Array[Byte] = 12 | val bytes = new ByteArrayOutputStream() 13 | val out = new ObjectOutputStream(bytes) 14 | out.writeObject(obj) 15 | out.close() 16 | bytes.toByteArray 17 | 18 | override def deserialize[T](bytes: Array[Byte]): T = 19 | val in = new ObjectInputStream(new ByteArrayInputStream(bytes)) 20 | in.readObject().asInstanceOf[T] 21 | } 22 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/compilers/PreCheckCompiler.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.compilers 2 | 3 | import portals.application.Application 4 | import portals.compiler.* 5 | import portals.compiler.phases.* 6 | import portals.compiler.physicalplan.* 7 | 8 | /** Compiler that performs all semantic checks before transforming the code to 9 | * its physical representation. 10 | */ 11 | private[portals] class PreCheckCompiler extends Compiler[Application, Application]: 12 | given ctx: CompilerContext = new CompilerContext() 13 | 14 | override def compile(application: Application): Application = 15 | CompilerPhase.empty 16 | .andThen(CompilerPhases.wellFormedCheck) 17 | .run(application) 18 | 19 | end PreCheckCompiler // class 20 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/compilers/PortalRewriteCompiler.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.compilers 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.application.Application 6 | import portals.compiler.* 7 | import portals.compiler.phases.* 8 | import portals.compiler.physicalplan.* 9 | 10 | /** Compiler that rewrites portals into sequencers and splitters. */ 11 | @experimental 12 | private[portals] class PortalRewriteCompiler extends Compiler[Application, Application]: 13 | given ctx: CompilerContext = new CompilerContext() 14 | 15 | override def compile(application: Application): Application = 16 | CompilerPhase.empty 17 | .andThen(CompilerPhases.rewritePortal) 18 | .run(application) 19 | 20 | end PortalRewriteCompiler // class 21 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/compilers/BaseCompiler.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.compilers 2 | 3 | import portals.application.Application 4 | import portals.compiler.* 5 | import portals.compiler.phases.* 6 | import portals.compiler.physicalplan.* 7 | 8 | /** Compiler that will run all checks and transform the code to its physical 9 | * representation. 10 | */ 11 | private[portals] class BaseCompiler extends Compiler[Application, PhysicalPlan[_]]: 12 | given ctx: CompilerContext = new CompilerContext() 13 | 14 | override def compile(application: Application): PhysicalPlan[_] = 15 | CompilerPhase.empty 16 | .andThen(CompilerPhases.wellFormedCheck) 17 | .andThen(CompilerPhases.codeGeneration) 18 | .run(application) 19 | 20 | end BaseCompiler // class 21 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/bankaccount/BankAccountData.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.bankaccount 2 | 3 | import scala.annotation.experimental 4 | import scala.util.Random 5 | 6 | object BankAccountData: 7 | import BankAccountConfig.* 8 | import BankAccountEvents.* 9 | 10 | val rand = new scala.util.Random 11 | 12 | private def op: AccountOperation = 13 | math.abs(rand.nextInt() % 2) match 14 | case 0 => Deposit(rand.nextInt(N_ACCOUNTS), rand.nextInt(100)) 15 | case 1 => Withdraw(rand.nextInt(N_ACCOUNTS), rand.nextInt(100)) 16 | 17 | def txnIter: Iterator[Iterator[Saga[AccountOperation]]] = 18 | Iterator 19 | .fill(N_EVENTS) { 20 | Saga(op, List.fill(N_OPS_PER_SAGA - 1)(op)) 21 | } 22 | .grouped(5) 23 | .map(_.iterator) 24 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/registry.js: -------------------------------------------------------------------------------- 1 | let builder1 = PortalsJS.ApplicationBuilder("app1"); 2 | let generator1 = builder1.generators.fromRange(0, 16, 1); 3 | let workflow1 = builder1.workflowsWithName("workflow1") 4 | .source(generator1.stream) 5 | .map(ctx => x => x * x) 6 | .sink() 7 | .freeze(); 8 | let app1 = builder1.build(); 9 | 10 | let builder2 = PortalsJS.ApplicationBuilder("app2"); 11 | let app1Stream = builder2.registry.streams.get("/app1/workflows/workflow1/stream"); 12 | let workflow2 = builder2.workflows 13 | .source(app1Stream) 14 | .map(ctx => x => x * x) 15 | .logger() 16 | .sink() 17 | .freeze(); 18 | let app2 = builder2.build(); 19 | 20 | let system = PortalsJS.System(); 21 | system.launch(app1); 22 | system.launch(app2); 23 | system.stepUntilComplete(); 24 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/portalStar.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("portalStar"); 2 | let portal = builder.portal.portal("portal"); 3 | builder.workflows 4 | .source(builder.generators.fromRange(0, 128, 8).stream) 5 | .replier(portal, ctx => x => { }, ctx => x => ctx.reply(-x)) 6 | .logger() 7 | .sink() 8 | .freeze(); 9 | builder.workflows 10 | .source(builder.generators.fromRange(0, 128, 8).stream) 11 | .asker(portal, ctx => x => { 12 | let f = ctx.ask(portal, x); 13 | ctx.await(f, c => { 14 | ctx.emit(f.value(ctx)); 15 | }); 16 | }) 17 | .logger() 18 | .sink() 19 | .freeze(); 20 | let portalStar = builder.build(); 21 | let system = PortalsJS.System(); 22 | system.launch(portalStar); 23 | system.stepUntilComplete(); 24 | system.shutdown(); 25 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Latest API Docs 2 | on: 3 | # push: 4 | # branches: 5 | # - main 6 | schedule: 7 | - cron: '0 4 * * *' # Every day at 4 AM 8 | 9 | jobs: 10 | docs-versioned: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v3 15 | - name: Setup JDK 16 | uses: actions/setup-java@v3 17 | with: 18 | distribution: temurin 19 | java-version: 17 20 | - name: Generate documentation 21 | run: sbt doc 22 | - name: Deploy 23 | uses: peaceiris/actions-gh-pages@v3 24 | with: 25 | deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} 26 | external_repository: portals-project/api 27 | publish_branch: main 28 | publish_dir: './portals-core/jvm/target/api' 29 | keep_files: false 30 | -------------------------------------------------------------------------------- /.github/workflows/docs-versioned.yml: -------------------------------------------------------------------------------- 1 | name: Versioned API Docs 2 | on: 3 | push: 4 | tags: 5 | - 'v[0-9]+.[0-9]+.[0-9]+' 6 | 7 | jobs: 8 | docs-versioned: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v3 13 | - name: Setup JDK 14 | uses: actions/setup-java@v3 15 | with: 16 | distribution: temurin 17 | java-version: 17 18 | - name: Generate documentation 19 | run: sbt doc 20 | - name: Deploy 21 | uses: peaceiris/actions-gh-pages@v3 22 | with: 23 | deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} 24 | external_repository: portals-project/api-versioned 25 | publish_branch: main 26 | publish_dir: './portals-core/jvm/target/api' 27 | destination_dir: ${{ github.ref_name }} 28 | keep_files: true 29 | -------------------------------------------------------------------------------- /portals-libraries/src/test/scala/portals/libraries/actor/ExamplesTest.scala: -------------------------------------------------------------------------------- 1 | package portals.libraries.actor 2 | 3 | import scala.annotation.experimental 4 | 5 | import org.junit.runner.RunWith 6 | import org.junit.runners.JUnit4 7 | import org.junit.Assert._ 8 | import org.junit.Test 9 | 10 | /** Tests that all examples compile and run, does not test that the output is 11 | * correct. 12 | */ 13 | @experimental 14 | @RunWith(classOf[JUnit4]) 15 | class ExamplesTest: 16 | 17 | @Test 18 | def testFibonacci(): Unit = 19 | import portals.libraries.actor.examples.FibonacciMain 20 | FibonacciMain 21 | 22 | @Test 23 | def testPingPong(): Unit = 24 | import portals.libraries.actor.examples.PingPongMain 25 | PingPongMain 26 | 27 | @Test 28 | def testCountingActor(): Unit = 29 | import portals.libraries.actor.examples.CountingActorMain 30 | CountingActorMain 31 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/tasks/AggregatingTask.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart.tasks 2 | 3 | import portals.api.dsl.DSL.* 4 | import portals.application.task.* 5 | import portals.examples.shoppingcart.ShoppingCartConfig 6 | import portals.examples.shoppingcart.ShoppingCartEvents.* 7 | 8 | object AggregatingTask: 9 | type I = Item 10 | type O = (Item, Count) 11 | type Req = Nothing 12 | type Res = Nothing 13 | type Context = MapTaskContext[I, O] 14 | type Task = GenericTask[I, O, Nothing, Nothing] 15 | 16 | private final val state: PerKeyState[Int] = 17 | PerKeyState[Int]("state", 0) 18 | 19 | private def onNext(item: Item)(using Context): (Item, Count) = 20 | val count = state.get() 21 | val newCount = count + 1 22 | state.set(newCount) 23 | (item, newCount) 24 | 25 | def apply(): Task = 26 | Tasks.map(onNext) 27 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/phases/portal/RewritePortalPhase.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.phases.portal 2 | 3 | import portals.application.* 4 | import portals.application.task.* 5 | import portals.compiler.* 6 | import portals.compiler.physicalplan.* 7 | import portals.util.Common.Types.* 8 | 9 | private[portals] object RewritePortalPhase extends CompilerPhase[Application, Application]: 10 | import RewritePortalHelpers.* 11 | 12 | override def run(application: Application)(using ctx: CompilerContext) = 13 | CompilerSubPhase.empty 14 | // NOTE: RewritePortalsSubPhase must be run before 15 | // RewriteWorkflowsSubPhase. This is due to the subsequent phase erasing 16 | // the portal dependency information in the workflows. 17 | .andThen(RewritePortalsSubPhase) 18 | .andThen(RewriteWorkflowsSubPhase) 19 | .run(application) 20 | 21 | end RewritePortalPhase // object 22 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/sequencer/Sequencer.scala: -------------------------------------------------------------------------------- 1 | package portals.application.sequencer 2 | 3 | import scala.util.Random 4 | 5 | /** Sequencer. */ 6 | trait Sequencer[T] extends Serializable: 7 | /** Sequencing strategy. 8 | * 9 | * @param paths 10 | * paths (with ready atoms) to be sequenced 11 | * @return 12 | * Next stream to be consumed in the produced sequence, can also be None 13 | */ 14 | def sequence(paths: String*): Option[String] 15 | end Sequencer 16 | 17 | private[portals] object SequencerImpls: 18 | case class RandomSequencer[T]() extends Sequencer[T]: 19 | def sequence(paths: String*): Option[String] = 20 | if paths.isEmpty then None 21 | else Some(paths(Random.nextInt(paths.size))) 22 | end RandomSequencer 23 | 24 | end SequencerImpls 25 | 26 | object Sequencers: 27 | import SequencerImpls.* 28 | 29 | def random[T](): Sequencer[T] = RandomSequencer[T]() 30 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/CompilerBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.application.* 6 | import portals.compiler.compilers.* 7 | import portals.compiler.physicalplan.* 8 | 9 | /** Factory for creating compilers. */ 10 | private[portals] object CompilerBuilder: 11 | /** Compiler that will run all semantic checks before the code 12 | * transformations. 13 | */ 14 | def preCompiler(): Compiler[Application, Application] = PreCheckCompiler() 15 | 16 | /** Compiler that will rewrite portals into sequencers and splitters. */ 17 | @experimental 18 | def portalRewriteCompiler(): Compiler[Application, Application] = PortalRewriteCompiler() 19 | 20 | /** Create a compiler that will run all checks and transform the code to its 21 | * physical representation. 22 | */ 23 | def compiler(): Compiler[Application, PhysicalPlan[_]] = BaseCompiler() 24 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/tasks/OrdersTask.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart.tasks 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.api.dsl.DSL.* 6 | import portals.application.task.* 7 | import portals.application.AtomicPortalRef 8 | import portals.application.AtomicPortalRefKind 9 | import portals.examples.shoppingcart.ShoppingCartConfig 10 | import portals.examples.shoppingcart.ShoppingCartEvents.* 11 | 12 | object OrdersTask: 13 | type I = OrderOps 14 | type O = OrderOps 15 | type Req = Nothing 16 | type Rep = Nothing 17 | type Context = MapTaskContext[OrderOps, OrderOps] 18 | type Task = GenericTask[OrderOps, OrderOps, Nothing, Nothing] 19 | 20 | def onNext(event: OrderOps)(using Context): OrderOps = 21 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Order for $event created") 22 | event 23 | 24 | def apply(): Task = 25 | Tasks.map(onNext) 26 | end OrdersTask // class 27 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/physicalplan/PhysicalPlan.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.physicalplan 2 | 3 | import portals.application.Application 4 | 5 | /** Physical plan representation of an application. 6 | * 7 | * @tparam CONFIG 8 | * The configuration-type of the physical plan. 9 | */ 10 | private[portals] sealed trait PhysicalPlan[CONFIG <: PlanConfig] { 11 | 12 | /** Returns the configuration of this physical plan. 13 | * 14 | * @return 15 | * the configuration of this physical plan. 16 | */ 17 | def config: CONFIG 18 | } 19 | 20 | /** Plan representation of an application. 21 | * 22 | * @tparam CONFIG 23 | * The configuration-type of the physical plan. 24 | * @param app 25 | * The application that this plan represents. 26 | * @param config 27 | * The configuration of this plan. 28 | */ 29 | private[portals] case class Plan[CONFIG <: PlanConfig](app: Application, config: CONFIG) extends PhysicalPlan[CONFIG] 30 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/taskState.js.out: -------------------------------------------------------------------------------- 1 | 1 2 | 1 3 | 1 4 | 1 5 | 1 6 | 1 7 | 1 8 | 1 9 | 1 10 | 1 11 | 1 12 | 1 13 | 1 14 | 1 15 | 1 16 | 1 17 | 2 18 | 2 19 | 2 20 | 2 21 | 2 22 | 2 23 | 2 24 | 2 25 | 2 26 | 2 27 | 2 28 | 2 29 | 2 30 | 2 31 | 2 32 | 2 33 | 3 34 | 3 35 | 3 36 | 3 37 | 3 38 | 3 39 | 3 40 | 3 41 | 3 42 | 3 43 | 3 44 | 3 45 | 3 46 | 3 47 | 3 48 | 3 49 | 4 50 | 4 51 | 4 52 | 4 53 | 4 54 | 4 55 | 4 56 | 4 57 | 4 58 | 4 59 | 4 60 | 4 61 | 4 62 | 4 63 | 4 64 | 4 65 | 5 66 | 5 67 | 5 68 | 5 69 | 5 70 | 5 71 | 5 72 | 5 73 | 5 74 | 5 75 | 5 76 | 5 77 | 5 78 | 5 79 | 5 80 | 5 81 | 6 82 | 6 83 | 6 84 | 6 85 | 6 86 | 6 87 | 6 88 | 6 89 | 6 90 | 6 91 | 6 92 | 6 93 | 6 94 | 6 95 | 6 96 | 6 97 | 7 98 | 7 99 | 7 100 | 7 101 | 7 102 | 7 103 | 7 104 | 7 105 | 7 106 | 7 107 | 7 108 | 7 109 | 7 110 | 7 111 | 7 112 | 7 113 | 8 114 | 8 115 | 8 116 | 8 117 | 8 118 | 8 119 | 8 120 | 8 121 | 8 122 | 8 123 | 8 124 | 8 125 | 8 126 | 8 127 | 8 128 | 8 129 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/generator.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("generatorTest"); 2 | let generator1 = builder.generators.fromRange(0, 128, 8); 3 | let generator2 = builder.generators.fromArray([1, 2, 3, 4, 5]); 4 | let generator3 = builder.generators.fromArrayOfArrays([[1, 2, 3], [4, 5, 6]]); 5 | let generator4 = builder.generators.signal("asdf"); 6 | let generator5 = builder.generators.empty(); 7 | let sequencer = builder.sequencers.random(); 8 | builder.connections.connect(generator1.stream, sequencer); 9 | builder.connections.connect(generator2.stream, sequencer); 10 | builder.connections.connect(generator3.stream, sequencer); 11 | builder.connections.connect(generator4.stream, sequencer); 12 | builder.connections.connect(generator5.stream, sequencer); 13 | builder.workflows 14 | .source(sequencer.stream) 15 | .logger() 16 | .sink() 17 | .freeze(); 18 | let generatorTest = builder.build(); 19 | let system = PortalsJS.System(); 20 | system.launch(generatorTest); 21 | system.stepUntilComplete(); 22 | system.shutdown(); 23 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/multiDataflow.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("multiDataflow") 2 | 3 | let sequencer = builder.sequencers.random() 4 | 5 | let workflow1 = builder.workflows 6 | .source(sequencer.stream) 7 | .flatMap(ctx => x => { if (x <= 0) { return []; } else { return [x - 1]; } }) 8 | .logger("workflow1: ") 9 | .sink() 10 | .freeze() 11 | 12 | let generator = builder.generators.signal(8) 13 | builder.connections.connect(generator.stream, sequencer) 14 | builder.connections.connect(workflow1.stream, sequencer) 15 | 16 | let workflow2 = builder.workflows 17 | .source(workflow1.stream) 18 | .map(ctx => x => { return x * x; }) 19 | .logger("workflow2: ") 20 | .sink() 21 | .freeze() 22 | 23 | let workflow3 = builder.workflows 24 | .source(workflow1.stream) 25 | .map(ctx => x => { return x * x * x; }) 26 | .logger("workflow3: ") 27 | .sink() 28 | .freeze() 29 | 30 | let multiDataflow = builder.build() 31 | let system = PortalsJS.System() 32 | system.launch(multiDataflow) 33 | system.stepUntilComplete() 34 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/CompilerSubPhase.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler 2 | 3 | /** Sub-phase for the compiler, not a subclass of compiler phase by design. 4 | * Cannot be used as a normal compiler phase. 5 | */ 6 | private[portals] trait CompilerSubPhase[T, U] { 7 | self => 8 | def run(t: T)(using ctx: CompilerContext): U 9 | def andThen[V](next: CompilerSubPhase[U, V]): CompilerSubPhase[T, V] = 10 | new CompilerSubPhase[T, V] { 11 | override def run(t: T)(using ctx: CompilerContext): V = 12 | next.run { self.run(t) } 13 | } 14 | } 15 | 16 | object CompilerSubPhase: 17 | /** Empty compiler phase, can be used to start a chain of compiler phases. */ 18 | def empty[T]: CompilerSubPhase[T, T] = new CompilerSubPhase[T, T] { 19 | override def run(t: T)(using ctx: CompilerContext): T = t 20 | } 21 | 22 | /** Compiler phase that maps with the provided function `f`. */ 23 | def map[T, U](f: T => U): CompilerSubPhase[T, U] = new CompilerSubPhase[T, U] { 24 | override def run(t: T)(using ctx: CompilerContext): U = f(t) 25 | } 26 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/phases/RuntimeCompilerPhases.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.phases 2 | 3 | import portals.application.Application 4 | import portals.runtime.interpreter.trackers.ApplicationTracker 5 | 6 | // phases to be executed at runtime 7 | private[portals] object RuntimeCompilerPhases: 8 | /** Check if the application is well-formed with respect to the dynamic 9 | * runtime. Throws exception. 10 | */ 11 | def wellFormedCheck(application: Application)(using rctx: ApplicationTracker): Unit = 12 | // 1. Check naming collision with other applications 13 | if rctx.applications.contains(application.path) then ??? 14 | // this test should suffice, as all other paths will be a subpath of the application 15 | 16 | // 2. Check that all external references exist 17 | if !application.externalPortals.forall(x => rctx.portals.contains(x.path)) then ??? 18 | if !application.externalSequencers.forall(x => rctx.sequencers.contains(x.path)) then ??? 19 | if !application.externalStreams.forall(x => rctx.streams.contains(x.path)) then ??? 20 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/task/TaskStateImpl.scala: -------------------------------------------------------------------------------- 1 | package portals.application.task 2 | 3 | import portals.application.task.TaskState 4 | import portals.runtime.state.MapStateBackendImpl 5 | import portals.runtime.state.StateBackend 6 | import portals.util.Key 7 | 8 | private[portals] class TaskStateImpl extends TaskState[Any, Any]: 9 | private[portals] var stateBackend: StateBackend = new MapStateBackendImpl() 10 | 11 | override def get(k: Any): Option[Any] = stateBackend.get(keyBuilder(k)).asInstanceOf[Option[Any]] 12 | 13 | override def set(k: Any, v: Any): Unit = stateBackend.set(keyBuilder(k), v) 14 | 15 | override def del(k: Any): Unit = stateBackend.del(keyBuilder(k)) 16 | 17 | override def clear(): Unit = stateBackend.clear() 18 | 19 | override def iterator: Iterator[(Any, Any)] = stateBackend.iterator.asInstanceOf[Iterator[(Any, Any)]] 20 | 21 | private[portals] var path: String = _ 22 | 23 | private[portals] var key: Key = _ 24 | 25 | private def keyBuilder(k: Any): (String, Any) = (path + "$" + key.x.toString(), k) 26 | end TaskStateImpl // class 27 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/pingPong.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("pingPong") 2 | 3 | let portal = builder.portal.portal("pingPongPortal") 4 | 5 | let generator = builder.generators.signal(8) 6 | 7 | function pingerPonger(ctx, x) { 8 | if (x > 0) { 9 | ctx.log.info(x); 10 | let future = ctx.ask(portal, x - 1); 11 | ctx.await(future, ctx => { 12 | let v = future.value(ctx); 13 | ctx.reply(pingerPonger(ctx, v)); 14 | }); 15 | } 16 | else { 17 | ctx.reply(x); 18 | } 19 | } 20 | 21 | let pingPongWorkflow = builder.workflows 22 | .source(generator.stream) 23 | .askerreplier( 24 | portal, 25 | portal, 26 | ctx => x => { 27 | let future = ctx.ask(portal, x); 28 | ctx.await(future, ctx => { 29 | ctx.log.info("Done!"); 30 | }); 31 | }, 32 | ctx => x => { 33 | pingerPonger(ctx, x) 34 | }, 35 | ) 36 | .logger() 37 | .sink() 38 | .freeze() 39 | 40 | let pingPong = builder.build() 41 | let system = PortalsJS.System() 42 | system.launch(pingPong) 43 | system.stepUntilComplete() 44 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/wordCount.js: -------------------------------------------------------------------------------- 1 | // hashcode function to compute key of a string 2 | let hashCode = str => { 3 | var hash = 0; 4 | for (var i = 0; i < str.length; i++) { 5 | let c = str.charCodeAt(i); 6 | hash = ((hash << 5) - hash) + c; 7 | hash = hash & hash; 8 | return hash; 9 | } 10 | } 11 | 12 | let builder = PortalsJS.ApplicationBuilder("application"); 13 | 14 | let input = ["the quick brown fox jumps over the lazy dog"]; 15 | let generator = builder.generators.fromArray(input); 16 | 17 | let _ = builder.workflows 18 | .source(generator.stream) 19 | .flatMap(ctx => line => line.split(/\s+/)) 20 | .map(ctx => word => [word, 1]) 21 | .key(pair => { return hashCode(pair[0]); }) 22 | .processor(ctx => x => { 23 | let count = PortalsJS.PerKeyState("count", 0, ctx); 24 | count.set(count.get() + x[1]); 25 | ctx.emit([x[0], count.get()]); 26 | }) 27 | .logger() 28 | .sink() 29 | .freeze(); 30 | 31 | let application = builder.build(); 32 | let system = PortalsJS.System(); 33 | system.launch(application); 34 | system.stepUntilComplete(); 35 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/processors/InterpreterStream.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.processors 2 | 3 | import scala.collection.mutable.ArrayDeque 4 | 5 | import portals.application.AtomicStream 6 | import portals.runtime.BatchedEvents.* 7 | 8 | private[portals] class InterpreterStream(stream: AtomicStream[_]): 9 | private var atomQueue = ArrayDeque.empty[EventBatch] 10 | private var index: Long = -1 11 | 12 | /** Enqueue an atom to the atomic stream. */ 13 | def enqueue(ta: EventBatch): Unit = 14 | atomQueue = atomQueue.append(ta) 15 | 16 | /** Read from the output atomic stream at the index idx. */ 17 | def read(idx: Long): EventBatch = 18 | atomQueue((idx - index).toInt) // trust me :_) 19 | 20 | /** Returns the range of indexes that can be read. The range is inclusive, 21 | * i.e. [0, 0] means idx 0 can be read. 22 | */ 23 | def getIdxRange(): (Long, Long) = (Math.max(0, index), index + atomQueue.size) 24 | 25 | /** Prune all atoms up to index idx. */ 26 | def prune(idx: Long): Unit = 27 | atomQueue = atomQueue.drop((idx - index).toInt) 28 | index = idx 29 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/key.js.out: -------------------------------------------------------------------------------- 1 | 1 2 | 1 3 | 2 4 | 2 5 | 3 6 | 3 7 | 4 8 | 4 9 | 5 10 | 5 11 | 6 12 | 6 13 | 7 14 | 7 15 | 8 16 | 8 17 | 9 18 | 9 19 | 10 20 | 10 21 | 11 22 | 11 23 | 12 24 | 12 25 | 13 26 | 13 27 | 14 28 | 14 29 | 15 30 | 15 31 | 16 32 | 16 33 | 17 34 | 17 35 | 18 36 | 18 37 | 19 38 | 19 39 | 20 40 | 20 41 | 21 42 | 21 43 | 22 44 | 22 45 | 23 46 | 23 47 | 24 48 | 24 49 | 25 50 | 25 51 | 26 52 | 26 53 | 27 54 | 27 55 | 28 56 | 28 57 | 29 58 | 29 59 | 30 60 | 30 61 | 31 62 | 31 63 | 32 64 | 32 65 | 33 66 | 33 67 | 34 68 | 34 69 | 35 70 | 35 71 | 36 72 | 36 73 | 37 74 | 37 75 | 38 76 | 38 77 | 39 78 | 39 79 | 40 80 | 40 81 | 41 82 | 41 83 | 42 84 | 42 85 | 43 86 | 43 87 | 44 88 | 44 89 | 45 90 | 45 91 | 46 92 | 46 93 | 47 94 | 47 95 | 48 96 | 48 97 | 49 98 | 49 99 | 50 100 | 50 101 | 51 102 | 51 103 | 52 104 | 52 105 | 53 106 | 53 107 | 54 108 | 54 109 | 55 110 | 55 111 | 56 112 | 56 113 | 57 114 | 57 115 | 58 116 | 58 117 | 59 118 | 59 119 | 60 120 | 60 121 | 61 122 | 61 123 | 62 124 | 62 125 | 63 126 | 63 127 | 64 128 | 64 129 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/processors/InterpreterGenerator.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.processors 2 | 3 | import portals.application.AtomicGenerator 4 | import portals.runtime.BatchedEvents.* 5 | import portals.runtime.WrappedEvents.* 6 | // import portals.Key 7 | 8 | /** Internal API. TestGenerator. */ 9 | private[portals] class InterpreterGenerator(val generator: AtomicGenerator[_]) extends GeneratingStepper: 10 | override def step(): List[EventBatch] = 11 | var atom = List.empty[WrappedEvent[_]] 12 | var stop = false 13 | while generator.generator.hasNext() && !stop do 14 | generator.generator.generate() match 15 | case event @ Event(key, t) => 16 | atom = Event(key, t) :: atom 17 | case Atom => 18 | atom = Atom :: atom 19 | stop = true 20 | case Seal => 21 | atom = Seal :: atom 22 | stop = true 23 | case Error(t) => 24 | atom = Error(t) :: atom 25 | stop = true 26 | case Ask(_, _, _) => 27 | ??? 28 | case Reply(_, _, _) => 29 | ??? 30 | List(AtomBatch(generator.stream.path, atom.reverse)) 31 | -------------------------------------------------------------------------------- /portals-core/jvm/src/main/scala/portals/util/Logger.scala: -------------------------------------------------------------------------------- 1 | package portals.util 2 | 3 | import org.slf4j.{Logger => SLLogger} 4 | import org.slf4j.{LoggerFactory => SLLoggerFactory} 5 | 6 | import ch.qos.logback.classic.{Level => LBLevel} 7 | import ch.qos.logback.classic.{Logger => LBLogger} 8 | import ch.qos.logback.classic.{LoggerContext => LBLoggerContext} 9 | 10 | type Logger = SLLogger 11 | 12 | object Logger: 13 | setLevel("INFO") // set log level to INFO 14 | 15 | def apply(name: String): Logger = 16 | SLLoggerFactory.getLogger(name) 17 | 18 | def setLevel(level: String): Unit = 19 | val loggerContext: LBLoggerContext = SLLoggerFactory.getILoggerFactory().asInstanceOf[LBLoggerContext] 20 | val rootLogger: ch.qos.logback.classic.Logger = loggerContext.getLogger(SLLogger.ROOT_LOGGER_NAME) 21 | level match 22 | case "DEBUG" => rootLogger.setLevel(LBLevel.DEBUG) 23 | case "INFO" => rootLogger.setLevel(LBLevel.INFO) 24 | case "WARN" => rootLogger.setLevel(LBLevel.WARN) 25 | case "ERROR" => rootLogger.setLevel(LBLevel.ERROR) 26 | case "OFF" => rootLogger.setLevel(LBLevel.OFF) 27 | case _ => throw new IllegalArgumentException(s"Unknown log level: $level") 28 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/tutorial/EventFiltering.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.tutorial 2 | 3 | import portals.api.builder.ApplicationBuilder 4 | import portals.application.Application 5 | import portals.system.Systems 6 | 7 | /** Event Filtering 8 | * 9 | * This example creates a workflow that filters out events containing the word 10 | * "interesting". And only logs those events. 11 | */ 12 | object EventFiltering: 13 | import portals.api.dsl.DSL.* 14 | 15 | val app: Application = { 16 | val builder = ApplicationBuilder("app") 17 | 18 | val messages = List("Hello, World!", "This Event seems interesting", "Goodbye!") 19 | 20 | val generator = builder.generators.fromList(messages) 21 | 22 | val _ = builder 23 | .workflows[String, String]("workflow") 24 | .source(generator.stream) 25 | .filter { _.contains("interesting") } 26 | .logger() 27 | .sink() 28 | .freeze() 29 | 30 | builder.build() 31 | } 32 | 33 | @main def EventFilteringMain(): Unit = 34 | val system = Systems.test() 35 | system.launch(EventFiltering.app) 36 | system.stepUntilComplete() 37 | system.shutdown() 38 | println("EventFilteringMain: done") 39 | -------------------------------------------------------------------------------- /portals-core/js/src/main/scala/portals/util/Logger.scala: -------------------------------------------------------------------------------- 1 | package portals.util 2 | 3 | import scala.scalajs.js 4 | import scala.scalajs.js.annotation.JSExport 5 | 6 | trait Logger: 7 | val name: String 8 | def log = this 9 | @JSExport 10 | def info(msg: Any): Unit = 11 | val _msg = Logger._level + name + " - " + msg.toString() 12 | println(Logger.getTimestamp + " INFO " + _msg) 13 | 14 | object Logger: 15 | setLevel("INFO") // set log level to INFO 16 | var _level = "" 17 | 18 | def apply(_name: String): Logger = 19 | new Logger { val name = _name } 20 | 21 | def setLevel(level: String): Unit = 22 | _level = level 23 | 24 | private def formatNumber(number: Int, digits: Int): String = 25 | val padded = "0" * (digits - number.toString.length) + number.toString 26 | padded.substring(0, digits) 27 | 28 | private def getTimestamp: String = 29 | val date = new js.Date() 30 | val hours = formatNumber(date.getHours().toInt, 2) 31 | val minutes = formatNumber(date.getMinutes().toInt, 2) 32 | val seconds = formatNumber(date.getSeconds().toInt, 2) 33 | val milliseconds = formatNumber(date.getMilliseconds().toInt, 3) 34 | s"$hours:$minutes:$seconds.$milliseconds" 35 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/identity.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 2 4 | 3 5 | 4 6 | 5 7 | 6 8 | 7 9 | 8 10 | 9 11 | 10 12 | 11 13 | 12 14 | 13 15 | 14 16 | 15 17 | 16 18 | 17 19 | 18 20 | 19 21 | 20 22 | 21 23 | 22 24 | 23 25 | 24 26 | 25 27 | 26 28 | 27 29 | 28 30 | 29 31 | 30 32 | 31 33 | 32 34 | 33 35 | 34 36 | 35 37 | 36 38 | 37 39 | 38 40 | 39 41 | 40 42 | 41 43 | 42 44 | 43 45 | 44 46 | 45 47 | 46 48 | 47 49 | 48 50 | 49 51 | 50 52 | 51 53 | 52 54 | 53 55 | 54 56 | 55 57 | 56 58 | 57 59 | 58 60 | 59 61 | 60 62 | 61 63 | 62 64 | 63 65 | 64 66 | 65 67 | 66 68 | 67 69 | 68 70 | 69 71 | 70 72 | 71 73 | 72 74 | 73 75 | 74 76 | 75 77 | 76 78 | 77 79 | 78 80 | 79 81 | 80 82 | 81 83 | 82 84 | 83 85 | 84 86 | 85 87 | 86 88 | 87 89 | 88 90 | 89 91 | 90 92 | 91 93 | 92 94 | 93 95 | 94 96 | 95 97 | 96 98 | 97 99 | 98 100 | 99 101 | 100 102 | 101 103 | 102 104 | 103 105 | 104 106 | 105 107 | 106 108 | 107 109 | 108 110 | 109 111 | 110 112 | 111 113 | 112 114 | 113 115 | 114 116 | 115 117 | 116 118 | 117 119 | 118 120 | 119 121 | 120 122 | 121 123 | 122 124 | 123 125 | 124 126 | 125 127 | 126 128 | 127 129 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/task/OutputCollector.scala: -------------------------------------------------------------------------------- 1 | package portals.application.task 2 | 3 | import portals.runtime.WrappedEvents.* 4 | import portals.util.Key 5 | 6 | trait OutputCollector[T, U, Req, Rep]: 7 | ////////////////////////////////////////////////////////////////////////////// 8 | // Task 9 | ////////////////////////////////////////////////////////////////////////////// 10 | def submit(event: WrappedEvent[U]): Unit 11 | 12 | ////////////////////////////////////////////////////////////////////////////// 13 | // Asker Task 14 | ////////////////////////////////////////////////////////////////////////////// 15 | // TODO: the param names are confusing, make it clear what they are heres 16 | def ask(portal: String, asker: String, req: Req, key: Key, id: Int, askingWF: String): Unit 17 | 18 | ////////////////////////////////////////////////////////////////////////////// 19 | // Replier Task 20 | ////////////////////////////////////////////////////////////////////////////// 21 | // TODO: the param names are confusing, make it clear what they are heres 22 | def reply(r: Rep, portal: String, asker: String, key: Key, id: Int, askingWF: String): Unit 23 | end OutputCollector // trait 24 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/portalService.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("portalService") 2 | 3 | let portal = builder.portal.portal("portal", x => { return 0; }) 4 | 5 | let generator = builder.generators.empty() 6 | 7 | let replierWorkflow = builder.workflows 8 | .source(generator.stream) 9 | .replier( 10 | portal, 11 | ctx => x => { }, 12 | ctx => x => { 13 | let state = PortalsJS.PerKeyState("prev", 0, ctx); 14 | let prev = state.get(); 15 | ctx.reply(prev); 16 | state.set(x); 17 | }, 18 | ) 19 | .sink() 20 | .freeze() 21 | 22 | let trigger = builder.generators.fromRange(0, 128, 8) 23 | 24 | let requesterWorkflow = builder.workflows 25 | .source(trigger.stream) 26 | .asker( 27 | portal, 28 | ctx => x => { 29 | let future = ctx.ask(portal, x); 30 | ctx.await(future, ctx => { 31 | let msg = JSON.stringify({ "x": x, "future": future.value(ctx) }); 32 | ctx.emit(msg) 33 | }); 34 | }, 35 | ) 36 | .logger("requesterWorkflow: ") 37 | .sink() 38 | .freeze() 39 | 40 | let portalService = builder.build() 41 | let system = PortalsJS.System() 42 | system.launch(portalService) 43 | system.stepUntilComplete() 44 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/taskTest.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 0 3 | 2 4 | -2 5 | 4 6 | -4 7 | 6 8 | -6 9 | 8 10 | -8 11 | 10 12 | -10 13 | 12 14 | -12 15 | 14 16 | -14 17 | 16 18 | -16 19 | 18 20 | -18 21 | 20 22 | -20 23 | 22 24 | -22 25 | 24 26 | -24 27 | 26 28 | -26 29 | 28 30 | -28 31 | 30 32 | -30 33 | 32 34 | -32 35 | 34 36 | -34 37 | 36 38 | -36 39 | 38 40 | -38 41 | 40 42 | -40 43 | 42 44 | -42 45 | 44 46 | -44 47 | 46 48 | -46 49 | 48 50 | -48 51 | 50 52 | -50 53 | 52 54 | -52 55 | 54 56 | -54 57 | 56 58 | -56 59 | 58 60 | -58 61 | 60 62 | -60 63 | 62 64 | -62 65 | 64 66 | -64 67 | 66 68 | -66 69 | 68 70 | -68 71 | 70 72 | -70 73 | 72 74 | -72 75 | 74 76 | -74 77 | 76 78 | -76 79 | 78 80 | -78 81 | 80 82 | -80 83 | 82 84 | -82 85 | 84 86 | -84 87 | 86 88 | -86 89 | 88 90 | -88 91 | 90 92 | -90 93 | 92 94 | -92 95 | 94 96 | -94 97 | 96 98 | -96 99 | 98 100 | -98 101 | 100 102 | -100 103 | 102 104 | -102 105 | 104 106 | -104 107 | 106 108 | -106 109 | 108 110 | -108 111 | 110 112 | -110 113 | 112 114 | -112 115 | 114 116 | -114 117 | 116 118 | -116 119 | 118 120 | -118 121 | 120 122 | -120 123 | 122 124 | -122 125 | 124 126 | -124 127 | 126 128 | -126 129 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/processor.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 2 3 | 4 4 | 6 5 | 8 6 | 10 7 | 12 8 | 14 9 | 16 10 | 18 11 | 20 12 | 22 13 | 24 14 | 26 15 | 28 16 | 30 17 | 32 18 | 34 19 | 36 20 | 38 21 | 40 22 | 42 23 | 44 24 | 46 25 | 48 26 | 50 27 | 52 28 | 54 29 | 56 30 | 58 31 | 60 32 | 62 33 | 64 34 | 66 35 | 68 36 | 70 37 | 72 38 | 74 39 | 76 40 | 78 41 | 80 42 | 82 43 | 84 44 | 86 45 | 88 46 | 90 47 | 92 48 | 94 49 | 96 50 | 98 51 | 100 52 | 102 53 | 104 54 | 106 55 | 108 56 | 110 57 | 112 58 | 114 59 | 116 60 | 118 61 | 120 62 | 122 63 | 124 64 | 126 65 | 128 66 | 130 67 | 132 68 | 134 69 | 136 70 | 138 71 | 140 72 | 142 73 | 144 74 | 146 75 | 148 76 | 150 77 | 152 78 | 154 79 | 156 80 | 158 81 | 160 82 | 162 83 | 164 84 | 166 85 | 168 86 | 170 87 | 172 88 | 174 89 | 176 90 | 178 91 | 180 92 | 182 93 | 184 94 | 186 95 | 188 96 | 190 97 | 192 98 | 194 99 | 196 100 | 198 101 | 200 102 | 202 103 | 204 104 | 206 105 | 208 106 | 210 107 | 212 108 | 214 109 | 216 110 | 218 111 | 220 112 | 222 113 | 224 114 | 226 115 | 228 116 | 230 117 | 232 118 | 234 119 | 236 120 | 238 121 | 240 122 | 242 123 | 244 124 | 246 125 | 248 126 | 250 127 | 252 128 | 254 129 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/tutorial/HelloWorld.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.tutorial 2 | 3 | import portals.api.builder.ApplicationBuilder 4 | import portals.application.Application 5 | import portals.system.Systems 6 | 7 | /** Hello World 8 | * 9 | * This example creates a workflow that prints all the ingested events to the 10 | * logger. We submit the event containing the message "Hello, World!" and 11 | * expect it to be printed. 12 | */ 13 | object HelloWorld: 14 | import portals.api.dsl.DSL.* 15 | 16 | val app: Application = { 17 | val builder = ApplicationBuilder("app") 18 | 19 | val message = "Hello, World!" 20 | val generator = builder.generators.fromList(List(message)) 21 | 22 | val _ = builder 23 | .workflows[String, String]("hello") 24 | .source(generator.stream) 25 | .map { x => x } 26 | .logger() 27 | .sink() 28 | .freeze() 29 | 30 | val application = builder.build() 31 | 32 | application 33 | } 34 | 35 | object HelloWorldMain extends App: 36 | val application = HelloWorld.app 37 | // ASTPrinter.println(application) // print the application AST 38 | val system = Systems.test() 39 | system.launch(application) 40 | system.stepUntilComplete() 41 | system.shutdown() 42 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/WorkflowBuilderContext.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | import portals.application.task.GenericTask 5 | 6 | class WorkflowBuilderContext[T, U](_path: String)(using val bctx: ApplicationBuilderContext): 7 | self => 8 | val path: String = _path 9 | 10 | var tasks: Map[String, GenericTask[_, _, _, _]] = Map.empty 11 | var source: Option[String] = None 12 | var sink: Option[String] = None 13 | var connections: List[(String, String)] = List.empty 14 | 15 | private val _stream: AtomicStream[U] = AtomicStream[U](path + "/" + "stream") 16 | 17 | val streamref: AtomicStreamRef[U] = AtomicStreamRef[U](_stream) 18 | 19 | var consumes: AtomicStreamRefKind[T] = null 20 | 21 | private var frozen: Boolean = false 22 | 23 | def freeze(): Workflow[T, U] = 24 | if frozen == true then ??? 25 | else 26 | frozen = true 27 | val wf = new Workflow[T, U]( 28 | path = path, 29 | consumes = consumes, 30 | stream = streamref, 31 | tasks = self.tasks, 32 | source = source.get, 33 | sink = sink.get, 34 | connections = connections 35 | ) 36 | bctx.addToContext(wf) 37 | bctx.addToContext(_stream) 38 | wf 39 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/examples/shoppingcart/Orders.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed.examples.shoppingcart 2 | 3 | import portals.api.dsl.DSL 4 | import portals.api.dsl.DSL.* 5 | import portals.api.dsl.DSL.PortalsApp 6 | import portals.api.dsl.ExperimentalDSL.* 7 | import portals.application.Application 8 | import portals.distributed.SubmittableApplication 9 | import portals.examples.shoppingcart.tasks.* 10 | import portals.examples.shoppingcart.ShoppingCartData 11 | import portals.examples.shoppingcart.ShoppingCartEvents.* 12 | 13 | /** Orders for the Shopping Cart example. 14 | * 15 | * @see 16 | * for more information on how to run this example: 17 | * [[portals.distributed.examples.shoppingcart.ShoppingCart]] 18 | * 19 | * @see 20 | * [[portals.examples.shoppingcart.ShoppingCart]] 21 | */ 22 | object Orders extends SubmittableApplication: 23 | override def apply(): Application = 24 | PortalsApp("Orders"): 25 | val cartStream = Registry.streams.get[OrderOps]("/Cart/workflows/cart/stream") 26 | 27 | val orders = Workflows[OrderOps, OrderOps]("orders") 28 | .source(cartStream) 29 | .key(keyFrom(_)) 30 | .task(OrdersTask()) 31 | .withName("orders") 32 | .sink() 33 | .freeze() 34 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/trackers/GraphTracker.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.trackers 2 | 3 | import scala.util.Random 4 | 5 | import portals.application.* 6 | import portals.application.task.AskerReplierTask 7 | import portals.application.task.AskerTask 8 | import portals.application.task.ReplierTask 9 | import portals.compiler.phases.RuntimeCompilerPhases 10 | import portals.runtime.BatchedEvents.* 11 | import portals.runtime.PortalsRuntime 12 | import portals.runtime.WrappedEvents.* 13 | 14 | /** Internal API. Tracks the graph which is spanned by all applications in 15 | * Portals. 16 | */ 17 | private[portals] class GraphTracker: 18 | /** Set of all pairs edges. */ 19 | private var _edges: Set[(String, String)] = 20 | Set.empty 21 | 22 | /** Add an edge to the graph. */ 23 | def addEdge(from: String, to: String): Unit = 24 | _edges += (from, to) 25 | 26 | /** Get all incoming edges to a graph node with the name 'path'. */ 27 | def getInputs(path: String): Option[Set[String]] = 28 | Some(_edges.filter(_._2 == path).map(_._1)) 29 | 30 | /** Get all outgoing edges to a graph node with the name 'path'. */ 31 | def getOutputs(path: String): Option[Set[String]] = 32 | Some(_edges.filter(_._1 == path).map(_._2)) 33 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/SteppingRuntime.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime 2 | 3 | import portals.application.Application 4 | 5 | trait SteppingRuntime extends PortalsRuntime: 6 | /** Returns true if the runtime can take another step. */ 7 | def canStep(): Boolean 8 | 9 | /** Take a step. 10 | * 11 | * A **step** will randomly choose one of the processing units (Workflows, 12 | * Sequencers, etc.) with valid input. It will take one atom from the input 13 | * and process it to completion. 14 | * 15 | * Throws an exception if it cannot take a step. 16 | */ 17 | def step(): Unit 18 | 19 | /** Take steps until cannot take more steps. */ 20 | def stepUntilComplete(): Unit = 21 | while canStep() do step() 22 | 23 | /** Take steps until either cannot take more steps or has reached the max. */ 24 | def stepUntilComplete(max: Int): Unit = 25 | var i = 0 26 | while canStep() && i < max do 27 | step() 28 | i += 1 29 | 30 | /** Take steps for `millis` milliseconds and then stop. */ 31 | def stepFor(millis: Long): Unit = 32 | val start = System.currentTimeMillis() 33 | while System.currentTimeMillis() - start < millis do 34 | if canStep() 35 | then step() 36 | else Thread.sleep(100) 37 | 38 | end SteppingRuntime 39 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/stateful.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 2 4 | 3 5 | 4 6 | 6 7 | 8 8 | 10 9 | 12 10 | 15 11 | 18 12 | 21 13 | 24 14 | 28 15 | 32 16 | 36 17 | 40 18 | 45 19 | 50 20 | 55 21 | 60 22 | 66 23 | 72 24 | 78 25 | 84 26 | 91 27 | 98 28 | 105 29 | 112 30 | 120 31 | 128 32 | 136 33 | 144 34 | 153 35 | 162 36 | 171 37 | 180 38 | 190 39 | 200 40 | 210 41 | 220 42 | 231 43 | 242 44 | 253 45 | 264 46 | 276 47 | 288 48 | 300 49 | 312 50 | 325 51 | 338 52 | 351 53 | 364 54 | 378 55 | 392 56 | 406 57 | 420 58 | 435 59 | 450 60 | 465 61 | 480 62 | 496 63 | 512 64 | 528 65 | 544 66 | 561 67 | 578 68 | 595 69 | 612 70 | 630 71 | 648 72 | 666 73 | 684 74 | 703 75 | 722 76 | 741 77 | 760 78 | 780 79 | 800 80 | 820 81 | 840 82 | 861 83 | 882 84 | 903 85 | 924 86 | 946 87 | 968 88 | 990 89 | 1012 90 | 1035 91 | 1058 92 | 1081 93 | 1104 94 | 1128 95 | 1152 96 | 1176 97 | 1200 98 | 1225 99 | 1250 100 | 1275 101 | 1300 102 | 1326 103 | 1352 104 | 1378 105 | 1404 106 | 1431 107 | 1458 108 | 1485 109 | 1512 110 | 1540 111 | 1568 112 | 1596 113 | 1624 114 | 1653 115 | 1682 116 | 1711 117 | 1740 118 | 1770 119 | 1800 120 | 1830 121 | 1860 122 | 1891 123 | 1922 124 | 1953 125 | 1984 126 | 2016 127 | 2048 128 | 2080 129 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/portalStar.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | -1 3 | -2 4 | -3 5 | -4 6 | -5 7 | -6 8 | -7 9 | -8 10 | -9 11 | -10 12 | -11 13 | -12 14 | -13 15 | -14 16 | -15 17 | -16 18 | -17 19 | -18 20 | -19 21 | -20 22 | -21 23 | -22 24 | -23 25 | -24 26 | -25 27 | -26 28 | -27 29 | -28 30 | -29 31 | -30 32 | -31 33 | -32 34 | -33 35 | -34 36 | -35 37 | -36 38 | -37 39 | -38 40 | -39 41 | -40 42 | -41 43 | -42 44 | -43 45 | -44 46 | -45 47 | -46 48 | -47 49 | -48 50 | -49 51 | -50 52 | -51 53 | -52 54 | -53 55 | -54 56 | -55 57 | -56 58 | -57 59 | -58 60 | -59 61 | -60 62 | -61 63 | -62 64 | -63 65 | -64 66 | -65 67 | -66 68 | -67 69 | -68 70 | -69 71 | -70 72 | -71 73 | -72 74 | -73 75 | -74 76 | -75 77 | -76 78 | -77 79 | -78 80 | -79 81 | -80 82 | -81 83 | -82 84 | -83 85 | -84 86 | -85 87 | -86 88 | -87 89 | -88 90 | -89 91 | -90 92 | -91 93 | -92 94 | -93 95 | -94 96 | -95 97 | -96 98 | -97 99 | -98 100 | -99 101 | -100 102 | -101 103 | -102 104 | -103 105 | -104 106 | -105 107 | -106 108 | -107 109 | -108 110 | -109 111 | -110 112 | -111 113 | -112 114 | -113 115 | -114 116 | -115 117 | -116 118 | -117 119 | -118 120 | -119 121 | -120 122 | -121 123 | -122 124 | -123 125 | -124 126 | -125 127 | -126 128 | -127 129 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/portalTask.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | -1 3 | -2 4 | -3 5 | -4 6 | -5 7 | -6 8 | -7 9 | -8 10 | -9 11 | -10 12 | -11 13 | -12 14 | -13 15 | -14 16 | -15 17 | -16 18 | -17 19 | -18 20 | -19 21 | -20 22 | -21 23 | -22 24 | -23 25 | -24 26 | -25 27 | -26 28 | -27 29 | -28 30 | -29 31 | -30 32 | -31 33 | -32 34 | -33 35 | -34 36 | -35 37 | -36 38 | -37 39 | -38 40 | -39 41 | -40 42 | -41 43 | -42 44 | -43 45 | -44 46 | -45 47 | -46 48 | -47 49 | -48 50 | -49 51 | -50 52 | -51 53 | -52 54 | -53 55 | -54 56 | -55 57 | -56 58 | -57 59 | -58 60 | -59 61 | -60 62 | -61 63 | -62 64 | -63 65 | -64 66 | -65 67 | -66 68 | -67 69 | -68 70 | -69 71 | -70 72 | -71 73 | -72 74 | -73 75 | -74 76 | -75 77 | -76 78 | -77 79 | -78 80 | -79 81 | -80 82 | -81 83 | -82 84 | -83 85 | -84 86 | -85 87 | -86 88 | -87 89 | -88 90 | -89 91 | -90 92 | -91 93 | -92 94 | -93 95 | -94 96 | -95 97 | -96 98 | -97 99 | -98 100 | -99 101 | -100 102 | -101 103 | -102 104 | -103 105 | -104 106 | -105 107 | -106 108 | -107 109 | -108 110 | -109 111 | -110 112 | -111 113 | -112 114 | -113 115 | -114 116 | -115 117 | -116 118 | -117 119 | -118 120 | -119 121 | -120 122 | -121 123 | -122 124 | -123 125 | -124 126 | -125 127 | -126 128 | -127 129 | -------------------------------------------------------------------------------- /portals-benchmark/src/main/scala/portals/benchmark/BenchmarkConfig.scala: -------------------------------------------------------------------------------- 1 | package portals.benchmark 2 | 3 | class BenchmarkConfig: 4 | private var config = Map.empty[String, String] 5 | private var required = Set.empty[String] 6 | 7 | def args: List[String] = config.flatMap { case (k, v) => List(k.toString(), v.toString()) }.toList 8 | 9 | // check if required config parameters are set 10 | private def checkRequired(): Boolean = 11 | required.forall(config.contains) 12 | 13 | // internal recursive method 14 | private def _parseArgs(args: List[String]): Unit = args match 15 | case c :: value :: tail => 16 | config += (c -> value) 17 | _parseArgs(tail) 18 | case Nil => () 19 | case _ => ??? 20 | 21 | def parseArgs(args: List[String]): this.type = 22 | _parseArgs(args) 23 | if !checkRequired() then 24 | val missing = required.diff(config.keys.toSet) 25 | throw new IllegalArgumentException("Missing required arguments for BenchmarkConfig: " + missing) 26 | this 27 | 28 | def set(key: String, value: Any): this.type = 29 | config += (key -> value.toString) 30 | this 31 | 32 | def setRequired(key: String): this.type = 33 | required += key 34 | this 35 | 36 | def get(key: String): String = config(key) 37 | 38 | def getInt(key: String): Int = config(key).toInt 39 | -------------------------------------------------------------------------------- /portals-libraries/src/test/scala/portals/libraries/actor/ActorTest.scala: -------------------------------------------------------------------------------- 1 | package portals.libraries.actor 2 | 3 | import scala.annotation.experimental 4 | 5 | import org.junit.runner.RunWith 6 | import org.junit.runners.JUnit4 7 | import org.junit.Assert._ 8 | import org.junit.Test 9 | 10 | import portals.libraries.actor.examples.* 11 | import portals.system.Systems 12 | 13 | @RunWith(classOf[JUnit4]) 14 | class ActorTest: 15 | 16 | @Test 17 | @experimental 18 | def testFibonacci(): Unit = 19 | import portals.api.dsl.DSL.* 20 | 21 | import ActorEvents.ActorMessage 22 | import ActorEvents.ActorCreate 23 | import Fibonacci.FibReply 24 | import Fibonacci.FibActors.initBehavior 25 | 26 | val FIB_N = 21 27 | val testBehavior = ActorTestUtils.TestBehavior(initBehavior(FIB_N)) 28 | 29 | val config = ActorConfig.default 30 | .replace("logging", true) 31 | 32 | val app = PortalsApp("Fibonacci") { 33 | val generator = Generators.signal[ActorMessage](ActorCreate(ActorRef.fresh(), testBehavior.behavior)) 34 | val wf = ActorWorkflow(generator.stream, config) 35 | } 36 | 37 | val system = Systems.test() 38 | system.launch(app) 39 | system.stepUntilComplete() 40 | system.shutdown() 41 | 42 | testBehavior 43 | .receiveAssert(FibReply(10946)) 44 | .isEmptyAssert() 45 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/init.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 3 4 | 6 5 | 10 6 | 15 7 | 21 8 | 28 9 | 36 10 | 45 11 | 55 12 | 66 13 | 78 14 | 91 15 | 105 16 | 120 17 | 136 18 | 153 19 | 171 20 | 190 21 | 210 22 | 231 23 | 253 24 | 276 25 | 300 26 | 325 27 | 351 28 | 378 29 | 406 30 | 435 31 | 465 32 | 496 33 | 528 34 | 561 35 | 595 36 | 630 37 | 666 38 | 703 39 | 741 40 | 780 41 | 820 42 | 861 43 | 903 44 | 946 45 | 990 46 | 1035 47 | 1081 48 | 1128 49 | 1176 50 | 1225 51 | 1275 52 | 1326 53 | 1378 54 | 1431 55 | 1485 56 | 1540 57 | 1596 58 | 1653 59 | 1711 60 | 1770 61 | 1830 62 | 1891 63 | 1953 64 | 2016 65 | 2080 66 | 2145 67 | 2211 68 | 2278 69 | 2346 70 | 2415 71 | 2485 72 | 2556 73 | 2628 74 | 2701 75 | 2775 76 | 2850 77 | 2926 78 | 3003 79 | 3081 80 | 3160 81 | 3240 82 | 3321 83 | 3403 84 | 3486 85 | 3570 86 | 3655 87 | 3741 88 | 3828 89 | 3916 90 | 4005 91 | 4095 92 | 4186 93 | 4278 94 | 4371 95 | 4465 96 | 4560 97 | 4656 98 | 4753 99 | 4851 100 | 4950 101 | 5050 102 | 5151 103 | 5253 104 | 5356 105 | 5460 106 | 5565 107 | 5671 108 | 5778 109 | 5886 110 | 5995 111 | 6105 112 | 6216 113 | 6328 114 | 6441 115 | 6555 116 | 6670 117 | 6786 118 | 6903 119 | 7021 120 | 7140 121 | 7260 122 | 7381 123 | 7503 124 | 7626 125 | 7750 126 | 7875 127 | 8001 128 | 8128 129 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/examples/shoppingcart/Cart.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed.examples.shoppingcart 2 | 3 | import portals.api.dsl.DSL 4 | import portals.api.dsl.DSL.* 5 | import portals.api.dsl.DSL.PortalsApp 6 | import portals.api.dsl.ExperimentalDSL.* 7 | import portals.application.Application 8 | import portals.distributed.SubmittableApplication 9 | import portals.examples.shoppingcart.tasks.* 10 | import portals.examples.shoppingcart.ShoppingCartData 11 | import portals.examples.shoppingcart.ShoppingCartEvents.* 12 | 13 | /** Cart for the Shopping Cart example. 14 | * 15 | * @see 16 | * for more information on how to run this example: 17 | * [[portals.distributed.examples.shoppingcart.ShoppingCart]] 18 | * 19 | * @see 20 | * [[portals.examples.shoppingcart.ShoppingCart]] 21 | */ 22 | object Cart extends SubmittableApplication: 23 | override def apply(): Application = 24 | PortalsApp("Cart"): 25 | val cartOpsGenerator = Generators.generator(ShoppingCartData.cartOpsGenerator) 26 | val portal = Registry.portals.get[InventoryReqs, InventoryReps]("/Inventory/portals/inventory") 27 | 28 | val cart = Workflows[CartOps, OrderOps]("cart") 29 | .source(cartOpsGenerator.stream) 30 | .key(keyFrom(_)) 31 | .task(CartTask(portal)) 32 | .withName("cart") 33 | .sink() 34 | .freeze() 35 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/ShoppingCartData.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart 2 | 3 | import scala.annotation.experimental 4 | import scala.util.Random 5 | 6 | import portals.application.generator.Generators 7 | 8 | object ShoppingCartData: 9 | import ShoppingCartConfig.* 10 | import ShoppingCartEvents.* 11 | 12 | val rand = new scala.util.Random 13 | 14 | private def cartOpsIter: Iterator[Iterator[CartOps]] = 15 | Iterator 16 | // -- info: to make the iterator infinite, use `Iterator.continually` instead -- 17 | .fill(N_EVENTS)(rand.nextInt(3) match 18 | case 0 => AddToCart(rand.nextInt(N_USERS), rand.nextInt(N_ITEMS)) 19 | case 1 => RemoveFromCart(rand.nextInt(N_USERS), rand.nextInt(N_ITEMS)) 20 | case 2 => Checkout(rand.nextInt(N_USERS)) 21 | ) 22 | .grouped(5) 23 | .map(_.iterator) 24 | 25 | def cartOpsGenerator = 26 | ThrottledGenerator( 27 | Generators.fromIteratorOfIterators(cartOpsIter), 28 | 128, 29 | ) 30 | 31 | private def inventoryOpsIter: Iterator[Iterator[InventoryReqs]] = 32 | Iterator 33 | .fill(N_EVENTS)(Put(rand.nextInt(N_ITEMS))) 34 | .grouped(5) 35 | .map(_.iterator) 36 | 37 | def inventoryOpsGenerator = 38 | ThrottledGenerator( 39 | Generators.fromIteratorOfIterators(inventoryOpsIter), 40 | 128, 41 | ) 42 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/map.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 4 4 | 9 5 | 16 6 | 25 7 | 36 8 | 49 9 | 64 10 | 81 11 | 100 12 | 121 13 | 144 14 | 169 15 | 196 16 | 225 17 | 256 18 | 289 19 | 324 20 | 361 21 | 400 22 | 441 23 | 484 24 | 529 25 | 576 26 | 625 27 | 676 28 | 729 29 | 784 30 | 841 31 | 900 32 | 961 33 | 1024 34 | 1089 35 | 1156 36 | 1225 37 | 1296 38 | 1369 39 | 1444 40 | 1521 41 | 1600 42 | 1681 43 | 1764 44 | 1849 45 | 1936 46 | 2025 47 | 2116 48 | 2209 49 | 2304 50 | 2401 51 | 2500 52 | 2601 53 | 2704 54 | 2809 55 | 2916 56 | 3025 57 | 3136 58 | 3249 59 | 3364 60 | 3481 61 | 3600 62 | 3721 63 | 3844 64 | 3969 65 | 4096 66 | 4225 67 | 4356 68 | 4489 69 | 4624 70 | 4761 71 | 4900 72 | 5041 73 | 5184 74 | 5329 75 | 5476 76 | 5625 77 | 5776 78 | 5929 79 | 6084 80 | 6241 81 | 6400 82 | 6561 83 | 6724 84 | 6889 85 | 7056 86 | 7225 87 | 7396 88 | 7569 89 | 7744 90 | 7921 91 | 8100 92 | 8281 93 | 8464 94 | 8649 95 | 8836 96 | 9025 97 | 9216 98 | 9409 99 | 9604 100 | 9801 101 | 10000 102 | 10201 103 | 10404 104 | 10609 105 | 10816 106 | 11025 107 | 11236 108 | 11449 109 | 11664 110 | 11881 111 | 12100 112 | 12321 113 | 12544 114 | 12769 115 | 12996 116 | 13225 117 | 13456 118 | 13689 119 | 13924 120 | 14161 121 | 14400 122 | 14641 123 | 14884 124 | 15129 125 | 15376 126 | 15625 127 | 15876 128 | 16129 129 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/task.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 1 3 | 4 4 | 9 5 | 16 6 | 25 7 | 36 8 | 49 9 | 64 10 | 81 11 | 100 12 | 121 13 | 144 14 | 169 15 | 196 16 | 225 17 | 256 18 | 289 19 | 324 20 | 361 21 | 400 22 | 441 23 | 484 24 | 529 25 | 576 26 | 625 27 | 676 28 | 729 29 | 784 30 | 841 31 | 900 32 | 961 33 | 1024 34 | 1089 35 | 1156 36 | 1225 37 | 1296 38 | 1369 39 | 1444 40 | 1521 41 | 1600 42 | 1681 43 | 1764 44 | 1849 45 | 1936 46 | 2025 47 | 2116 48 | 2209 49 | 2304 50 | 2401 51 | 2500 52 | 2601 53 | 2704 54 | 2809 55 | 2916 56 | 3025 57 | 3136 58 | 3249 59 | 3364 60 | 3481 61 | 3600 62 | 3721 63 | 3844 64 | 3969 65 | 4096 66 | 4225 67 | 4356 68 | 4489 69 | 4624 70 | 4761 71 | 4900 72 | 5041 73 | 5184 74 | 5329 75 | 5476 76 | 5625 77 | 5776 78 | 5929 79 | 6084 80 | 6241 81 | 6400 82 | 6561 83 | 6724 84 | 6889 85 | 7056 86 | 7225 87 | 7396 88 | 7569 89 | 7744 90 | 7921 91 | 8100 92 | 8281 93 | 8464 94 | 8649 95 | 8836 96 | 9025 97 | 9216 98 | 9409 99 | 9604 100 | 9801 101 | 10000 102 | 10201 103 | 10404 104 | 10609 105 | 10816 106 | 11025 107 | 11236 108 | 11449 109 | 11664 110 | 11881 111 | 12100 112 | 12321 113 | 12544 114 | 12769 115 | 12996 116 | 13225 117 | 13456 118 | 13689 119 | 13924 120 | 14161 121 | 14400 122 | 14641 123 | 14884 124 | 15129 125 | 15376 126 | 15625 127 | 15876 128 | 16129 129 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/WrappedEvents.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime 2 | 3 | import portals.util.Key 4 | 5 | // Warning: if WrappedEvents is private[portals] then the tests will fail, not sure why 6 | object WrappedEvents: 7 | private[portals] sealed trait WrappedEvent[+T] 8 | private[portals] case class Event[T](key: Key, event: T) extends WrappedEvent[T] 9 | private[portals] case class Error[T](t: Throwable) extends WrappedEvent[T] 10 | private[portals] case object Atom extends WrappedEvent[Nothing] 11 | private[portals] case object Seal extends WrappedEvent[Nothing] 12 | 13 | /** internal API */ 14 | private[portals] case class PortalMeta( 15 | portal: String, 16 | askingTask: String, 17 | askingKey: Key, 18 | id: Int, // request id 19 | askingWF: String, 20 | ) 21 | 22 | private[portals] case class Ask[T]( 23 | key: Key, 24 | meta: PortalMeta, 25 | event: T 26 | ) extends WrappedEvent[T] 27 | 28 | /** internal API */ 29 | private[portals] case class Reply[T]( 30 | key: Key, 31 | meta: PortalMeta, 32 | event: T 33 | ) extends WrappedEvent[T] 34 | 35 | extension (event: WrappedEvent[_]) 36 | def key: Key = event match 37 | case Event(key, _) => key 38 | case Ask(key, _, _) => key 39 | case Reply(key, _, _) => key 40 | case Error(_) => Key(-1) 41 | case Atom => Key(-1) 42 | case Seal => Key(-1) 43 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/examples/shoppingcart/Inventory.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed.examples.shoppingcart 2 | 3 | import portals.api.dsl.DSL 4 | import portals.api.dsl.DSL.* 5 | import portals.api.dsl.DSL.PortalsApp 6 | import portals.api.dsl.ExperimentalDSL.* 7 | import portals.application.Application 8 | import portals.distributed.SubmittableApplication 9 | import portals.examples.shoppingcart.tasks.* 10 | import portals.examples.shoppingcart.ShoppingCartData 11 | import portals.examples.shoppingcart.ShoppingCartEvents.* 12 | 13 | /** Inventory for the Shopping Cart example. 14 | * 15 | * @see 16 | * for more information on how to run this example: 17 | * [[portals.distributed.examples.shoppingcart.ShoppingCart]] 18 | * 19 | * @see 20 | * [[portals.examples.shoppingcart.ShoppingCart]] 21 | */ 22 | object Inventory extends SubmittableApplication: 23 | override def apply(): Application = 24 | PortalsApp("Inventory"): 25 | val inventoryOpsGenerator = Generators.generator(ShoppingCartData.inventoryOpsGenerator) 26 | val portal = Portal[InventoryReqs, InventoryReps]("inventory", keyFrom) 27 | val inventory = Workflows[InventoryReqs, Nothing]("inventory") 28 | .source(inventoryOpsGenerator.stream) 29 | .key(keyFrom(_)) 30 | .logger() 31 | .task(InventoryTask(portal)) 32 | .withName("inventory") 33 | .sink() 34 | .freeze() 35 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/map.js.out: -------------------------------------------------------------------------------- 1 | 0,1 2 | 1,1 3 | 2,1 4 | 3,1 5 | 4,1 6 | 5,1 7 | 6,1 8 | 7,1 9 | 8,1 10 | 9,1 11 | 10,2 12 | 11,2 13 | 12,2 14 | 13,2 15 | 14,2 16 | 15,2 17 | 16,2 18 | 17,2 19 | 18,2 20 | 19,2 21 | 20,2 22 | 21,2 23 | 22,2 24 | 23,2 25 | 24,2 26 | 25,2 27 | 26,2 28 | 27,2 29 | 28,2 30 | 29,2 31 | 30,2 32 | 31,2 33 | 32,2 34 | 33,2 35 | 34,2 36 | 35,2 37 | 36,2 38 | 37,2 39 | 38,2 40 | 39,2 41 | 40,2 42 | 41,2 43 | 42,2 44 | 43,2 45 | 44,2 46 | 45,2 47 | 46,2 48 | 47,2 49 | 48,2 50 | 49,2 51 | 50,2 52 | 51,2 53 | 52,2 54 | 53,2 55 | 54,2 56 | 55,2 57 | 56,2 58 | 57,2 59 | 58,2 60 | 59,2 61 | 60,2 62 | 61,2 63 | 62,2 64 | 63,2 65 | 64,2 66 | 65,2 67 | 66,2 68 | 67,2 69 | 68,2 70 | 69,2 71 | 70,2 72 | 71,2 73 | 72,2 74 | 73,2 75 | 74,2 76 | 75,2 77 | 76,2 78 | 77,2 79 | 78,2 80 | 79,2 81 | 80,2 82 | 81,2 83 | 82,2 84 | 83,2 85 | 84,2 86 | 85,2 87 | 86,2 88 | 87,2 89 | 88,2 90 | 89,2 91 | 90,2 92 | 91,2 93 | 92,2 94 | 93,2 95 | 94,2 96 | 95,2 97 | 96,2 98 | 97,2 99 | 98,2 100 | 99,2 101 | 100,3 102 | 101,3 103 | 102,3 104 | 103,3 105 | 104,3 106 | 105,3 107 | 106,3 108 | 107,3 109 | 108,3 110 | 109,3 111 | 110,3 112 | 111,3 113 | 112,3 114 | 113,3 115 | 114,3 116 | 115,3 117 | 116,3 118 | 117,3 119 | 118,3 120 | 119,3 121 | 120,3 122 | 121,3 123 | 122,3 124 | 123,3 125 | 124,3 126 | 125,3 127 | 126,3 128 | 127,3 129 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/system/TestSystem.scala: -------------------------------------------------------------------------------- 1 | package portals.system 2 | 3 | import portals.application.Application 4 | import portals.runtime.test.TestRuntime 5 | import portals.system.PortalsSystem 6 | 7 | /** Test system and runtime for Portals. This system is single-threaded, 8 | * synchronous, and lets the user proceed the computation by taking steps over 9 | * atoms. Alternatively, the computation can be carried out until the end by 10 | * stepping until it has completed. 11 | */ 12 | class TestSystem(seed: Option[Int] = None) extends PortalsSystem: 13 | private val runtime: TestRuntime = TestRuntime(seed) 14 | 15 | /** Launch a Portals application. */ 16 | def launch(application: Application): Unit = runtime.launch(application) 17 | 18 | /** Take a step over an atom. */ 19 | def step(): Unit = runtime.step() 20 | 21 | /** Check if the system can take a step. */ 22 | def canStep(): Boolean = runtime.canStep() 23 | 24 | /** Take steps until completion. */ 25 | def stepUntilComplete(): Unit = runtime.stepUntilComplete() 26 | 27 | /** Take steps until completion or until reaching the max number of steps. */ 28 | def stepUntilComplete(max: Int): Unit = runtime.stepUntilComplete(max) 29 | 30 | /** Take steps for `millis` milliseconds and then stop. */ 31 | def stepFor(millis: Long): Unit = runtime.stepFor(millis) 32 | 33 | /** Terminate the system and cleanup. */ 34 | def shutdown(): Unit = runtime.shutdown() 35 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/withStar.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | -1 3 | -2 4 | -3 5 | -4 6 | -5 7 | -6 8 | -7 9 | 1 10 | -8 11 | -9 12 | -10 13 | -11 14 | -12 15 | -13 16 | -14 17 | -15 18 | 1 19 | -16 20 | -17 21 | -18 22 | -19 23 | -20 24 | -21 25 | -22 26 | -23 27 | 1 28 | -24 29 | -25 30 | -26 31 | -27 32 | -28 33 | -29 34 | -30 35 | -31 36 | 1 37 | -32 38 | -33 39 | -34 40 | -35 41 | -36 42 | -37 43 | -38 44 | -39 45 | 1 46 | -40 47 | -41 48 | -42 49 | -43 50 | -44 51 | -45 52 | -46 53 | -47 54 | 1 55 | -48 56 | -49 57 | -50 58 | -51 59 | -52 60 | -53 61 | -54 62 | -55 63 | 1 64 | -56 65 | -57 66 | -58 67 | -59 68 | -60 69 | -61 70 | -62 71 | -63 72 | 1 73 | -64 74 | -65 75 | -66 76 | -67 77 | -68 78 | -69 79 | -70 80 | -71 81 | 1 82 | -72 83 | -73 84 | -74 85 | -75 86 | -76 87 | -77 88 | -78 89 | -79 90 | 1 91 | -80 92 | -81 93 | -82 94 | -83 95 | -84 96 | -85 97 | -86 98 | -87 99 | 1 100 | -88 101 | -89 102 | -90 103 | -91 104 | -92 105 | -93 106 | -94 107 | -95 108 | 1 109 | -96 110 | -97 111 | -98 112 | -99 113 | -100 114 | -101 115 | -102 116 | -103 117 | 1 118 | -104 119 | -105 120 | -106 121 | -107 122 | -108 123 | -109 124 | -110 125 | -111 126 | 1 127 | -112 128 | -113 129 | -114 130 | -115 131 | -116 132 | -117 133 | -118 134 | -119 135 | 1 136 | -120 137 | -121 138 | -122 139 | -123 140 | -124 141 | -125 142 | -126 143 | -127 144 | 1 145 | 2 146 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/examples/shoppingcart/Analytics.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed.examples.shoppingcart 2 | 3 | import portals.api.dsl.DSL 4 | import portals.api.dsl.DSL.* 5 | import portals.api.dsl.ExperimentalDSL.* 6 | import portals.application.Application 7 | import portals.examples.shoppingcart.tasks.* 8 | import portals.examples.shoppingcart.ShoppingCartEvents.* 9 | 10 | /** Analytics for the Shopping Cart example. 11 | * 12 | * @see 13 | * for more information on how to run this example: 14 | * [[portals.distributed.examples.shoppingcart.ShoppingCart]] 15 | * 16 | * @see 17 | * [[portals.examples.shoppingcart.ShoppingCart]] 18 | */ 19 | object Analytics extends portals.distributed.SubmittableApplication: 20 | override def apply(): portals.application.Application = 21 | portals.api.dsl.DSL.PortalsApp("Analytics"): 22 | val ordersStream = Registry.streams.get[OrderOps]("/Orders/workflows/orders/stream") 23 | 24 | val analyticsportal = Portal[AnalyticsReqs, AnalyticsReps]("analytics", keyFrom) 25 | 26 | val analytics = Workflows[OrderOps, Nothing]("analytics") 27 | .source(ordersStream) 28 | .flatMap { case Order(_, CartState(items)) => items } 29 | .key(keyFrom(_)) 30 | .task(AggregatingTask()) 31 | .key(_ => 0L) 32 | .task(AnalyticsTask(analyticsportal)) 33 | .logger() 34 | .nothing() 35 | .sink() 36 | .freeze() 37 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/task/TaskState.scala: -------------------------------------------------------------------------------- 1 | package portals.application.task 2 | 3 | import portals.application.task.TaskStateImpl 4 | import portals.util.Key 5 | 6 | private[portals] trait TaskState[K, V]: 7 | /** get the value of the key, scoped by the dynamic invocation context */ 8 | def get(k: K): Option[V] 9 | 10 | /** set the key to the value, scoped by the dynamic invocation context */ 11 | def set(k: K, v: V): Unit 12 | 13 | /** delete the key, scoped by the dynamic invocation context */ 14 | def del(k: K): Unit 15 | 16 | /** iterate over all key-value pairs, **NOT** scoped by the dynamic context 17 | */ 18 | def iterator: Iterator[(K, V)] 19 | 20 | /** clear the state of the current instance, **NOT** scoped by the dynamic 21 | * context 22 | */ 23 | def clear(): Unit 24 | 25 | ////////////////////////////////////////////////////////////////////////////// 26 | // Execution Context 27 | ////////////////////////////////////////////////////////////////////////////// 28 | 29 | /** Path of the task */ 30 | // has to be var so that it can be swapped at runtime 31 | private[portals] var path: String 32 | 33 | /** Contextual key for per-key execution */ 34 | // has to be var so that it can be swapped at runtime 35 | private[portals] var key: Key 36 | 37 | end TaskState // trait 38 | 39 | private[portals] object TaskState: 40 | def apply(): TaskState[Any, Any] = 41 | new TaskStateImpl 42 | 43 | end TaskState // object 44 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/playground/portalAggregation.js: -------------------------------------------------------------------------------- 1 | let builder = PortalsJS.ApplicationBuilder("portalAggregation") 2 | 3 | let portal = builder.portal.portal("portal", x => { return 0; }) 4 | 5 | let generator = builder.generators.fromRange(0, 128, 8) 6 | 7 | let aggregationWorkflow = builder.workflows 8 | .source(generator.stream) 9 | .key(x => { return 0; }) 10 | .replier( 11 | portal, 12 | ctx => x => { 13 | // aggregate state from generator input 14 | let state = PortalsJS.PerKeyState("state", 0, ctx); 15 | state.set(x + state.get()); 16 | ctx.emit(state.get()); 17 | }, 18 | ctx => x => { 19 | // reply to portal requests with the aggregated state 20 | let state = PortalsJS.PerKeyState("state", 0, ctx); 21 | ctx.reply(state.get()); 22 | }, 23 | ) 24 | .sink() 25 | .freeze() 26 | 27 | let trigger = builder.generators.fromArrayOfArrays([[0], [0], [0], [0], [0]]) 28 | 29 | let requestingWorkflow = builder.workflows 30 | .source(trigger.stream) 31 | .asker( 32 | portal, 33 | ctx => x => { 34 | // query the portal for the aggregated state 35 | let future = ctx.ask(portal, x); 36 | ctx.await(future, ctx => { ctx.emit(future.value(ctx)); }); 37 | }, 38 | ) 39 | .logger("requestingWorkflow: ") 40 | .sink() 41 | .freeze() 42 | 43 | let portalAggregation = builder.build() 44 | let system = PortalsJS.System() 45 | system.launch(portalAggregation) 46 | system.stepUntilComplete() 47 | -------------------------------------------------------------------------------- /portals-benchmark/src/main/scala/portals/benchmark/BenchmarkGrid.scala: -------------------------------------------------------------------------------- 1 | package portals.benchmark 2 | 3 | /** the benchmark grid will span all combinations of the entries of the 4 | * parameter lists 5 | */ 6 | class BenchmarkGrid: 7 | // TODO: consider adding option to add a List of Lists of params, so that we can 8 | // have some form of nested grouping, allowing for grouping specific param combinations together. 9 | // e.g. see https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.ParameterGrid.html 10 | 11 | private type ParamKeyValue = (String, String) // key value 12 | 13 | private var grid: List[List[ParamKeyValue]] = List.empty 14 | 15 | private def crossProduct(l: List[List[ParamKeyValue]]): Iterator[List[ParamKeyValue]] = 16 | l match 17 | case x :: xs => x.iterator.flatMap(i => crossProduct(xs).map(i :: _)) 18 | case Nil => Iterator.single(List.empty[ParamKeyValue]) 19 | 20 | def set(paramList: List[ParamKeyValue]): this.type = 21 | grid ::= paramList 22 | this 23 | 24 | def set(key: String, values: Any*): this.type = 25 | val paramList = values.map { x => (key, x.toString()) }.toList 26 | set(paramList) 27 | 28 | def setParam(key: String, value: Any): this.type = 29 | set(List((key, value.toString()))) 30 | 31 | def iterator: Iterator[List[ParamKeyValue]] = 32 | crossProduct(grid.reverse) 33 | 34 | def configs: Iterator[BenchmarkConfig] = 35 | this.iterator.map { params => BenchmarkConfig().parseArgs(params.flatMap(List(_, _))) } 36 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/phases/portal/RewritePortalEvents.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.phases.portal 2 | 3 | import portals.runtime.WrappedEvents 4 | import portals.util.Key 5 | 6 | /** Events used for the portal rewrite. */ 7 | private[portals] object RewritePortalEvents: 8 | ////////////////////////////////////////////////////////////////////////////// 9 | // Rewrite Portal Events 10 | ////////////////////////////////////////////////////////////////////////////// 11 | 12 | /** Event type used to wrap system events, and used as a normal user event. */ 13 | private[portals] case class RewriteEvent[T](we: WrappedEvents.WrappedEvent[T]) 14 | 15 | ////////////////////////////////////////////////////////////////////////////// 16 | // Mapping from WrappedEvents to RewritePortalEvents 17 | ////////////////////////////////////////////////////////////////////////////// 18 | 19 | extension (we: WrappedEvents.WrappedEvent[_]) 20 | /** Convert a wrapped event to a rewrite event. */ 21 | def toRewrite: RewriteEvent[_] = 22 | RewriteEvent(we) 23 | 24 | ////////////////////////////////////////////////////////////////////////////// 25 | // Mapping from RewritePortalEvents to WrappedEvents 26 | ////////////////////////////////////////////////////////////////////////////// 27 | 28 | extension (re: RewriteEvent[_]) 29 | /** Convert a rewrite event to a wrapped event. */ 30 | def toWrapped: WrappedEvents.WrappedEvent[_] = 31 | re.we 32 | 33 | end RewritePortalEvents // object 34 | -------------------------------------------------------------------------------- /portals-examples/src/test/scala/portals/examples/ExamplesTest.scala: -------------------------------------------------------------------------------- 1 | package portals.examples 2 | 3 | import scala.annotation.experimental 4 | 5 | import org.junit.runner.RunWith 6 | import org.junit.runners.JUnit4 7 | import org.junit.Assert._ 8 | import org.junit.Test 9 | 10 | /** Tests that all examples compile and run, does not test that the output is 11 | * correct. 12 | */ 13 | @experimental 14 | @RunWith(classOf[JUnit4]) 15 | class ExamplesTest: 16 | 17 | @Test 18 | def testShoppingCart(): Unit = 19 | import portals.examples.shoppingcart.ShoppingCartMain 20 | ShoppingCartMain 21 | 22 | @Test 23 | def testBankAccount(): Unit = 24 | import portals.examples.bankaccount.BankAccountMain 25 | BankAccountMain 26 | 27 | @Test 28 | def testHelloWorld(): Unit = 29 | import portals.examples.tutorial.HelloWorldMain 30 | HelloWorldMain 31 | 32 | @Test 33 | def testDynamicQuery(): Unit = 34 | import portals.examples.tutorial.DynamicQuery 35 | DynamicQuery() 36 | 37 | @Test 38 | def testIncrementalWordCount(): Unit = 39 | import portals.examples.tutorial.IncrementalWordCount 40 | IncrementalWordCount() 41 | 42 | @Test 43 | def testPortalPingPong(): Unit = 44 | import portals.examples.tutorial.PortalPingPong 45 | PortalPingPong() 46 | 47 | @Test 48 | def testStepByStep(): Unit = 49 | import portals.examples.tutorial.StepByStep 50 | StepByStep() 51 | 52 | @Test 53 | def testWordCount(): Unit = 54 | import portals.examples.tutorial.WordCount 55 | WordCount() 56 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/bankaccount/BankAccount.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.bankaccount 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.api.dsl.DSL.* 6 | import portals.api.dsl.ExperimentalDSL.* 7 | import portals.application.Application 8 | import portals.examples.bankaccount.BankAccountEvents.* 9 | 10 | //////////////////////////////////////////////////////////////////////////////// 11 | // Bank Account 12 | //////////////////////////////////////////////////////////////////////////////// 13 | object BankAccount: 14 | 15 | // the bank account application 16 | def app: Application = PortalsApp("BankAccount"): 17 | 18 | // generated stream of transactions 19 | val txnOps = Generators.fromIteratorOfIterators(BankAccountData.txnIter) 20 | 21 | // bank account service portal 22 | val account = Portal[Saga[AccountOperation], SagaReply[AccountOperation]]("account", keyFrom) 23 | 24 | // bank account workflow 25 | val accountWorkflow = Workflows[Nothing, Nothing]("accountWorkflow") 26 | .source(Generators.empty.stream) // no input 27 | .task[Nothing](AccountTask(account)) 28 | .sink() 29 | .freeze() 30 | 31 | // trigger workflow 32 | val _ = Workflows[Saga[AccountOperation], Nothing]("txns") 33 | .source(txnOps.stream) 34 | .task(TriggerTask(account)) 35 | .filter(_.isInstanceOf[SagaSuccess[AccountOperation]]) 36 | .logger() 37 | .empty[Nothing]() // consume the stream 38 | .sink() 39 | .freeze() 40 | 41 | end app 42 | end BankAccount 43 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/SBTRunServer.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed 2 | 3 | import cask.main.Main 4 | import io.undertow.Undertow 5 | 6 | /** An alternative server that can run with `sbt run` without stopping. 7 | * 8 | * @example 9 | * {{{ 10 | * sbt "distributed/runMain portals.distributed.SBTRunServer localhost 8080" 11 | * }}} 12 | */ 13 | object SBTRunServer: 14 | object InternalSBTRunServer extends cask.Main: 15 | // define the routes which this server handles 16 | override val allRoutes = Seq(Server) 17 | 18 | // override the default main method to handle the port argument 19 | override def main(args: Array[String]): Unit = { 20 | val host = if args.length > 0 then Some(args(0).toString) else Some("localhost") 21 | val port = if args.length > 1 then Some(args(1).toInt) else Some(8080) 22 | if (!verbose) Main.silenceJboss() 23 | val server = Undertow.builder 24 | .addHttpListener(port.get, host.get) 25 | .setHandler(defaultHandler) 26 | .build 27 | server.start() 28 | } 29 | 30 | // using main method here instead of extending App, see: 31 | // https://users.scala-lang.org/t/args-is-null-while-extending-app-even-when-runtime-args-are-provided/8564 32 | def main(args: Array[String]): Unit = 33 | // execute the main method of the server, starting it 34 | InternalSBTRunServer.main(args) 35 | 36 | // sleep so we don't exit prematurely 37 | Thread.sleep(Long.MaxValue) 38 | 39 | // exit the application (not sure if necessary here) 40 | System.exit(0) 41 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/ConnectionBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | 5 | /** Builder for Connections. 6 | * 7 | * Connect a stream to a sequencer using the ConnectionBuilder. This is done by 8 | * calling the `connect` method. 9 | * 10 | * @example 11 | * {{{builder.connections.connect(stream, sequencer)}}} 12 | */ 13 | trait ConnectionBuilder: 14 | /** Connect a stream to a sequencer. 15 | * 16 | * @example 17 | * {{{builder.connections.connect(stream, sequencer)}}} 18 | * 19 | * @param aStream 20 | * the stream to connect 21 | * @param sequencer 22 | * the sequencer to connect 23 | * @return 24 | * the connection 25 | */ 26 | def connect[T](aStream: AtomicStreamRefKind[T], sequencer: AtomicSequencerRefKind[T]): AtomicConnection[T] 27 | 28 | private[portals] object ConnectionBuilder: 29 | /** Internal API. Create a ConnectionBuilder using the builder context. */ 30 | def apply(name: String)(using bctx: ApplicationBuilderContext): ConnectionBuilder = 31 | val _name = bctx.name_or_id(name) 32 | new ConnectionBuilderImpl(_name) 33 | 34 | /** Internal API. Implementation of the ConnectionBuilder. */ 35 | class ConnectionBuilderImpl(name: String)(using bctx: ApplicationBuilderContext) extends ConnectionBuilder: 36 | def connect[T](aStream: AtomicStreamRefKind[T], sequencer: AtomicSequencerRefKind[T]): AtomicConnection[T] = 37 | val path = bctx.app.path + "/connections/" + name 38 | val connection = AtomicConnection[T](path, aStream, sequencer) 39 | bctx.addToContext(connection) 40 | connection 41 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/tutorial/PortalPingPong.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.tutorial 2 | 3 | import scala.annotation.experimental 4 | import scala.concurrent.Await 5 | 6 | import portals.api.dsl.DSL 7 | import portals.api.dsl.ExperimentalDSL 8 | import portals.system.Systems 9 | import portals.util.Future 10 | 11 | @experimental 12 | @main def PortalPingPong(): Unit = 13 | import portals.api.dsl.DSL.* 14 | 15 | import portals.api.dsl.ExperimentalDSL.* 16 | 17 | val app = PortalsApp("PortalPingPong") { 18 | sealed trait PingPong 19 | case class Ping(x: Int) extends PingPong 20 | case class Pong(x: Int) extends PingPong 21 | 22 | val generator = Generators.fromList(List(1024 * 128)) 23 | 24 | val portal = Portal[Ping, Pong]("portal") 25 | 26 | val replier = Workflows[Nothing, Nothing]("replier") 27 | .source(Generators.empty.stream) 28 | .replier[Nothing](portal) { _ => () } { case Ping(x) => 29 | reply(Pong(x - 1)) 30 | } 31 | .sink() 32 | .freeze() 33 | 34 | val asker = Workflows[Int, Int]("asker") 35 | .source(generator.stream) 36 | .recursiveAsker[Int](portal) { self => x => 37 | val future: Future[Pong] = ask(portal)(Ping(x)) 38 | await(future) { 39 | ctx.emit(future.value.get.x) 40 | if future.value.get.x > 0 then self(future.value.get.x) 41 | } 42 | } 43 | .filter(_ % 1024 == 0) 44 | .logger() 45 | .sink() 46 | .freeze() 47 | } 48 | 49 | val system = Systems.test() 50 | 51 | system.launch(app) 52 | 53 | system.stepUntilComplete() 54 | 55 | system.shutdown() 56 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/BatchedEvents.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime 2 | 3 | import portals.runtime.WrappedEvents.* 4 | import portals.util.Common.Types.Path 5 | 6 | object BatchedEvents: 7 | /** Internal API. Atom of events together with meta data. */ 8 | sealed trait EventBatch 9 | 10 | /** Atom of regular events. */ 11 | case class AtomBatch[T](path: String, list: List[WrappedEvent[T]]) extends EventBatch 12 | 13 | /** Metadata for ask and reply events. */ 14 | private[portals] case class AskReplyMeta( 15 | portal: Path, 16 | askingWF: String, 17 | replyingWF: Option[String] = None, 18 | replyingTask: Option[String] = None, 19 | ) 20 | 21 | /** Atom of ask events. */ 22 | private[portals] case class AskBatch[T](meta: AskReplyMeta, list: List[WrappedEvent[T]]) extends EventBatch 23 | 24 | /** Atom of reply events. */ 25 | private[portals] case class ReplyBatch[T](meta: AskReplyMeta, list: List[WrappedEvent[T]]) extends EventBatch 26 | 27 | /** Events that are shuffled due to a key change. */ 28 | private[portals] case class ShuffleBatch[T](path: Path, task: Path, list: List[WrappedEvent[T]]) extends EventBatch 29 | 30 | extension (batch: EventBatch) { 31 | def list: List[WrappedEvent[_]] = batch match 32 | case AtomBatch(_, list) => list 33 | case AskBatch(_, list) => list 34 | case ReplyBatch(_, list) => list 35 | case ShuffleBatch(_, _, list) => list 36 | 37 | def nonEmpty: Boolean = batch match 38 | case AtomBatch(_, list) => list.nonEmpty 39 | case AskBatch(_, list) => list.nonEmpty 40 | case ReplyBatch(_, list) => list.nonEmpty 41 | case ShuffleBatch(_, _, list) => list.nonEmpty 42 | } 43 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/CompilerPhase.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler 2 | 3 | /** Phase of the compiler that transforms an input of type `T` into an output of 4 | * type `U`. 5 | * 6 | * @tparam T 7 | * input type 8 | * @tparam U 9 | * output type 10 | */ 11 | private[portals] trait CompilerPhase[T, U] { 12 | self => 13 | 14 | /** Run the compiler phase on input `t`. 15 | * 16 | * @param t 17 | * input to be transformed 18 | * @param ctx 19 | * compiler context 20 | * @return 21 | * the transformed output 22 | */ 23 | def run(t: T)(using ctx: CompilerContext): U 24 | 25 | /** Compose this compiler phase with another `next` compiler phase. 26 | * 27 | * @param next 28 | * next compiler phase 29 | * @tparam V 30 | * output type of the next compiler phase 31 | * @return 32 | * a new compiler phase that first runs this compiler phase and then the 33 | * next compiler phase in sequence 34 | */ 35 | def andThen[V](next: CompilerPhase[U, V]): CompilerPhase[T, V] = 36 | new CompilerPhase[T, V] { 37 | override def run(t: T)(using ctx: CompilerContext): V = 38 | next.run { self.run(t) } 39 | } 40 | } 41 | 42 | object CompilerPhase: 43 | /** Empty compiler phase, can be used to start a chain of compiler phases. */ 44 | def empty[T]: CompilerPhase[T, T] = new CompilerPhase[T, T] { 45 | override def run(t: T)(using ctx: CompilerContext): T = t 46 | } 47 | 48 | /** Compiler phase that maps with the provided function `f`. */ 49 | def map[T, U](f: T => U): CompilerSubPhase[T, U] = new CompilerSubPhase[T, U] { 50 | override def run(t: T)(using ctx: CompilerContext): U = f(t) 51 | } 52 | -------------------------------------------------------------------------------- /portals-benchmark/src/main/scala/portals/benchmark/BenchmarkRunner.scala: -------------------------------------------------------------------------------- 1 | package portals.benchmark 2 | 3 | class BenchmarkRunner(): 4 | import BenchmarkRunner.* 5 | 6 | private val config = new BenchmarkConfig 7 | config.setRequired("--nIterations") // 5 8 | 9 | def warmup(benchmark: Benchmark, args: List[String] = List.empty) = 10 | config.parseArgs(args) 11 | benchmark.initialize(args) 12 | for (i <- 1 to config.getInt("--nIterations")) { 13 | benchmark.runOneIteration() 14 | benchmark.cleanupOneIteration() 15 | } 16 | 17 | // TODO: this should be guarded against long running blocking executions 18 | def run(benchmark: Benchmark, args: List[String] = List.empty) = 19 | config.parseArgs(args) 20 | benchmark.initialize(args) 21 | 22 | val timer = BenchmarkTimer() 23 | 24 | for (i <- 1 to config.getInt("--nIterations")) { 25 | timer.run { benchmark.runOneIteration() } 26 | benchmark.cleanupOneIteration() 27 | } 28 | 29 | println("name " + benchmark.name + " " + args.mkString(" ") + " " + timer.statistics) 30 | 31 | object BenchmarkRunner: 32 | class BenchmarkTimer(): 33 | var results = List[Long]() 34 | 35 | def run(f: => Unit) = { 36 | val tStart = System.nanoTime() 37 | f 38 | val tStop = System.nanoTime() 39 | results ::= (tStop - tStart) 40 | } 41 | 42 | private def toSeconds(nanos: Long) = nanos / 1000000000.0 43 | private def toMillis(nanos: Long) = nanos / 1000000.0 44 | private def toMicros(nanos: Long) = nanos / 1000.0 45 | private def toNanos(nanos: Long) = nanos 46 | 47 | private def average(results: List[Long]) = results.sum / results.length 48 | 49 | def statistics: String = "average(s) " + toSeconds(average(results)) 50 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/ThrottledGenerator.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart 2 | 3 | import portals.application.generator.Generator 4 | import portals.runtime.WrappedEvents.* 5 | import portals.util.Key 6 | 7 | /** ThrottledGenerator. Wrap a provided `generator`, and throttle its output to 8 | * at most `eventsPerSecond`. 9 | * 10 | * Note that the rate limiting is per partitioned generator instance. 11 | * 12 | * Note that the rate limiting is approximate. Once an atom has started 13 | * generating, it will not be interrupted, even in the case that the rate limit 14 | * is exceeded. That is, the last atom may exceed the rate limit since the last 15 | * threshold. 16 | * 17 | * @param generator 18 | * the wrapped generator 19 | * @param eventsPerSecond 20 | * the maximum number of events per second (approximate) 21 | * @tparam T 22 | * the generated event type 23 | */ 24 | class ThrottledGenerator[T](generator: Generator[T], eventsPerSecond: Int) extends Generator[T]: 25 | private var eventCount = 0L 26 | private var timeOfLastRateLimit = 0L 27 | 28 | private def rateLimitExceeded(): Boolean = 29 | if eventCount < eventsPerSecond then // 30 | false 31 | else 32 | val now = System.currentTimeMillis() 33 | val elapsed = now - timeOfLastRateLimit 34 | if elapsed > 1000 then 35 | timeOfLastRateLimit = now 36 | eventCount = 0 37 | false 38 | else // 39 | true 40 | 41 | override def generate(): WrappedEvent[T] = 42 | generator.generate() match 43 | case e @ Event(key, value) => 44 | eventCount += 1 45 | e 46 | case e @ _ => 47 | e 48 | 49 | override def hasNext(): Boolean = 50 | !rateLimitExceeded() && generator.hasNext() 51 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/physicalplan/PhysicalPlanBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.physicalplan 2 | 3 | import portals.application.Application 4 | 5 | private[portals] object PhysicalPlanBuilder: 6 | def fromApplication(app: Application): Plan[PlanConfig] = 7 | Plan(app, EmptyConfig) 8 | 9 | def withConfig(plan: Plan[_], config: PlanConfig): Plan[config.type] = 10 | Plan(plan.app, config) 11 | 12 | /** Split a physical plan into num ranges */ 13 | def split(plan: PhysicalPlan[_], num_ranges: Int): List[PhysicalPlan[BaseConfig]] = plan match 14 | case Plan(app, _) => 15 | val splits = compute_splits(num_ranges) 16 | splits.sliding(2).map(x => Plan(app, BaseConfig((x(0), x(1))))).toList 17 | 18 | /** Returns a sorted list of #num_ranges ranges */ 19 | private def compute_splits(num_ranges: Int): List[Long] = 20 | assert(num_ranges > 0) 21 | if num_ranges == 1 then List(Long.MinValue, Long.MaxValue) 22 | else if num_ranges == 2 then List(Long.MinValue, 0L, Long.MaxValue) 23 | else if num_ranges % 2 == 0 then 24 | val splitSize = (Long.MaxValue / num_ranges.toLong) - (Long.MinValue / num_ranges.toLong) 25 | var splits = List(Long.MinValue, 0L, Long.MaxValue) 26 | for i <- 1 to ((num_ranges / 2) - 1) do 27 | splits = splits :+ (splitSize * i) 28 | splits = splits :+ (-splitSize * i) 29 | splits.sorted 30 | else 31 | val splitSize = (Long.MaxValue / num_ranges.toLong) - (Long.MinValue / num_ranges.toLong) 32 | var splits = List(Long.MinValue, -(splitSize / 2), (splitSize / 2), Long.MaxValue) 33 | for i <- 1 to ((num_ranges / 2) - 1) do 34 | splits = splits :+ ((splitSize / 2) + (splitSize * i)) 35 | splits = splits :+ -((splitSize / 2) + (splitSize * i)) 36 | splits.sorted 37 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/bankaccount/BankAccountEvents.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.bankaccount 2 | 3 | import scala.annotation.experimental 4 | 5 | object BankAccountEvents: 6 | ////////////////////////////////////////////////////////////////////////////// 7 | // Saga Events 8 | ////////////////////////////////////////////////////////////////////////////// 9 | 10 | // Saga Operations 11 | sealed trait SagaOperation[T] 12 | 13 | // Saga Requests 14 | sealed trait SagaRequest[T] extends SagaOperation[T] 15 | 16 | case class Saga[T](head: T, tail: List[T]) extends SagaRequest[T] 17 | 18 | // Saga Replies 19 | sealed trait SagaReply[T] extends SagaOperation[T] 20 | 21 | case class SagaSuccess[T](saga: Option[Saga[T]] = None) extends SagaReply[T] 22 | case class SagaAbort[T](saga: Option[Saga[T]] = None) extends SagaReply[T] 23 | 24 | ////////////////////////////////////////////////////////////////////////////// 25 | // Account Events 26 | ////////////////////////////////////////////////////////////////////////////// 27 | 28 | // Account Operations 29 | sealed trait AccountOperation 30 | 31 | case class Deposit(id: Long, amount: Int) extends AccountOperation 32 | case class Withdraw(id: Long, amount: Int) extends AccountOperation 33 | 34 | ////////////////////////////////////////////////////////////////////////////// 35 | // Key Extractors 36 | ////////////////////////////////////////////////////////////////////////////// 37 | 38 | def keyFrom(op: AccountOperation): Long = op match 39 | case Deposit(id, _) => id 40 | case Withdraw(id, _) => id 41 | 42 | def keyFrom(saga: SagaOperation[AccountOperation]): Long = saga match 43 | case Saga(head, _) => 44 | keyFrom(head) 45 | case SagaSuccess(Some(Saga(head, _))) => 46 | keyFrom(head) 47 | case SagaAbort(Some(Saga(head, _))) => 48 | keyFrom(head) 49 | case _ => ??? 50 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/Server.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed 2 | 3 | import java.io.File 4 | 5 | import portals.application.* 6 | import portals.application.Application 7 | import portals.application.AtomicStreamRefKind 8 | import portals.distributed.ApplicationLoader.PortalsClassLoader 9 | import portals.distributed.Events.* 10 | import portals.system.Systems 11 | 12 | import upickle.default.* 13 | import upickle.default.read 14 | 15 | /** Portals Server for handling remote requests to submit and launch 16 | * applications. 17 | * 18 | * Note: if you intend to run this with `sbt run` then you should use 19 | * `SBTRunServer` instead. 20 | */ 21 | object Server extends cask.MainRoutes: 22 | /** Handle a `SubmitClassFiles` event. Submits class files to the server. */ 23 | @cask.post("/submitClassFiles") 24 | def submitClassFiles(request: cask.Request) = 25 | val bytes = request.readAllBytes() 26 | read[SubmitClassFiles](bytes) match 27 | case SubmitClassFiles(classFiles) => 28 | classFiles.foreach: 29 | case cfi @ ClassFileInfo(_, _) => 30 | PortalsClassLoader.addClassFile(cfi) 31 | cask.Response("success", statusCode = 200) 32 | 33 | /** Handle a `Launch` event. Launches an application together with all of its 34 | * dependencies. 35 | */ 36 | @cask.post("/launch") 37 | def launch(request: cask.Request) = 38 | val bytes = request.readAllBytes() 39 | read[Launch](bytes) match 40 | case Launch(app) => 41 | val clazz = ApplicationLoader.loadClassFromName(app) 42 | val application = ApplicationLoader.createInstanceFromClass(clazz).asInstanceOf[SubmittableApplication].apply() 43 | ASTPrinter.println(application) 44 | ServerRuntime.launch(application) 45 | cask.Response("success", statusCode = 200) 46 | 47 | // initialize the server (cask.MainRoutes method) 48 | initialize() 49 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/examples/HelloWorld.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed.examples 2 | 3 | import portals.api.dsl.DSL.* 4 | import portals.application.Application 5 | import portals.distributed.SubmittableApplication 6 | import portals.system.Systems 7 | 8 | /** A simple application that prints "Hello World!" to the log. 9 | * 10 | * Run this application either locally (see the main method, example below), or 11 | * submit it to a distributed/remote execution (see examples below). 12 | * 13 | * Note: it is important to use the correct Java path when submitting the apps 14 | * (typically ending with a $ symbol). 15 | * 16 | * @example 17 | * Run it locally 18 | * {{{ 19 | * sbt "distributed/runMain portals.distributed.examples.HelloWorld" 20 | * }}} 21 | * 22 | * @example 23 | * Submit it to a server 24 | * {{{ 25 | * // start the server (in a different terminal) 26 | * sbt "distributed/runMain portals.distributed.SBTRunServer" 27 | * sbt "distributed/runMain portals.distributed.ClientCLI submitDir --directory portals-distributed/target/scala-3.3.0/classes" 28 | * sbt "distributed/runMain portals.distributed.ClientCLI launch --application portals.distributed.examples.HelloWorld$" 29 | * }}} 30 | */ 31 | object HelloWorld extends SubmittableApplication: 32 | override def apply(): Application = 33 | // simple hello world example that prints Hello World! in reverse 34 | PortalsApp("HelloWorld") { 35 | val generator = Generators.signal("Hello World!") 36 | val workflow = Workflows[String, String]() 37 | .source(generator.stream) 38 | .map(_.reverse) 39 | .logger() 40 | .sink() 41 | .freeze() 42 | } 43 | 44 | def main(args: Array[String]): Unit = 45 | val system = Systems.test() 46 | system.launch(HelloWorld.apply()) 47 | system.stepUntilComplete() 48 | system.shutdown() 49 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/ServerRuntime.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed 2 | 3 | import java.io.File 4 | import java.util.concurrent.ConcurrentLinkedQueue 5 | 6 | import scala.concurrent.Future 7 | 8 | import portals.application.* 9 | import portals.application.Application 10 | import portals.application.AtomicStreamRefKind 11 | import portals.distributed.ApplicationLoader.PortalsClassLoader 12 | import portals.distributed.Events.* 13 | import portals.system.Systems 14 | 15 | import upickle.default.* 16 | import upickle.default.read 17 | 18 | /** Server runtime system used by the distributed server. New applications can 19 | * be submitted via the `launch` method. Note that the system automatically 20 | * starts upon initialization. 21 | */ 22 | object ServerRuntime: 23 | private class Worker: 24 | private val queue = new ConcurrentLinkedQueue[() => Unit]() 25 | 26 | def submitJob(job: () => Unit): Unit = 27 | queue.offer(job) 28 | 29 | // start automatically on creation 30 | new Thread(new Runnable { 31 | override def run(): Unit = 32 | while (true) do 33 | Option(queue.poll()) match 34 | case Some(job) => 35 | job() 36 | case None => 37 | Thread.sleep(100) 38 | }).start() 39 | end Worker 40 | 41 | private def stepJob: () => Unit = 42 | () => 43 | if system.canStep() then // 44 | system.stepUntilComplete(1024) 45 | Thread.sleep(100) 46 | worker.submitJob(stepJob) 47 | 48 | private def launchJob(application: Application): () => Unit = 49 | () => system.launch(application) 50 | 51 | // start system and worker 52 | private val system = Systems.test() 53 | private val worker = new Worker() 54 | worker.submitJob(stepJob) 55 | 56 | /** Launch an application on the server runtime. */ 57 | def launch(application: Application): Unit = 58 | worker.submitJob(launchJob(application)) 59 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/trackers/ProgressTracker.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.trackers 2 | 3 | import scala.util.Random 4 | 5 | import portals.application.* 6 | import portals.application.task.AskerReplierTask 7 | import portals.application.task.AskerTask 8 | import portals.application.task.ReplierTask 9 | import portals.compiler.phases.RuntimeCompilerPhases 10 | import portals.runtime.BatchedEvents.* 11 | import portals.runtime.PortalsRuntime 12 | import portals.runtime.WrappedEvents.* 13 | 14 | /** Internal API. Tracks the progress for a path with respect to other streams. 15 | */ 16 | private[portals] class ProgressTracker: 17 | // progress tracker for each Path; 18 | // for a Path (String) this gives the progress w.r.t. all input dependencies (Map[String, Long]) 19 | private var progress: Map[String, Map[String, Long]] = Map.empty 20 | 21 | /** Set the progress of path and dependency to index. */ 22 | def setProgress(path: String, dependency: String, idx: Long): Unit = 23 | progress += 24 | path -> 25 | (progress(path) + (dependency -> idx)) 26 | 27 | /** Increments the progress of path w.r.t. dependency by 1. */ 28 | def incrementProgress(path: String, dependency: String): Unit = 29 | progress += 30 | path -> 31 | (progress(path) + (dependency -> (progress(path)(dependency) + 1))) 32 | 33 | /** Initialize the progress tracker for a certain path and dependency to -1. 34 | */ 35 | def initProgress(path: String, dependency: String): Unit = 36 | progress += path -> (progress.getOrElse(path, Map.empty) + (dependency -> -1L)) 37 | 38 | /** Get the current progress of the path and dependency. */ 39 | def getProgress(path: String, dependency: String): Option[Long] = 40 | progress.get(path).flatMap(deps => deps.get(dependency)) 41 | 42 | /** Get the current progress of the path. */ 43 | def getProgress(path: String): Option[Map[String, Long]] = 44 | progress.get(path) 45 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/SuspendingRuntime.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime 2 | 3 | import portals.application.Application 4 | import portals.runtime.BatchedEvents.* 5 | import portals.util.Common.Types.Path 6 | 7 | object SuspendingRuntime: 8 | /** Resulting output produced by taking a `step()`. */ 9 | sealed trait StepResult 10 | 11 | /** Complete `output` from processing an atom to completion. */ 12 | case class Completed( 13 | path: Path, 14 | outputs: List[EventBatch] 15 | ) extends StepResult 16 | 17 | /** Suspended processing of `path` with `shuffle` side-effects and 18 | * `intermediate` output. 19 | */ 20 | case class Suspended( 21 | path: Path, 22 | shuffle: List[ShuffleBatch[_]], 23 | intermediate: List[EventBatch], 24 | ) extends StepResult 25 | 26 | trait SuspendingRuntime extends PortalsRuntime: 27 | import SuspendingRuntime.* 28 | 29 | /** Returns true if the runtime can take another step. */ 30 | def canStep(): Boolean 31 | 32 | /** Take and return the result of a step. 33 | * 34 | * Returns `Completed` if the step processes an atom until completion. 35 | * 36 | * Returns `Suspended` if the processing was suspended, for example due to a 37 | * `ShuffleTask`. A suspended processor will not be able to take another step 38 | * until it is `resume()`d. A suspended processor is resumed through the call 39 | * `resume()` with the corresponding `path` and `shuffle` side-effects. 40 | */ 41 | def step(): StepResult 42 | 43 | /** Resume a suspended processor for `path` and input `shuffles`. */ 44 | def resume(path: Path, shuffles: List[ShuffleBatch[_]]): StepResult 45 | 46 | /** Feed a `listOfAtoms` to the runtime. 47 | * 48 | * Distributes the atoms to the corresponding processing units. Can be used 49 | * to feed back the `outputs` from `Completed` steps. 50 | */ 51 | def feedAtoms(listOfAtoms: List[EventBatch]): Unit 52 | 53 | end SuspendingRuntime 54 | -------------------------------------------------------------------------------- /portals-libraries/src/main/scala/portals/libraries/actor/examples/PingPong.scala: -------------------------------------------------------------------------------- 1 | package portals.libraries.actor.examples 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.api.dsl.DSL 6 | import portals.api.dsl.ExperimentalDSL 7 | import portals.libraries.actor.* 8 | import portals.system.Systems 9 | 10 | @experimental 11 | object PingPong: 12 | inline val PINGPONG_N = 1024 13 | 14 | case class PingPong(i: Int, replyTo: ActorRef[PingPong]) 15 | 16 | object PingPongActors: 17 | val pingPongBehavior: ActorBehavior[PingPong] = ActorBehaviors.receive { ctx ?=> msg => 18 | ctx.log(msg) 19 | msg match 20 | case PingPong(i, replyTo) => 21 | i match 22 | case -1 => 23 | ctx.log("PingPong finished") 24 | ActorBehaviors.stopped 25 | case 0 => 26 | ctx.log("PingPong finished") 27 | ctx.send(replyTo)(PingPong(i - 1, ctx.self)) 28 | ActorBehaviors.stopped 29 | case _ => 30 | ctx.send(replyTo)(PingPong(i - 1, ctx.self)) 31 | ActorBehaviors.same 32 | } 33 | 34 | val initBehavior: ActorBehavior[Unit] = ActorBehaviors.init { ctx ?=> 35 | val p1 = ctx.create(pingPongBehavior) 36 | val p2 = ctx.create(pingPongBehavior) 37 | ctx.send(p1)(PingPong(PINGPONG_N, p2)) 38 | ActorBehaviors.stopped 39 | } 40 | 41 | @experimental 42 | object PingPongMain extends App: 43 | import portals.api.dsl.DSL.* 44 | import portals.api.dsl.ExperimentalDSL.* 45 | 46 | import ActorEvents.* 47 | import PingPong.PingPongActors.* 48 | 49 | val config = ActorConfig.default 50 | .replace("logging", false) 51 | 52 | val app = PortalsApp("PingPong") { 53 | val generator = Generators.signal[ActorMessage](ActorCreate(ActorRef.fresh(), initBehavior)) 54 | val wf = ActorWorkflow(generator.stream, config) 55 | } 56 | 57 | /** synchronous interpreter */ 58 | val system = Systems.test() 59 | system.launch(app) 60 | system.stepUntilComplete() 61 | system.shutdown() 62 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/examples/shoppingcart/ShoppingCart.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed.examples.shoppingcart 2 | 3 | import portals.distributed.Client 4 | 5 | /** Incrementally launches a shopping cart application using the `Client`. 6 | * 7 | * Note: in order to run this example, a corresponding `Server` must be 8 | * running. @see [[portals.distributed.SBTRunServer]]. 9 | * 10 | * Note: this will use the programmatic `Client` object interface to launch the 11 | * application. 12 | * 13 | * Optionally, to launch the application using the `ClientCLI` see the 14 | * following example: 15 | * ``` 16 | * // comment out the files that you want to submit to the server (so that they are not compiled with the server.) 17 | * 18 | * // start the server (in a different terminal) 19 | * sbt "distributed/runMain portals.distributed.SBTRunServer" 20 | * 21 | * // uncomment the files you want to submit 22 | * 23 | * // submit the class files with the client 24 | * sbt "distributed/runMain portals.distributed.ClientCLI submitDir --directory portals-distributed/target/scala-3.3.0/classes" 25 | * 26 | * // launch each application with the client 27 | * sbt "distributed/runMain portals.distributed.ClientCLI launch --application portals.distributed.examples.shoppingcart.Inventory$" 28 | * sbt "distributed/runMain portals.distributed.ClientCLI launch --application portals.distributed.examples.shoppingcart.Cart$" 29 | * sbt "distributed/runMain portals.distributed.ClientCLI launch --application portals.distributed.examples.shoppingcart.Orders$" 30 | * sbt "distributed/runMain portals.distributed.ClientCLI launch --application portals.distributed.examples.shoppingcart.Analytics$" 31 | * ``` 32 | */ 33 | object ShoppingCart extends App: 34 | // submit all the class files with the client 35 | Client.submitObjectWithDependencies(this) 36 | 37 | // launch the applications 38 | Client.launchObject(Inventory) 39 | Client.launchObject(Cart) 40 | Client.launchObject(Orders) 41 | Client.launchObject(Analytics) 42 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/SplitBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | 5 | /** Builder for splitting an atomic stream into multiple atomic streams. 6 | * 7 | * Splits can be added to a splitter via the `builder.splits.split` method. 8 | * Accessed from the application builder via `builder.splits`. 9 | * 10 | * @example 11 | * {{{builder.splits.split[String](splitter, filter)}}} 12 | */ 13 | trait SplitBuilder: 14 | /** Add a split to a splitter. 15 | * 16 | * @example 17 | * {{{builder.splits.split[String](splitter, filter)}}} 18 | * 19 | * @param splitter 20 | * the splitter to add the split to 21 | * @param filter 22 | * the filter to determine which events are filtered for the split 23 | * @return 24 | * the new split atomic stream 25 | */ 26 | def split[T](splitter: AtomicSplitterRefKind[T], filter: T => Boolean): AtomicStreamRef[T] 27 | 28 | /** Internal API. The split builder. */ 29 | object SplitBuilder: 30 | /** Internal API. Create a split builder using the application context. */ 31 | def apply(name: String)(using bctx: ApplicationBuilderContext): SplitBuilder = 32 | val _name = bctx.name_or_id(name) 33 | new SplitBuilderImpl(_name) 34 | end SplitBuilder // object 35 | 36 | /** Internal API. Implementation of the split builder. */ 37 | class SplitBuilderImpl(name: String)(using bctx: ApplicationBuilderContext) extends SplitBuilder: 38 | override def split[T](splitter: AtomicSplitterRefKind[T], filter: T => Boolean): AtomicStreamRef[T] = 39 | val _name = bctx.name_or_id() 40 | val _path = splitter.path + "/" + _name 41 | val _split_path = splitter.path + "/" + "split" + "/" + bctx.name_or_id() 42 | val _newStream = AtomicStream[T](_path) 43 | val _newSplit = AtomicSplit[T](_split_path, AtomicSplitterRef(splitter.path), AtomicStreamRef(_newStream), filter) 44 | bctx.addToContext(_newStream) 45 | bctx.addToContext(_newSplit) 46 | AtomicStreamRef(_newStream) 47 | end SplitBuilderImpl // class 48 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/SplitterBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | import portals.application.splitter.Splitter 5 | import portals.application.splitter.Splitters 6 | 7 | /** Build splitters to split an atomic stream into multiple atomic streams. 8 | * 9 | * Accessed from the application builder via `builder.splitters`. 10 | * 11 | * @example 12 | * {{{builder.splitters.empty[String](stream)}}} 13 | */ 14 | trait SplitterBuilder: 15 | /** Create an empty splitter with no splits from a `stream`. 16 | * 17 | * @example 18 | * {{{builder.splitters.empty[String](stream)}}} 19 | * 20 | * @param stream 21 | * the stream to split 22 | * @return 23 | * the splitter reference with no splits 24 | */ 25 | def empty[T](stream: AtomicStreamRefKind[T]): AtomicSplitterRef[T] 26 | end SplitterBuilder // trait 27 | 28 | /** Internal API. The splitter builder. */ 29 | object SplitterBuilder: 30 | /** Internal API. Create a SplitterBuilder using the application context. */ 31 | def apply(name: String)(using bctx: ApplicationBuilderContext): SplitterBuilder = 32 | val _name = bctx.name_or_id(name) 33 | new SplitterBuilderImpl(_name) 34 | end SplitterBuilder // trait 35 | 36 | /** Internal API. Implementation of the splitter builder. */ 37 | class SplitterBuilderImpl(name: String)(using bctx: ApplicationBuilderContext) extends SplitterBuilder: 38 | private def build[T](_stream: AtomicStreamRefKind[T], _splitter: Splitter[T]): AtomicSplitterRef[T] = 39 | val _path = bctx.app.path + "/splitters/" + name 40 | val _in = _stream 41 | val _streams = List.empty 42 | val aSplitter = AtomicSplitter[T]( 43 | path = _path, 44 | in = _in, 45 | streams = _streams, 46 | splitter = _splitter, 47 | ) 48 | bctx.addToContext(aSplitter) 49 | AtomicSplitterRef(aSplitter) 50 | 51 | override def empty[T](stream: AtomicStreamRefKind[T]): AtomicSplitterRef[T] = 52 | val _splitter = Splitters.empty[T]() 53 | this.build(stream, _splitter) 54 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/trackers/StreamTracker.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.trackers 2 | 3 | import scala.util.Random 4 | 5 | import portals.application.* 6 | import portals.application.task.AskerReplierTask 7 | import portals.application.task.AskerTask 8 | import portals.application.task.ReplierTask 9 | import portals.compiler.phases.RuntimeCompilerPhases 10 | import portals.runtime.BatchedEvents.* 11 | import portals.runtime.PortalsRuntime 12 | import portals.runtime.WrappedEvents.* 13 | 14 | /** Internal API. Tracks all streams of all applications. 15 | * 16 | * The stream tracker is used to track the progress of the streams, i.e. what 17 | * range of indices of the stream that can be read. The smallest index may be 18 | * incremented due to garbage collection over time. 19 | */ 20 | private[portals] class StreamTracker: 21 | /** Maps the progress of a path (String) to a pair [from, to], the range is 22 | * inclusive and means that all indices starting from 'from' until 23 | * (including) 'to' can be read. 24 | */ 25 | private var _progress: Map[String, (Long, Long)] = 26 | Map.empty 27 | 28 | /** Initialize a new stream by settings its progress to <0, -1>, that is it is 29 | * empty for now. 30 | */ 31 | def initStream(stream: String): Unit = 32 | _progress += stream -> (0, -1) 33 | 34 | /** Set the progress of a stream to , for which the range is 35 | * inclusive. Use this with care, use incrementProgress instead where 36 | * possible. 37 | */ 38 | def setProgress(stream: String, from: Long, to: Long): Unit = 39 | _progress += stream -> (from, to) 40 | 41 | /** Increments the progress of a stream by 1. */ 42 | def incrementProgress(stream: String): Unit = 43 | _progress += stream -> (_progress(stream)._1, _progress(stream)._2 + 1) 44 | 45 | /** Returns the progress of a stream as an optional range , for 46 | * which the range is inclusive. 47 | */ 48 | def getProgress(stream: String): Option[(Long, Long)] = 49 | _progress.get(stream) 50 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/ApplicationBuilderImpl.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | import portals.compiler.* 5 | 6 | /** Application Builder Implementation. */ 7 | private[portals] class ApplicationBuilderImpl(using val bctx: ApplicationBuilderContext) extends ApplicationBuilder: 8 | override def build(): Application = 9 | bctx.freeze() 10 | // may throw exception 11 | CompilerBuilder 12 | .preCompiler() 13 | .compile(bctx.app) 14 | 15 | override def registry: RegistryBuilder = RegistryBuilder() 16 | 17 | override def workflows[T, U]: WorkflowBuilder[T, U] = this.workflows(null) 18 | 19 | override def workflows[T, U](name: String = null): WorkflowBuilder[T, U] = 20 | val _name = bctx.name_or_id(name) 21 | val wfb = WorkflowBuilder[T, U](_name) 22 | // added to the context, so that we later can check if it was frozen or not, 23 | // if it wasn't frozen, then we try to freeze it when the application is built 24 | bctx._workflowBuilders = bctx._workflowBuilders :+ wfb 25 | wfb 26 | 27 | override def splitters: SplitterBuilder = this.splitters(null) 28 | 29 | override def splitters(name: String = null): SplitterBuilder = SplitterBuilder(name) 30 | 31 | override def splits: SplitBuilder = this.splits(null) 32 | 33 | override def splits(name: String = null): SplitBuilder = SplitBuilder(name) 34 | 35 | override def generators: GeneratorBuilder = this.generators(null) 36 | 37 | override def generators(name: String = null): GeneratorBuilder = GeneratorBuilder(name) 38 | 39 | override def sequencers: SequencerBuilder = this.sequencers(null) 40 | 41 | override def sequencers(name: String = null): SequencerBuilder = SequencerBuilder(name) 42 | 43 | override def connections: ConnectionBuilder = this.connections(null) 44 | 45 | override def connections(name: String = null): ConnectionBuilder = ConnectionBuilder(name) 46 | 47 | override def portals: PortalBuilder = this.portals(null) 48 | 49 | override def portals(name: String = null): PortalBuilder = PortalBuilder(name) 50 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/SequencerBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | import portals.application.sequencer.Sequencer 5 | import portals.application.sequencer.Sequencers 6 | 7 | /** Build sequencers. 8 | * 9 | * Sequencers are used to sequence multiple atomic streams into a single atomic 10 | * stream. Atomic streams are connected to a sequencer via using the 11 | * [[ConnectionBuilder]] `builder.connections.connect`. 12 | * 13 | * Accessed from the application builder via `builder.sequencers`. 14 | * 15 | * @example 16 | * {{{builder.sequencers.random[String]()}}} 17 | */ 18 | trait SequencerBuilder: 19 | /** Create a random sequencer. 20 | * 21 | * @example 22 | * {{{builder.sequencers.random[String]()}}} 23 | * 24 | * @return 25 | * the sequencer reference 26 | */ 27 | def random[T](): AtomicSequencerRef[T] 28 | end SequencerBuilder 29 | 30 | /** Internal API. The sequencer builder. */ 31 | object SequencerBuilder: 32 | /** Internal API. Create a SequencerBuilder using the application context. */ 33 | def apply(name: String)(using bctx: ApplicationBuilderContext): SequencerBuilder = 34 | val _name = bctx.name_or_id(name) 35 | new SequencerBuilderImpl(_name) 36 | end SequencerBuilder 37 | 38 | /** Internal API. Implementation of the SequencerBuilder. */ 39 | class SequencerBuilderImpl(name: String)(using bctx: ApplicationBuilderContext) extends SequencerBuilder: 40 | private def build[T](_sequencer: Sequencer[T]): AtomicSequencerRef[T] = 41 | val _path = bctx.app.path + "/sequencers/" + name 42 | val aStream = AtomicStream[T](path = _path + "/stream") 43 | val _stream = AtomicStreamRef(aStream) 44 | val aSequencer = AtomicSequencer[T]( 45 | path = _path, 46 | stream = _stream, 47 | sequencer = _sequencer, 48 | ) 49 | bctx.addToContext(aSequencer) 50 | bctx.addToContext(aStream) 51 | AtomicSequencerRef(aSequencer) 52 | 53 | override def random[T](): AtomicSequencerRef[T] = 54 | val _sequencer = Sequencers.random[T]() 55 | build(_sequencer) 56 | 57 | end SequencerBuilderImpl 58 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/WorkflowBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | 5 | /** Builder for workflows. 6 | * 7 | * Accessed from the application builder via `builder.workflows`. 8 | * 9 | * @example 10 | * {{{builder.workflows[String, Int]("workflowName").source().map(x => x.length()).sink().freeze()}}} 11 | */ 12 | trait WorkflowBuilder[T, U]: 13 | /** Internal API. Freezes and completes a workflow that was not frozen by the 14 | * user. Called by the system when the application is built. 15 | */ 16 | private[portals] def complete(): Unit 17 | 18 | /** Freeze a workflow, this will prevent any further changes to the workflow. 19 | * 20 | * In order to use a workflow's output further in the application, it must be 21 | * frozen so that its output stream can be accessed. 22 | * 23 | * @return 24 | * reference to the workflow 25 | */ 26 | def freeze(): Workflow[T, U] 27 | 28 | /** Start the workflow from the source stream `ref`. Returns a flow. 29 | * 30 | * The additional type parameter is used to make manual checks that the type 31 | * is correct, i.e. `source[Int](ref)` ensures that the input type is Int. 32 | * 33 | * @example 34 | * {{{builder.workflows[String, Int]("workflowName").source(stream).map(x => x.length())}}} 35 | * 36 | * @tparam TT 37 | * type of the source stream 38 | * @param ref 39 | * reference to the source stream 40 | * @return 41 | * a flow starting from the source stream 42 | */ 43 | def source[TT >: T <: T](ref: AtomicStreamRefKind[T]): FlowBuilder[T, U, TT, TT] 44 | end WorkflowBuilder // trait 45 | 46 | /** Internal API. The workflow builder. */ 47 | object WorkflowBuilder: 48 | /** Internal API. Create a WorkflowBuilder using the application context. */ 49 | def apply[T, U](name: String)(using bctx: ApplicationBuilderContext): WorkflowBuilder[T, U] = 50 | val _path = bctx.app.path + "/workflows/" + name 51 | given wbctx: WorkflowBuilderContext[T, U] = new WorkflowBuilderContext[T, U](_path = _path) 52 | new WorkflowBuilderImpl[T, U]() 53 | end WorkflowBuilder // object 54 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/ApplicationBuilderContext.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | 5 | /** Application Builder Context. */ 6 | class ApplicationBuilderContext(_path: String): 7 | var app: Application = Application(path = _path) 8 | 9 | def addToContext(e: AST): Unit = e match 10 | case x: Workflow[_, _] => 11 | app = app.copy(workflows = app.workflows :+ x) 12 | case x: AtomicGenerator[_] => 13 | app = app.copy(generators = app.generators :+ x) 14 | case x: AtomicStream[_] => 15 | app = app.copy(streams = app.streams :+ x) 16 | case x: AtomicSequencer[_] => 17 | app = app.copy(sequencers = app.sequencers :+ x) 18 | case x: AtomicSplitter[_] => 19 | app = app.copy(splitters = app.splitters :+ x) 20 | case x: AtomicConnection[_] => 21 | app = app.copy(connections = app.connections :+ x) 22 | case x: AtomicSplit[_] => 23 | app = app.copy(splits = app.splits :+ x) 24 | case x: AtomicPortal[_, _] => 25 | app = app.copy(portals = app.portals :+ x) 26 | case x: ExtAtomicStreamRef[_] => 27 | app = app.copy(externalStreams = app.externalStreams :+ x) 28 | case x: ExtAtomicSequencerRef[_] => 29 | app = app.copy(externalSequencers = app.externalSequencers :+ x) 30 | case x: ExtAtomicSplitterRef[_] => 31 | app = app.copy(externalSplitters = app.externalSplitters :+ x) 32 | case x: ExtAtomicPortalRef[_, _] => 33 | app = app.copy(externalPortals = app.externalPortals :+ x) 34 | case x: Application => ??? 35 | case _ => ??? 36 | 37 | private var _next_id: Int = 0 38 | def next_id(): String = 39 | _next_id = _next_id + 1 40 | "$" + _next_id.toString 41 | 42 | def name_or_id(name: String): String = if name == null then this.next_id() else name 43 | 44 | def name_or_id(): String = this.next_id() 45 | 46 | // added to the context, so that we later can check if it was frozen or not, 47 | // if it wasn't frozen, then we try to freeze it when the application is built 48 | var _workflowBuilders: List[WorkflowBuilder[_, _]] = List.empty 49 | def freeze(): Unit = 50 | _workflowBuilders.foreach { _.complete() } 51 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/processors/InterpreterSplitter.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.processors 2 | 3 | import portals.application.AtomicSplitter 4 | import portals.runtime.BatchedEvents.* 5 | import portals.runtime.WrappedEvents.* 6 | 7 | /** Internal API. Test Runtime wrapper around the Splitter. */ 8 | private[portals] class InterpreterSplitter(splitter: AtomicSplitter[_]) extends ProcessingStepper: 9 | 10 | /** Add an output to the splitter, that filters out the events for the path. 11 | * 12 | * @tparam X 13 | * the type of the events that will be filtered. 14 | * @param path 15 | * the path to which the filtered events will be sent. 16 | * @param filter 17 | * the filter function that will be applied to the events. 18 | */ 19 | def addOutput[X](path: String, filter: X => Boolean): Unit = 20 | // TODO: shouldn't have a type param 21 | splitter.splitter.addOutput(path, filter.asInstanceOf[Any => Boolean]) 22 | 23 | /** Remove an output from the splitter. 24 | * 25 | * @param path 26 | * the path of the output to remove. 27 | */ 28 | def removeOutput(path: String): Unit = splitter.splitter.removeOutput(path) 29 | 30 | /** Create a list representation using the splitter events. */ 31 | private def toSplitterAtom(atom: EventBatch): List[WrappedEvent[Any]] = 32 | atom match 33 | case AtomBatch(path, list) => 34 | list 35 | case _ => ??? 36 | 37 | /** Create an atom representation from the splitter representation. */ 38 | private def fromSplitterAtom(path: String, satom: List[WrappedEvent[Any]]): EventBatch = 39 | AtomBatch(path, satom) 40 | 41 | /** Process an atom on the test splitter. This will produce a list of new 42 | * atoms, one for each nonempty output. 43 | * 44 | * @param atom 45 | * the atom to process. 46 | * @return 47 | * a list of new atoms, one for each nonempty output. 48 | */ 49 | override def step(atom: EventBatch): List[EventBatch] = 50 | splitter 51 | .asInstanceOf[AtomicSplitter[Any]] 52 | .splitter 53 | .split(toSplitterAtom(atom)) 54 | .map(x => fromSplitterAtom(x._1, x._2)) 55 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/ShoppingCart.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.api.dsl.DSL 6 | import portals.api.dsl.DSL.* 7 | import portals.api.dsl.ExperimentalDSL.* 8 | import portals.application.Application 9 | import portals.examples.shoppingcart.tasks.* 10 | import portals.examples.shoppingcart.ShoppingCartEvents.* 11 | 12 | //////////////////////////////////////////////////////////////////////////////// 13 | // Shopping Cart 14 | //////////////////////////////////////////////////////////////////////////////// 15 | object ShoppingCart: 16 | def app: Application = PortalsApp("shopping-cart"): 17 | val inventoryOpsGenerator = Generators.generator(ShoppingCartData.inventoryOpsGenerator) 18 | 19 | val cartOpsGenerator = Generators.generator(ShoppingCartData.cartOpsGenerator) 20 | 21 | val portal = Portal[InventoryReqs, InventoryReps]("inventory", keyFrom) 22 | 23 | val analyticsportal = Portal[AnalyticsReqs, AnalyticsReps]("analytics", keyFrom) 24 | 25 | val cart = Workflows[CartOps, OrderOps]("cart") 26 | .source(cartOpsGenerator.stream) 27 | .key(keyFrom(_)) 28 | .task(CartTask(portal)) 29 | .withName("cart") 30 | .sink() 31 | .freeze() 32 | 33 | val inventory = Workflows[InventoryReqs, Nothing]("inventory") 34 | .source(inventoryOpsGenerator.stream) 35 | .key(keyFrom(_)) 36 | .task(InventoryTask(portal)) 37 | .withName("inventory") 38 | .sink() 39 | .freeze() 40 | 41 | val orders = Workflows[OrderOps, OrderOps]("orders") 42 | .source(cart.stream) 43 | .key(keyFrom(_)) 44 | .task(OrdersTask()) 45 | .withName("orders") 46 | .sink() 47 | .freeze() 48 | 49 | val analytics = Workflows[OrderOps, Nothing]("analyics") 50 | .source(orders.stream) 51 | .flatMap { case Order(_, CartState(items)) => items } 52 | .key(keyFrom(_)) 53 | .task(AggregatingTask()) 54 | .key(_ => 0L) 55 | .task(AnalyticsTask(analyticsportal)) 56 | .logger() 57 | .nothing() 58 | .sink() 59 | .freeze() 60 | end app // def 61 | end ShoppingCart // object 62 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/bankaccount/tasks/TriggerTask.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.bankaccount 2 | 3 | import portals.api.builder.TaskBuilder 4 | import portals.api.dsl.* 5 | import portals.api.dsl.DSL.* 6 | import portals.application.task.AskerReplierTaskContext 7 | import portals.application.task.AskerTaskContext 8 | import portals.application.task.GenericTask 9 | import portals.application.task.LoggingTaskContext 10 | import portals.application.task.PerKeyState 11 | import portals.application.task.StatefulTaskContext 12 | import portals.application.AtomicPortalRefKind 13 | import portals.examples.bankaccount.BankAccountEvents.* 14 | 15 | // Task that triggers the account Portal with requests, and emits the replies 16 | object TriggerTask: 17 | type I = Saga[AccountOperation] 18 | type O = SagaReply[AccountOperation] 19 | type Req = Saga[AccountOperation] 20 | type Res = SagaReply[AccountOperation] 21 | type Context = AskerTaskContext[Req, Res, Req, Res] 22 | type PortalRef = AtomicPortalRefKind[Req, Res] 23 | type Task = GenericTask[Req, Res, Req, Res] 24 | 25 | // Handle onNext events 26 | private def onNext(portal: PortalRef)(event: I)(using Context): Unit = 27 | event match 28 | case saga @ Saga(head, tail) => 29 | if BankAccountConfig.LOGGING then log.info(s"Asking for transaction: $saga") 30 | 31 | // Send the saga transaction to the account portal 32 | val f = ask(portal)(saga) 33 | 34 | // Wait for the reply (async) 35 | await(f) { 36 | f.value.get match 37 | // Transaction succeeded, emit SagaSuccess 38 | case SagaSuccess(_) => 39 | if BankAccountConfig.LOGGING then ctx.log.info(s"Whole transaction success: $saga") 40 | ctx.emit(SagaSuccess(Some(saga))) 41 | 42 | // Transaction aborted, emit SagaAbort 43 | case SagaAbort(_) => 44 | if BankAccountConfig.LOGGING then ctx.log.info(s"Whole transaction aborted: $saga") 45 | ctx.emit(SagaAbort(Some(saga))) 46 | } 47 | 48 | // Create a new trigger task 49 | def apply(portal: PortalRef): Task = 50 | Tasks.asker(portal)(onNext(portal)) 51 | 52 | end TriggerTask 53 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/compiler/phases/WellFormedCheck.scala: -------------------------------------------------------------------------------- 1 | package portals.compiler.phases 2 | 3 | import portals.application.task.AskerReplierTask 4 | import portals.application.task.ReplierTask 5 | import portals.application.Application 6 | import portals.compiler.* 7 | 8 | /** Check if the application is well-formed. Throws exception. */ 9 | private[portals] object WellFormedCheck extends CompilerPhase[Application, Application]: 10 | override def run(application: Application)(using CompilerContext): Application = 11 | // 1. check for naming collissions 12 | // X: no two distinct elements may share the same path 13 | val allNames = application.workflows.map(_.path) 14 | ++ application.generators.map(_.path) 15 | ++ application.streams.map(_.path) 16 | ++ application.sequencers.map(_.path) 17 | ++ application.splitters.map(_.path) 18 | ++ application.connections.map(_.path) 19 | ++ application.portals.map(_.path) 20 | ++ application.externalStreams.map(_.path) 21 | ++ application.externalSequencers.map(_.path) 22 | ++ application.externalPortals.map(_.path) 23 | if allNames.length != allNames.toSet.toList.length then ??? 24 | 25 | // 2. check that a portal has a single replier 26 | // X: If an application defines a portal, then it must also define exactly one portal handler, and vice versa. 27 | // format: off 28 | val portals = application.portals.map(_.path) 29 | val portalTasks = application.workflows.map(_.tasks).flatMap(_.values) 30 | val _portals = { 31 | val replierTaskPortals = portalTasks.filter { case r @ ReplierTask(_, _) => true; case _ => false }.flatMap(_.asInstanceOf[ReplierTask[_, _, _, _]].portals).map(_.path) 32 | val askerReplierTaskPortals = portalTasks.filter { case r @ AskerReplierTask(_, _) => true; case _ => false }.flatMap(_.asInstanceOf[AskerReplierTask[_, _, _, _]].replyerportals).map(_.path) 33 | replierTaskPortals ++ askerReplierTaskPortals 34 | } 35 | if !(portals.length == _portals.length && portals.forall { _portals.contains(_) } && _portals.forall {portals.contains(_)}) then ??? 36 | // format: on 37 | 38 | // forward the app 39 | application 40 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/tutorial/StepByStep.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.tutorial 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.api.builder.TaskBuilder 6 | import portals.api.dsl.DSL 7 | import portals.api.dsl.ExperimentalDSL 8 | import portals.system.Systems 9 | 10 | /** Step By Step 11 | * 12 | * Lyrics from Step by Step of the Album Step By Step by Eddie Rabbit, 1981. 13 | */ 14 | @experimental 15 | @main def StepByStep(): Unit = 16 | import portals.api.dsl.DSL.* 17 | 18 | import portals.api.dsl.ExperimentalDSL.* 19 | 20 | val app = PortalsApp("StepByStep") { 21 | 22 | val lyrics = List( 23 | "First step, ask her out and treat her like a lady", 24 | "Second step, tell her she's the one you're dreaming of", 25 | "Third step, take her in your arms and never let her go", 26 | "Don't you know that step by step, step by step", 27 | "You'll win her love", 28 | "Second step", 29 | "Third step", 30 | "Don't you know step by step, step by step", 31 | "--", 32 | ) 33 | 34 | val generator = 35 | Generators.fromIteratorOfIterators[Unit](Iterator.continually(Iterator.single(()))) 36 | 37 | // The workflow takes steps over atoms, and for every new atom it will output the next line in the lyric to the log 38 | Workflows[Unit, Nothing]("stepper") 39 | .source(generator.stream) 40 | .task(TaskBuilder.processor { _ => log.info(lyrics(0)) }) 41 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(1)) }) 42 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(2)) }) 43 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(3)) }) 44 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(4)) }) 45 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(5)) }) 46 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(6)) }) 47 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(7)) }) 48 | .withStep(TaskBuilder.processor { _ => log.info(lyrics(8)) }) 49 | .empty[Nothing]() 50 | .sink() 51 | .freeze() 52 | } 53 | 54 | val system = Systems.test() 55 | 56 | system.launch(app) 57 | 58 | Range(0, 128).foreach { _ => 59 | system.step() 60 | } 61 | 62 | system.shutdown() 63 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/test/TestRuntime.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.test 2 | 3 | import scala.util.Random 4 | 5 | import portals.application.* 6 | import portals.runtime.* 7 | import portals.runtime.interpreter.* 8 | import portals.runtime.BatchedEvents.* 9 | import portals.runtime.SuspendingRuntime.* 10 | import portals.util.Common.Types.* 11 | 12 | private[portals] class TestRuntime(val seed: Option[Int] = None) extends SteppingRuntime: 13 | private val ctx: InterpreterContext = new InterpreterContext(seed) 14 | private val interpreter: Interpreter = new Interpreter(ctx) 15 | 16 | // TODO: make part of config 17 | private inline val GC_INTERVAL = 4 18 | 19 | /** The current step number of the execution. */ 20 | private var _stepN: Long = 0 21 | private def stepN(): Long = 22 | _stepN += 1 23 | _stepN 24 | 25 | override def launch(application: Application): Unit = 26 | interpreter.launch(application) 27 | end launch 28 | 29 | override def canStep(): Boolean = 30 | interpreter.canStep() 31 | end canStep 32 | 33 | /** Recursively call resume until fully completed. 34 | * 35 | * This is guaranteed to terminate due to acyclic processing units. The 36 | * number of recursive steps is bounded by the number of `ShuffleTask`s. 37 | */ 38 | private def recursiveResume( 39 | path: Path, 40 | shuffle: List[ShuffleBatch[_]], 41 | intermediate: List[EventBatch], 42 | ): Completed = 43 | interpreter.resume(path, shuffle) match 44 | case Completed(path, outputs) => 45 | Completed(path, outputs ::: intermediate) 46 | case Suspended(path, shuffle, newIntermediate) => 47 | recursiveResume(path, shuffle, newIntermediate ::: intermediate) 48 | end recursiveResume 49 | 50 | override def step(): Unit = 51 | interpreter.step() match 52 | case Completed(path, outputs) => 53 | interpreter.feedAtoms(outputs) 54 | case Suspended(path, shuffle, outputs) => 55 | recursiveResume(path, shuffle, outputs) match 56 | case Completed(path, outputs) => 57 | interpreter.feedAtoms(outputs) 58 | 59 | if stepN() % GC_INTERVAL == 0 then interpreter.garbageCollect() 60 | end step 61 | 62 | override def shutdown(): Unit = () // do nothing 63 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/util/Future.scala: -------------------------------------------------------------------------------- 1 | package portals.util 2 | 3 | import portals.application.task.AskerTaskContext 4 | import portals.application.task.PerTaskState 5 | 6 | /** A future is a value that may be completed and available at a later time. 7 | * 8 | * Futures are created by requesting a value from a portal. The value is 9 | * completed by a corresponding reply from the portal. The `await` method 10 | * registers a continuation on the future that is executed when the future is 11 | * completed. 12 | * 13 | * @example 14 | * {{{ 15 | * val future = ask(portal, request) 16 | * await(future) { future.value.get match 17 | * case Rep(value) => 18 | * log.info(value.toString()) 19 | * } 20 | * }}} 21 | * 22 | * @tparam T 23 | * the type of the value 24 | */ 25 | trait Future[T]: 26 | /** Internal API. The unique identifier of the future. Used to match against 27 | * replies to complete the future and execute the continuation. 28 | */ 29 | private[portals] val id: Int 30 | 31 | /** The value of the future. If the future is not yet completed, the value is 32 | * `None`. If the future is completed, the value is `Some(value)`. Try using 33 | * `await` instead to register a continuation for when the future is 34 | * completed. 35 | * 36 | * @return 37 | * the optional value of the future 38 | */ 39 | def value: AskerTaskContext[_, _, _, T] ?=> Option[T] 40 | 41 | override def toString(): String = "Future(id=" + id + ")" 42 | 43 | /** Internal API. Implementation of the [[Future]] trait. */ 44 | private[portals] class FutureImpl[T](override val id: Int) extends Future[T]: 45 | // TODO: should be a global shared value instead to ensure that it is consistent with others 46 | private lazy val _futures: AskerTaskContext[_, _, _, T] ?=> PerTaskState[Map[Int, T]] = 47 | PerTaskState[Map[Int, T]]("futures", Map.empty) 48 | 49 | override def value: AskerTaskContext[_, _, _, T] ?=> Option[T] = 50 | this._futures.get().get(id) 51 | 52 | /** Internal API. Future factory. */ 53 | private[portals] object Future: 54 | private val _rand = new scala.util.Random 55 | private def generate_id: Int = { _rand.nextInt() } 56 | 57 | def apply[T](): Future[T] = new FutureImpl[T](generate_id) 58 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/tasks/InventoryTask.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart.tasks 2 | 3 | import portals.api.dsl.DSL.* 4 | import portals.application.task.* 5 | import portals.application.AtomicPortalRef 6 | import portals.application.AtomicPortalRefKind 7 | import portals.examples.shoppingcart.ShoppingCartConfig 8 | import portals.examples.shoppingcart.ShoppingCartEvents.* 9 | 10 | object InventoryTask: 11 | type I = InventoryReqs 12 | type O = Nothing 13 | type Req = InventoryReqs 14 | type Res = InventoryReps 15 | type Context = ProcessorTaskContext[I, O] 16 | type RepContext = ReplierTaskContext[I, O, Req, Res] 17 | type PortalRef = AtomicPortalRefKind[Req, Res] 18 | type Task = GenericTask[I, O, Req, Res] 19 | 20 | private final val state: PerKeyState[Int] = 21 | PerKeyState[Int]("state", 0) 22 | 23 | private def get(e: Get)(using Context): Unit = 24 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Taking ${e.item} from inventory") 25 | state.get() match 26 | case x if x > 0 => state.set(x - 1) 27 | case _ => ??? 28 | 29 | private def put(e: Put)(using Context): Unit = 30 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Putting ${e.item} in inventory") 31 | state.set(state.get() + 1) 32 | 33 | private def get_req(e: Get)(using RepContext): Unit = 34 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Checking if ${e.item} is in inventory") 35 | state.get() match 36 | case x if x > 0 => 37 | reply(GetReply(e.item, true)) 38 | state.set(x - 1) 39 | case _ => 40 | reply(GetReply(e.item, false)) 41 | 42 | private def put_req(e: Put)(using RepContext): Unit = 43 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Putting ${e.item} in inventory") 44 | state.set(state.get() + 1) 45 | reply(PutReply(e.item, true)) 46 | 47 | private def onNext(event: InventoryReqs)(using Context): Unit = event match 48 | case e: Get => get(e) 49 | case e: Put => put(e) 50 | 51 | private def onAsk(event: InventoryReqs)(using RepContext): Unit = event match 52 | case e: Get => get_req(e) 53 | case e: Put => put_req(e) 54 | 55 | def apply(portal: PortalRef): Task = 56 | Tasks.replier(portal)(onNext)(onAsk) 57 | 58 | end InventoryTask 59 | -------------------------------------------------------------------------------- /portals-distributed/src/main/scala/portals/distributed/Util.scala: -------------------------------------------------------------------------------- 1 | package portals.distributed 2 | 3 | import java.io.File 4 | import java.nio.file.Files 5 | import java.nio.file.Paths 6 | import java.util.jar.JarFile 7 | 8 | import scala.jdk.CollectionConverters.* 9 | 10 | /** Utility functions used in the scope of the distributed library. */ 11 | object Util: 12 | def getBytes(path: String): Array[Byte] = 13 | val file = File(path) 14 | val bytes = Files.readAllBytes(file.toPath()) 15 | bytes 16 | 17 | def getBytes(path: String, directory: String): Array[Byte] = 18 | if directory.endsWith(".jar") then // 19 | getBytesFromJar(path, directory) 20 | else // 21 | getBytesFromFile(path, directory) 22 | 23 | def getBytesFromFile(path: String, directory: String): Array[Byte] = 24 | val file = File(directory, path) 25 | val bytes = Files.readAllBytes(file.toPath()) 26 | bytes 27 | 28 | def getBytesFromJar(path: String, directory: String): Array[Byte] = 29 | val jarFile = JarFile(directory) 30 | val jarEntry = jarFile.getJarEntry(path) 31 | val inputStream = jarFile.getInputStream(jarEntry) 32 | val bytes = inputStream.readAllBytes() 33 | bytes 34 | 35 | def getBytesFromDirectory(directory: String): Seq[(String, Array[Byte])] = 36 | if directory.endsWith(".jar") then // 37 | getBytesFromDirectoryFromJar(directory) 38 | else // 39 | getBytesFromDirectoryFromDirectory(directory) 40 | 41 | def getBytesFromDirectoryFromDirectory(directory: String): Seq[(String, Array[Byte])] = 42 | val directoryPath = Paths.get(directory) 43 | val files = Files.walk(directoryPath).iterator().asScala.toSeq 44 | val classFiles = files.filter(_.toString.endsWith(".class")) 45 | val bytes = classFiles.map(Files.readAllBytes).toSeq 46 | val paths = classFiles.map(directoryPath.relativize(_).toString) 47 | paths.zip(bytes) 48 | 49 | def getBytesFromDirectoryFromJar(directory: String): Seq[(String, Array[Byte])] = 50 | val jarFile = JarFile(directory) 51 | val entries = jarFile.entries().asScala.toSeq 52 | val classFiles = entries.filter(_.getName.endsWith(".class")) 53 | val bytes = classFiles.map(jarFile.getInputStream(_).readAllBytes()) 54 | val paths = classFiles.map(_.getName) 55 | paths.zip(bytes) 56 | -------------------------------------------------------------------------------- /portals-libraries/src/main/scala/portals/libraries/actor/examples/CountingActor.scala: -------------------------------------------------------------------------------- 1 | package portals.libraries.actor.examples 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.libraries.actor.* 6 | import portals.system.Systems 7 | 8 | @experimental 9 | object CountingActor: 10 | inline val COUNTING_N = 1024 11 | 12 | sealed trait CountingActorCommand 13 | case object Increment extends CountingActorCommand 14 | case class Retrieve(replyTo: ActorRef[ProducerActorCommand]) extends CountingActorCommand 15 | 16 | sealed trait ProducerActorCommand 17 | case class RetrieveReply(value: Int) extends ProducerActorCommand 18 | 19 | object CountingBehaviors: 20 | val countingBehavior: ActorBehavior[CountingActorCommand] = 21 | ActorBehaviors.init { ctx ?=> 22 | val count = ValueTypedActorState[Int]("count") 23 | 24 | ActorBehaviors.receive { 25 | case Increment => 26 | count.set(count.get().getOrElse(0) + 1) 27 | ActorBehaviors.same 28 | 29 | case Retrieve(replyTo) => 30 | ctx.send(replyTo)(RetrieveReply(count.get().getOrElse(0))) 31 | ActorBehaviors.stopped 32 | } 33 | } 34 | 35 | val producerBehavior: ActorBehavior[ProducerActorCommand] = 36 | ActorBehaviors.init { ctx ?=> 37 | val countingActor = ctx.create(countingBehavior) 38 | 39 | for (i <- 0 until COUNTING_N) do ctx.send(countingActor)(Increment) 40 | 41 | ctx.send(countingActor)(Retrieve(ctx.self)) 42 | 43 | ActorBehaviors.receive { case RetrieveReply(value) => 44 | ctx.log(s"CountingActor replied with $value") 45 | ActorBehaviors.stopped 46 | } 47 | } 48 | 49 | @experimental 50 | object CountingActorMain extends App: 51 | import portals.api.dsl.DSL.* 52 | import portals.api.dsl.ExperimentalDSL.* 53 | 54 | import ActorEvents.* 55 | import CountingActor.CountingBehaviors.* 56 | 57 | val config = ActorConfig.default 58 | .replace("logging", false) 59 | 60 | val app = PortalsApp("CountingActor") { 61 | val generator = Generators.signal[ActorMessage](ActorCreate(ActorRef.fresh(), producerBehavior)) 62 | val wf = ActorWorkflow(generator.stream, config) 63 | } 64 | 65 | val system = Systems.test() 66 | system.launch(app) 67 | system.stepUntilComplete() 68 | system.shutdown() 69 | -------------------------------------------------------------------------------- /portals-core/jvm/src/test/scala/portals/api/builder/SequencerTest.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import org.junit.runner.RunWith 4 | import org.junit.runners.JUnit4 5 | import org.junit.Assert._ 6 | import org.junit.Ignore 7 | import org.junit.Test 8 | 9 | import portals.api.builder.ApplicationBuilder 10 | import portals.api.dsl.DSL 11 | import portals.system.Systems 12 | import portals.test.* 13 | import portals.test.TestUtils 14 | 15 | @RunWith(classOf[JUnit4]) 16 | class SequencerTest: 17 | 18 | @Test 19 | def randomTest(): Unit = 20 | import portals.api.dsl.DSL.* 21 | 22 | val tester = new TestUtils.Tester[Int]() 23 | 24 | val builder = ApplicationBuilder("app") 25 | 26 | val sequencer = builder.sequencers 27 | .random[Int]() 28 | 29 | // producers 30 | val generator1 = builder.generators.fromRange(0, 128, 5) 31 | val generator2 = builder.generators.fromRange(128, 256, 5) 32 | 33 | // connect producers to sequencer 34 | val _ = builder.connections.connect(generator1.stream, sequencer) 35 | val _ = builder.connections.connect(generator2.stream, sequencer) 36 | 37 | // consumers 38 | val _ = builder 39 | .workflows[Int, Int]("consumingWorkflow") 40 | .source(sequencer.stream) 41 | .task(tester.task) 42 | .sink() 43 | .freeze() 44 | 45 | val app = builder.build() 46 | 47 | // ASTPrinter.println(app) 48 | 49 | val system = Systems.test() 50 | 51 | system.launch(app) 52 | 53 | system.stepUntilComplete() 54 | 55 | system.shutdown() 56 | 57 | val received = tester.receiveAll() 58 | val receivedAtoms = tester.receiveAllAtoms() 59 | val testData = Iterator.range(0, 256).toList 60 | val testDataAtoms = Iterator.range(0, 128).grouped(5).toList ++ Iterator.range(128, 256).grouped(5).toList 61 | 62 | // 1. all events have been produced and consumed 63 | assertEquals(received.sorted, testData) 64 | 65 | // 2. the received events are in the same order as the events that were produced 66 | val testData21 = testData.filter(_ < 128) 67 | val testData22 = testData.filter(_ >= 128) 68 | assertEquals(received.filter(_ < 128), testData21) 69 | assertEquals(received.filter(_ >= 128), testData22) 70 | 71 | // 3. the atoms are not fused / jumbled by the sequencer 72 | testDataAtoms.foreach { atom => 73 | assertEquals(true, receivedAtoms.contains(atom)) 74 | } 75 | -------------------------------------------------------------------------------- /portals-core/jvm/src/test/scala/portals/api/builder/GeneratorTest.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import scala.annotation.experimental 4 | 5 | import org.junit.runner.RunWith 6 | import org.junit.runners.JUnit4 7 | import org.junit.Assert._ 8 | import org.junit.Ignore 9 | import org.junit.Test 10 | 11 | import portals.api.builder.ApplicationBuilder 12 | import portals.api.dsl.DSL 13 | import portals.system.Systems 14 | import portals.test.* 15 | import portals.test.TestUtils 16 | 17 | @RunWith(classOf[JUnit4]) 18 | class GeneratorTest: 19 | 20 | @Test 21 | def fromIteratorTest(): Unit = 22 | import portals.api.dsl.DSL.* 23 | 24 | val tester = new TestUtils.Tester[Int]() 25 | 26 | val builder = ApplicationBuilder("app") 27 | 28 | val generator = builder.generators 29 | .fromIterator[Int](Iterator.range(0, 10)) 30 | 31 | val _ = builder 32 | .workflows[Int, Int]("wf") 33 | .source(generator.stream) 34 | .task(tester.task) 35 | // .logger() 36 | .sink() 37 | .freeze() 38 | 39 | val app = builder.build() 40 | 41 | val system = Systems.test() 42 | 43 | // ASTPrinter.println(app) 44 | 45 | system.launch(app) 46 | 47 | system.stepUntilComplete() 48 | system.shutdown() 49 | 50 | Iterator.range(0, 10).foreach { x => 51 | tester.receiveAssert(x) 52 | } 53 | 54 | @Test 55 | def fromIteratorOfIteratorsTest(): Unit = 56 | import portals.api.dsl.DSL.* 57 | 58 | val tester = new TestUtils.Tester[Int]() 59 | 60 | val builder = ApplicationBuilder("app") 61 | 62 | val generator = builder.generators 63 | .fromIteratorOfIterators[Int](Iterator.from(0).map { x => Iterator.range(0, 5).map(_ + 5 * x) }) 64 | 65 | val workflow = builder 66 | .workflows[Int, Int]("wf") 67 | .source(generator.stream) 68 | .task(tester.task) 69 | // .logger() 70 | .sink() 71 | .freeze() 72 | 73 | val app = builder.build() 74 | 75 | val system = Systems.test() 76 | 77 | // ASTPrinter.println(app) 78 | 79 | system.launch(app) 80 | 81 | // take two steps 82 | 83 | system.step() 84 | system.step() 85 | (0 until 5).foreach { i => 86 | tester.receiveAssert(i) 87 | } 88 | 89 | // take two more steps 90 | system.step() 91 | system.step() 92 | (5 until 10).foreach { i => 93 | tester.receiveAssert(i) 94 | } 95 | system.shutdown() 96 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/tasks/AnalyticsTask.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart.tasks 2 | 3 | import portals.api.dsl.DSL.* 4 | import portals.application.* 5 | import portals.application.task.* 6 | import portals.application.task.MapTaskStateExtension.* 7 | import portals.examples.shoppingcart.ShoppingCartConfig 8 | import portals.examples.shoppingcart.ShoppingCartEvents.* 9 | 10 | object AnalyticsTask: 11 | type I = (Item, Count) 12 | type O = (List[(Item)]) // Top100 13 | type Req = AnalyticsReqs 14 | type Res = AnalyticsReps 15 | type Context = ProcessorTaskContext[I, O] 16 | type RepContext = ReplierTaskContext[I, O, Req, Res] 17 | type PortalRef = AtomicPortalRefKind[Req, Res] 18 | type Task = GenericTask[I, O, Req, Res] 19 | 20 | private final val top100: PerTaskState[Map[Item, Int]] = 21 | PerTaskState[Map[Item, Int]]("state", Map.empty) 22 | 23 | private final val max: PerTaskState[Int] = 24 | PerTaskState[Int]("max", 0) 25 | 26 | private final val min: PerTaskState[Int] = 27 | PerTaskState[Int]("min", 0) 28 | 29 | private def setMaxMin()(using Context): Unit = 30 | val t100 = top100.get() 31 | max.set(t100.values.max) 32 | min.set(t100.values.min) 33 | 34 | private def pruneTop100()(using Context): Unit = 35 | val t100 = top100.get() 36 | if t100.size >= 100 then 37 | val minItemCount = t100.minBy(_._2) 38 | top100.remove(minItemCount._1) 39 | 40 | private def onNext(event: (Item, Count))(using Context): Unit = 41 | event match 42 | case (item, count) if count < min.get() => 43 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Item $item with count $count is not added to top100") 44 | case (item, count) => 45 | if top100.get().contains(item) then 46 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Item $item with count $count is updated in the top100") 47 | top100.update(item, count) 48 | else 49 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Item $item with count $count is added to the top100") 50 | top100.update(item, count) 51 | pruneTop100() 52 | setMaxMin() 53 | ctx.emit(top100.get().keys.toList) 54 | 55 | private def onAsk(event: Req)(using RepContext): Unit = 56 | val t100 = top100.get().keys.toList 57 | reply(Top100Reply(t100)) 58 | 59 | def apply(portal: PortalRef): Task = 60 | Tasks.replier(portal: PortalRef)(onNext)(onAsk) 61 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/tutorial/DynamicQuery.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.tutorial 2 | 3 | import scala.annotation.experimental 4 | import scala.concurrent.Await 5 | 6 | import portals.api.dsl.DSL 7 | import portals.api.dsl.ExperimentalDSL 8 | import portals.application.task.PerTaskState 9 | import portals.system.Systems 10 | import portals.util.Future 11 | 12 | @experimental 13 | @main def DynamicQuery(): Unit = 14 | import portals.api.dsl.DSL.* 15 | 16 | import portals.api.dsl.ExperimentalDSL.* 17 | 18 | case class Query() 19 | case class QueryReply(x: Int) 20 | 21 | val dynamicQueryApp = PortalsApp("DynamicQuery") { 22 | //////////////////////////////////////////////////////////////////////////// 23 | // Aggregator 24 | //////////////////////////////////////////////////////////////////////////// 25 | // Atom(1, 2, 3, 4), Atom(1, 2, 3, 4), ... 26 | val generator = Generators.fromListOfLists(List.fill(10)(List(1, 2, 3, 4))) 27 | 28 | val portal = Portal[Query, QueryReply]("portal") 29 | 30 | val aggregator = Workflows[Int, Nothing]("aggregator") 31 | .source(generator.stream) 32 | .replier[Nothing](portal) { x => 33 | val sum = PerTaskState("sum", 0) 34 | sum.set(sum.get() + x) // aggregates sum 35 | } { case Query() => 36 | reply(QueryReply(PerTaskState("sum", 0).get())) 37 | } 38 | .sink() 39 | .freeze() 40 | 41 | //////////////////////////////////////////////////////////////////////////// 42 | // Query 43 | //////////////////////////////////////////////////////////////////////////// 44 | val queryTrigger = Generators.fromListOfLists(List.fill(10)(List(0))) 45 | 46 | val queryWorkflow = Workflows[Int, Nothing]("queryWorkflow") 47 | .source(queryTrigger.stream) 48 | .asker[Nothing](portal) { x => 49 | // query the aggregate 50 | val future: Future[QueryReply] = ask(portal)(Query()) 51 | await(future) { 52 | future.value.get match 53 | case QueryReply(x) => 54 | // print the aggregate to log 55 | ctx.log.info(x.toString()) 56 | } 57 | } 58 | .sink() 59 | .freeze() 60 | 61 | } // end dynamicQueryApp 62 | 63 | val system = Systems.test() 64 | 65 | system.launch(dynamicQueryApp) 66 | 67 | // stepping should output only replied aggregates of whole numbers, i.e. divisible by 10. 68 | system.stepUntilComplete() 69 | 70 | system.shutdown() 71 | -------------------------------------------------------------------------------- /portals-examples/src/test/scala/portals/examples/tutorial/PingPongTest.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.tutorial 2 | 3 | import org.junit.runner.RunWith 4 | import org.junit.runners.JUnit4 5 | import org.junit.Assert._ 6 | import org.junit.Ignore 7 | import org.junit.Test 8 | 9 | import portals.api.builder.ApplicationBuilder 10 | import portals.system.Systems 11 | import portals.test.TestUtils 12 | 13 | @RunWith(classOf[JUnit4]) 14 | class PingPongTest: 15 | 16 | @Test 17 | def testPingPong(): Unit = 18 | import portals.api.dsl.DSL.* 19 | 20 | case class Ping(i: Int) 21 | case class Pong(i: Int) 22 | 23 | val tester = new TestUtils.Tester[Pong]() 24 | 25 | val system = Systems.test() 26 | 27 | { 28 | val pinger = ApplicationBuilder("pinger") 29 | 30 | val sequencer = pinger.sequencers("sequencer").random[Pong]() 31 | 32 | val _ = pinger 33 | .workflows[Pong, Ping]("workflow") 34 | .source[Pong](sequencer.stream) 35 | .map { case Pong(i) => Ping(i) } 36 | .sink[Ping]() 37 | .freeze() 38 | 39 | val pingerApp = pinger.build() 40 | 41 | system.launch(pingerApp) 42 | } 43 | 44 | { 45 | val ponger = ApplicationBuilder("ponger") 46 | 47 | val extStream = ponger.registry.streams.get[Ping]("/pinger/workflows/workflow/stream") 48 | 49 | val pongerwf = ponger 50 | .workflows[Ping, Pong]("ponger") 51 | .source[Ping](extStream) 52 | .map { case Ping(i) => Pong(i - 1) } 53 | .filter(_.i > 0) 54 | // .logger() 55 | .task(tester.task) 56 | .sink[Pong]() 57 | .freeze() 58 | 59 | val extSequencer = ponger.registry.sequencers.get[Pong]("/pinger/sequencers/sequencer") 60 | val _ = ponger.connections.connect(pongerwf.stream, extSequencer) 61 | 62 | val pongerApp = ponger.build() 63 | 64 | system.launch(pongerApp) 65 | } 66 | 67 | { 68 | val builder = ApplicationBuilder("generator") 69 | 70 | val generator = builder.generators.fromList(List(Pong(128))) 71 | 72 | val extSequencer = builder.registry.sequencers.get[Pong]("/pinger/sequencers/sequencer") 73 | 74 | val connection = builder.connections.connect(generator.stream, extSequencer) 75 | 76 | val app = builder.build() 77 | 78 | system.launch(app) 79 | } 80 | 81 | system.stepUntilComplete() 82 | 83 | system.shutdown() 84 | 85 | assertEquals((127 until 0 by -1).map(Pong(_)).toList, tester.receiveAll()) 86 | -------------------------------------------------------------------------------- /portals-examples/src/main/scala/portals/examples/shoppingcart/tasks/CartTask.scala: -------------------------------------------------------------------------------- 1 | package portals.examples.shoppingcart.tasks 2 | 3 | import scala.annotation.experimental 4 | 5 | import portals.api.dsl.DSL.* 6 | import portals.application.task.* 7 | import portals.application.AtomicPortalRef 8 | import portals.application.AtomicPortalRefKind 9 | import portals.examples.shoppingcart.* 10 | import portals.examples.shoppingcart.ShoppingCartEvents.* 11 | 12 | object CartTask: 13 | type I = CartOps 14 | type O = OrderOps 15 | type Req = InventoryReqs 16 | type Res = InventoryReps 17 | type Context = AskerTaskContext[I, O, Req, Res] 18 | type PortalRef = AtomicPortalRefKind[Req, Res] 19 | type Task = GenericTask[I, O, Req, Res] 20 | 21 | private final val state: PerKeyState[CartState] = 22 | PerKeyState[CartState]("state", CartState.zero) 23 | 24 | private def addToCart(event: AddToCart, portal: PortalRef)(using Context): Unit = 25 | val req = Get(event.item) 26 | val resp = ask(portal)(req) 27 | await(resp): 28 | resp.value match 29 | case Some(GetReply(item, true)) => 30 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"User ${event.user} added $item to cart") 31 | state.set(state.get().add(item)) 32 | case Some(GetReply(item, false)) => 33 | if ShoppingCartConfig.LOGGING then 34 | ctx.log.info(s"User ${event.user} tried to add $item to cart, but it was not in inventory") 35 | case _ => 36 | if ShoppingCartConfig.LOGGING then ctx.log.info("Unexpected response") 37 | ??? 38 | 39 | private def removeFromCart(event: RemoveFromCart, portal: PortalRef)(using Context): Unit = 40 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"User ${event.user} removed ${event.item} from cart") 41 | val req = Put(event.item) 42 | val _ = ask(portal)(req) 43 | 44 | private def checkout(event: Checkout)(using Context): Unit = 45 | if ShoppingCartConfig.LOGGING then ctx.log.info(s"Checking out ${event.user}") 46 | val cart = state.get() 47 | ctx.emit(Order(event.user, cart)) 48 | state.del() 49 | 50 | private def onNext(portal: PortalRef)(event: CartOps)(using Context): Unit = 51 | event match 52 | case event: AddToCart => addToCart(event, portal) 53 | case event: RemoveFromCart => removeFromCart(event, portal) 54 | case event: Checkout => checkout(event) 55 | 56 | def apply(portal: PortalRef): Task = 57 | Tasks.asker(portal)(onNext(portal)) 58 | 59 | end CartTask 60 | -------------------------------------------------------------------------------- /portals-libraries/src/test/scala/portals/libraries/actor/ActorTestUtils.scala: -------------------------------------------------------------------------------- 1 | package portals.libraries.actor 2 | 3 | import scala.annotation.experimental 4 | import scala.collection.mutable.Queue 5 | import scala.util.Try 6 | 7 | import org.junit.Assert._ 8 | 9 | import portals.libraries.actor.ActorBehaviors 10 | 11 | @experimental 12 | object ActorTestUtils: 13 | class TestBehavior[T](wrappedBehavior: ActorBehavior[T]): 14 | // queue of received messages of the wrapped actor behavior 15 | private val queue: Queue[T] = Queue[T]() 16 | 17 | // enqueue a message to the receive queue 18 | private def enqueue(msg: T): Unit = 19 | queue.enqueue(msg) 20 | 21 | // the behavior which wraps around the submitted behavior, it intercepts all messages 22 | val behavior = ActorBehaviors.InitBehavior[T] { ctx => 23 | var innerBehavior = ActorBehaviors.prepareBehavior(wrappedBehavior, ctx) 24 | ActorBehaviors.ReceiveActorBehavior[T] { ctx => msg => 25 | enqueue(msg) 26 | innerBehavior match 27 | case ActorBehaviors.ReceiveActorBehavior(f) => 28 | f(ctx)(msg) match 29 | case newInnerBehavior @ ActorBehaviors.ReceiveActorBehavior(f) => 30 | innerBehavior = newInnerBehavior 31 | ActorBehaviors.same 32 | case ActorBehaviors.StoppedBehavior => 33 | ActorBehaviors.stopped 34 | case ActorBehaviors.SameBehavior => 35 | ActorBehaviors.same 36 | case _ => ??? 37 | case _ => ??? 38 | } 39 | } 40 | 41 | // take the first element from the receive queue and dequeue it 42 | def receive(): Option[T] = 43 | Try(queue.dequeue()).toOption 44 | 45 | // receive all messages from the queue, does not dequeue the elements 46 | def receiveAll(): Seq[T] = 47 | queue.toSeq 48 | 49 | // check if the receive queue is empty 50 | def isEmpty(): Boolean = 51 | queue.isEmpty 52 | 53 | // true if the receive queue contains the given element 54 | def contains(element: T): Boolean = 55 | queue.contains(element) 56 | 57 | // empty the receive queue 58 | def clear(): Unit = 59 | queue.clear() 60 | 61 | // receive and assert that the received element is the same as the given element 62 | def receiveAssert(event: T): this.type = { assertEquals(Some(event), receive()); this } 63 | 64 | // assert that the receive queue is empty 65 | def isEmptyAssert(): this.type = { assertEquals(true, isEmpty()); this } 66 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/flowbuilder/flatMap.js.out: -------------------------------------------------------------------------------- 1 | 0 2 | 0 3 | 1 4 | 1 5 | 2 6 | 4 7 | 3 8 | 9 9 | 4 10 | 16 11 | 5 12 | 25 13 | 6 14 | 36 15 | 7 16 | 49 17 | 8 18 | 64 19 | 9 20 | 81 21 | 10 22 | 100 23 | 11 24 | 121 25 | 12 26 | 144 27 | 13 28 | 169 29 | 14 30 | 196 31 | 15 32 | 225 33 | 16 34 | 256 35 | 17 36 | 289 37 | 18 38 | 324 39 | 19 40 | 361 41 | 20 42 | 400 43 | 21 44 | 441 45 | 22 46 | 484 47 | 23 48 | 529 49 | 24 50 | 576 51 | 25 52 | 625 53 | 26 54 | 676 55 | 27 56 | 729 57 | 28 58 | 784 59 | 29 60 | 841 61 | 30 62 | 900 63 | 31 64 | 961 65 | 32 66 | 1024 67 | 33 68 | 1089 69 | 34 70 | 1156 71 | 35 72 | 1225 73 | 36 74 | 1296 75 | 37 76 | 1369 77 | 38 78 | 1444 79 | 39 80 | 1521 81 | 40 82 | 1600 83 | 41 84 | 1681 85 | 42 86 | 1764 87 | 43 88 | 1849 89 | 44 90 | 1936 91 | 45 92 | 2025 93 | 46 94 | 2116 95 | 47 96 | 2209 97 | 48 98 | 2304 99 | 49 100 | 2401 101 | 50 102 | 2500 103 | 51 104 | 2601 105 | 52 106 | 2704 107 | 53 108 | 2809 109 | 54 110 | 2916 111 | 55 112 | 3025 113 | 56 114 | 3136 115 | 57 116 | 3249 117 | 58 118 | 3364 119 | 59 120 | 3481 121 | 60 122 | 3600 123 | 61 124 | 3721 125 | 62 126 | 3844 127 | 63 128 | 3969 129 | 64 130 | 4096 131 | 65 132 | 4225 133 | 66 134 | 4356 135 | 67 136 | 4489 137 | 68 138 | 4624 139 | 69 140 | 4761 141 | 70 142 | 4900 143 | 71 144 | 5041 145 | 72 146 | 5184 147 | 73 148 | 5329 149 | 74 150 | 5476 151 | 75 152 | 5625 153 | 76 154 | 5776 155 | 77 156 | 5929 157 | 78 158 | 6084 159 | 79 160 | 6241 161 | 80 162 | 6400 163 | 81 164 | 6561 165 | 82 166 | 6724 167 | 83 168 | 6889 169 | 84 170 | 7056 171 | 85 172 | 7225 173 | 86 174 | 7396 175 | 87 176 | 7569 177 | 88 178 | 7744 179 | 89 180 | 7921 181 | 90 182 | 8100 183 | 91 184 | 8281 185 | 92 186 | 8464 187 | 93 188 | 8649 189 | 94 190 | 8836 191 | 95 192 | 9025 193 | 96 194 | 9216 195 | 97 196 | 9409 197 | 98 198 | 9604 199 | 99 200 | 9801 201 | 100 202 | 10000 203 | 101 204 | 10201 205 | 102 206 | 10404 207 | 103 208 | 10609 209 | 104 210 | 10816 211 | 105 212 | 11025 213 | 106 214 | 11236 215 | 107 216 | 11449 217 | 108 218 | 11664 219 | 109 220 | 11881 221 | 110 222 | 12100 223 | 111 224 | 12321 225 | 112 226 | 12544 227 | 113 228 | 12769 229 | 114 230 | 12996 231 | 115 232 | 13225 233 | 116 234 | 13456 235 | 117 236 | 13689 237 | 118 238 | 13924 239 | 119 240 | 14161 241 | 120 242 | 14400 243 | 121 244 | 14641 245 | 122 246 | 14884 247 | 123 248 | 15129 249 | 124 250 | 15376 251 | 125 252 | 15625 253 | 126 254 | 15876 255 | 127 256 | 16129 257 | -------------------------------------------------------------------------------- /portals-portalsjs/js/src/main/resources/examples/basic/registry.js.out: -------------------------------------------------------------------------------- 1 | from app2: 0 2 | from app2: 1 3 | from app2: 2 4 | from app2: 3 5 | from app2: 4 6 | from app2: 5 7 | from app2: 6 8 | from app2: 7 9 | from app2: 8 10 | from app2: 9 11 | from app2: 10 12 | from app2: 11 13 | from app2: 12 14 | from app2: 13 15 | from app2: 14 16 | from app2: 15 17 | from app2: 16 18 | from app2: 17 19 | from app2: 18 20 | from app2: 19 21 | from app2: 20 22 | from app2: 21 23 | from app2: 22 24 | from app2: 23 25 | from app2: 24 26 | from app2: 25 27 | from app2: 26 28 | from app2: 27 29 | from app2: 28 30 | from app2: 29 31 | from app2: 30 32 | from app2: 31 33 | from app2: 32 34 | from app2: 33 35 | from app2: 34 36 | from app2: 35 37 | from app2: 36 38 | from app2: 37 39 | from app2: 38 40 | from app2: 39 41 | from app2: 40 42 | from app2: 41 43 | from app2: 42 44 | from app2: 43 45 | from app2: 44 46 | from app2: 45 47 | from app2: 46 48 | from app2: 47 49 | from app2: 48 50 | from app2: 49 51 | from app2: 50 52 | from app2: 51 53 | from app2: 52 54 | from app2: 53 55 | from app2: 54 56 | from app2: 55 57 | from app2: 56 58 | from app2: 57 59 | from app2: 58 60 | from app2: 59 61 | from app2: 60 62 | from app2: 61 63 | from app2: 62 64 | from app2: 63 65 | from app2: 64 66 | from app2: 65 67 | from app2: 66 68 | from app2: 67 69 | from app2: 68 70 | from app2: 69 71 | from app2: 70 72 | from app2: 71 73 | from app2: 72 74 | from app2: 73 75 | from app2: 74 76 | from app2: 75 77 | from app2: 76 78 | from app2: 77 79 | from app2: 78 80 | from app2: 79 81 | from app2: 80 82 | from app2: 81 83 | from app2: 82 84 | from app2: 83 85 | from app2: 84 86 | from app2: 85 87 | from app2: 86 88 | from app2: 87 89 | from app2: 88 90 | from app2: 89 91 | from app2: 90 92 | from app2: 91 93 | from app2: 92 94 | from app2: 93 95 | from app2: 94 96 | from app2: 95 97 | from app2: 96 98 | from app2: 97 99 | from app2: 98 100 | from app2: 99 101 | from app2: 100 102 | from app2: 101 103 | from app2: 102 104 | from app2: 103 105 | from app2: 104 106 | from app2: 105 107 | from app2: 106 108 | from app2: 107 109 | from app2: 108 110 | from app2: 109 111 | from app2: 110 112 | from app2: 111 113 | from app2: 112 114 | from app2: 113 115 | from app2: 114 116 | from app2: 115 117 | from app2: 116 118 | from app2: 117 119 | from app2: 118 120 | from app2: 119 121 | from app2: 120 122 | from app2: 121 123 | from app2: 122 124 | from app2: 123 125 | from app2: 124 126 | from app2: 125 127 | from app2: 126 128 | from app2: 127 129 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/runtime/interpreter/processors/InterpreterPortal.scala: -------------------------------------------------------------------------------- 1 | package portals.runtime.interpreter.processors 2 | 3 | import scala.collection.mutable.ArrayDeque 4 | import scala.util.Try 5 | 6 | import portals.application.AtomicPortal 7 | import portals.runtime.BatchedEvents.* 8 | import portals.runtime.SuspendingRuntime.Completed 9 | import portals.runtime.WrappedEvents.* 10 | import portals.util.Key 11 | 12 | /** Internal API. The TestPortal connects the askers with the replier. The 13 | * replier needs to be set dynamically by the replying workflow. 14 | */ 15 | private[portals] class InterpreterPortal( 16 | portal: AtomicPortal[_, _], 17 | var replier: String = null, 18 | var replierTask: String = null 19 | ): 20 | // A single queue is used both for the TestAskBatch and TestRepBatch events. 21 | // This is sufficient, as the events have information with respect to the asker and replier. 22 | private val queue: ArrayDeque[EventBatch] = ArrayDeque.empty 23 | 24 | /** Set the key correctly if a key function exists. */ 25 | private[portals] def setKeyAndMeta(ta: EventBatch): EventBatch = 26 | if !portal.key.isDefined then 27 | ta match 28 | case AskBatch(meta, list) => 29 | AskBatch( 30 | meta.copy(replyingTask = Some(replierTask), replyingWF = Some(replier)), 31 | list 32 | ) 33 | case _ => ta 34 | else 35 | ta match 36 | case AskBatch(meta, list) => 37 | AskBatch( 38 | meta.copy(replyingTask = Some(replierTask), replyingWF = Some(replier)), 39 | list.map { 40 | case Ask(key, meta, event) => 41 | portals.runtime.WrappedEvents.Ask(Key(portal.key.get(event.asInstanceOf)), meta, event) 42 | case x @ _ => x 43 | } 44 | ) 45 | case ReplyBatch(meta, list) => 46 | ReplyBatch( 47 | meta, 48 | list.map { 49 | case Reply(key, meta, event) => 50 | portals.runtime.WrappedEvents.Reply(meta.askingKey, meta, event) 51 | case x @ _ => x 52 | } 53 | ) 54 | case _ => ??? 55 | 56 | /** Enqueue an atom. */ 57 | def enqueue(tp: EventBatch) = tp match { 58 | case AskBatch(meta, list) => 59 | queue.append(setKeyAndMeta(tp)) 60 | case ReplyBatch(_, _) => 61 | queue.append(setKeyAndMeta(tp)) 62 | case _ => ??? // not allowed 63 | } 64 | 65 | def isEmpty: Boolean = queue.isEmpty 66 | 67 | def dequeue(): Option[EventBatch] = Some(queue.remove(0)) 68 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/api/builder/PortalBuilder.scala: -------------------------------------------------------------------------------- 1 | package portals.api.builder 2 | 3 | import portals.application.* 4 | 5 | /** Builder for portals. 6 | * 7 | * The PortalBuilder is used to build portals. A workflow can connect to a 8 | * portal via either an `asker` task or a `replier` task. 9 | * 10 | * Accessed from the application builder via `builder.portals`. 11 | * 12 | * @example 13 | * {{{builder.portals.portal[String, Int]("portalName")}}} 14 | * 15 | * @example 16 | * {{{builder.portals.portal[String, Int]("portalName", keyFrom)}}} 17 | */ 18 | trait PortalBuilder: 19 | /** Build a portal with `name` and no key-extractor. 20 | * 21 | * @example 22 | * {{{builder.portals.portal[String, Int]("portalName")}}} 23 | * 24 | * @param name 25 | * the name of the portal 26 | * @tparam T 27 | * the type of the request 28 | * @tparam R 29 | * the type of the response 30 | * @return 31 | * the portal reference 32 | */ 33 | def portal[T, R](name: String): AtomicPortalRef[T, R] 34 | 35 | /** Build a portal with `name` and a key-extractor `f`. 36 | * 37 | * @example 38 | * {{{builder.portals.portal[String, Int]("portalName", keyFrom)}}} 39 | * 40 | * @param name 41 | * the name of the portal 42 | * @param f 43 | * the key-extractor function 44 | * @tparam T 45 | * the type of the request 46 | * @tparam R 47 | * the type of the response 48 | * @return 49 | * the portal reference 50 | */ 51 | def portal[T, R](name: String, f: T => Long): AtomicPortalRef[T, R] 52 | 53 | /** Internal API. The portal builder. */ 54 | object PortalBuilder: 55 | /** Internal API. Create a PortalBuilder using the application context. */ 56 | def apply(name: String)(using bctx: ApplicationBuilderContext): PortalBuilder = 57 | val _name = bctx.name_or_id(name) 58 | new PortalBuilderImpl(_name) 59 | 60 | /** Internal API. Implementation of the PortalBuilder. */ 61 | class PortalBuilderImpl(name: String)(using bctx: ApplicationBuilderContext) extends PortalBuilder: 62 | override def portal[T, R](name: String): AtomicPortalRef[T, R] = 63 | val path = bctx.app.path + "/portals/" + name 64 | val portal = AtomicPortal[T, R](path) 65 | bctx.addToContext(portal) 66 | AtomicPortalRef[T, R](portal) 67 | 68 | override def portal[T, R](name: String, f: T => Long): AtomicPortalRef[T, R] = 69 | val path = bctx.app.path + "/portals/" + name 70 | val portal = AtomicPortal[T, R](path, Some(f)) 71 | bctx.addToContext(portal) 72 | AtomicPortalRef[T, R](portal) 73 | -------------------------------------------------------------------------------- /portals-core/shared/src/main/scala/portals/application/task/OutputCollectorImpl.scala: -------------------------------------------------------------------------------- 1 | package portals.application.task 2 | 3 | import scala.collection.mutable.ArrayDeque 4 | 5 | import portals.runtime.WrappedEvents.* 6 | import portals.util.Future 7 | import portals.util.Key 8 | import portals.util.Logger 9 | 10 | /** Internal API. OutputCollector to collect submitted events as side effects. 11 | * Works for all kinds of tasks. 12 | */ 13 | private[portals] class OutputCollectorImpl[T, U, Req, Rep] extends OutputCollector[T, U, Req, Rep]: 14 | ////////////////////////////////////////////////////////////////////////////// 15 | // Task 16 | ////////////////////////////////////////////////////////////////////////////// 17 | private var _output = ArrayDeque[WrappedEvent[U]]() 18 | override def submit(event: WrappedEvent[U]): Unit = _output.addOne(event) 19 | def getOutput(): List[WrappedEvent[U]] = _output.toList 20 | def removeAll(): Seq[WrappedEvent[U]] = _output.removeAll() // clears and returns 21 | def clear(): Unit = _output.clear() 22 | 23 | ////////////////////////////////////////////////////////////////////////////// 24 | // Asker Task 25 | ////////////////////////////////////////////////////////////////////////////// 26 | private var _asks = ArrayDeque[Ask[Req]]() 27 | override def ask( 28 | portal: String, 29 | askingTask: String, 30 | req: Req, 31 | key: Key, // TODO: this is confusing exactly which key is meant here, askingKey or replyingKey 32 | id: Int, 33 | askingWF: String, 34 | ): Unit = 35 | _asks.addOne(Ask(key, PortalMeta(portal, askingTask, key, id, askingWF), req)) 36 | def getAskOutput(): List[Ask[Req]] = _asks.toList 37 | def removeAllAsks(): Seq[Ask[Req]] = _asks.removeAll() // clears and returns 38 | def clearAsks(): Unit = _asks.clear() 39 | 40 | ////////////////////////////////////////////////////////////////////////////// 41 | // Replier Task 42 | ////////////////////////////////////////////////////////////////////////////// 43 | private var _reps = ArrayDeque[Reply[Rep]]() 44 | override def reply( 45 | r: Rep, 46 | portal: String, 47 | askingTask: String, 48 | key: Key, // TODO: this is confusing exactly which key is meant here, askingKey or replyingKey 49 | id: Int, 50 | askingWF: String, 51 | ): Unit = _reps.addOne(Reply(key, PortalMeta(portal, askingTask, key, id, askingWF), r)) 52 | def getRepOutput(): List[Reply[Rep]] = _reps.toList 53 | def removeAllReps(): Seq[Reply[Rep]] = _reps.removeAll() // clears and returns 54 | def clearReps(): Unit = _reps.clear() 55 | 56 | end OutputCollectorImpl // class 57 | --------------------------------------------------------------------------------