├── .gitignore
├── .signed
├── .travis.yml
├── LICENSE
├── NOTICE
├── README.md
├── pom.xml
└── src
├── main
├── resources
│ ├── reference.conf
│ └── web-data
│ │ ├── css
│ │ ├── introjs-customizations.css
│ │ └── main.css
│ │ ├── html
│ │ ├── 404.html
│ │ └── main.html
│ │ ├── img
│ │ ├── favicon.ico
│ │ ├── loading.gif
│ │ ├── sc.png
│ │ ├── select.png
│ │ ├── sprites.svg
│ │ └── sprites_small.svg
│ │ └── js
│ │ ├── breakconditions.js
│ │ ├── configuration.js
│ │ ├── graph-drawing.js
│ │ ├── graph.js
│ │ ├── intro.js
│ │ ├── layout.js
│ │ ├── linechart.js
│ │ ├── main.js
│ │ ├── resources-charts.js
│ │ ├── resources-configuration.js
│ │ ├── resources-log.js
│ │ ├── resources.js
│ │ ├── scc.js
│ │ ├── state.js
│ │ ├── strings.js
│ │ └── util.js
└── scala
│ └── com
│ └── signalcollect
│ ├── AbstractVertex.scala
│ ├── AggregationOperations.scala
│ ├── DataFlowVertex.scala
│ ├── DataGraphVertex.scala
│ ├── DefaultEdge.scala
│ ├── DefaultGraph.scala
│ ├── Edge.scala
│ ├── ExecutionConfiguration.scala
│ ├── ExecutionInformation.scala
│ ├── Graph.scala
│ ├── GraphBuilder.scala
│ ├── GraphEditor.scala
│ ├── MemoryEfficientDataFlowVertex.scala
│ ├── MemoryEfficientDataGraphVertex.scala
│ ├── OnlySignalOnChangeEdge.scala
│ ├── OptionalSignalEdge.scala
│ ├── ProcessingVertex.scala
│ ├── ResetStateAfterSignaling.scala
│ ├── StateForwarderEdge.scala
│ ├── StressTest.scala
│ ├── SumOfOutWeights.scala
│ ├── Timeable.scala
│ ├── Vertex.scala
│ ├── configuration
│ ├── Akka.scala
│ ├── ExecutionMode.scala
│ ├── GraphConfiguration.scala
│ ├── KryoInit.scala
│ └── TerminationReason.scala
│ ├── console
│ ├── AggregationOperation.scala
│ ├── ConsoleLogger.scala
│ ├── ConsoleServer.scala
│ └── DataProvider.scala
│ ├── coordinator
│ ├── DefaultCoordinator.scala
│ └── DefaultWorkerApi.scala
│ ├── examples
│ ├── ChineseWhispersClustering.scala
│ ├── ClusteringCoefficient.scala
│ ├── CompanyValuation.scala
│ ├── EfficientPageRank.scala
│ ├── EfficientSssp.scala
│ ├── GameOfLife.scala
│ ├── Hamiltonian.scala
│ ├── LodNeighbourhoodPageRank.scala
│ ├── PageRank.scala
│ ├── PathQuery.scala
│ ├── SchellingSegregation.scala
│ ├── Sssp.scala
│ ├── Sudoku.scala
│ ├── VertexColoring.scala
│ └── WebCrawler.scala
│ ├── factory
│ ├── handler
│ │ └── DefaultHandlerFactory.scala
│ ├── mapper
│ │ └── DefaultMapperFactory.scala
│ ├── messagebus
│ │ └── AkkaMessageBusFactory.scala
│ ├── scheduler
│ │ ├── LowLatency.scala
│ │ └── Throughput.scala
│ ├── storage
│ │ ├── JavaMapStorage.scala
│ │ ├── MemoryEfficientStorage.scala
│ │ └── MixedStorage.scala
│ ├── worker
│ │ └── Akka.scala
│ └── workerapi
│ │ └── WorkerApiFactory.scala
│ ├── interfaces
│ ├── AggregationOperation.scala
│ ├── Coordinator.scala
│ ├── Factory.scala
│ ├── Handlers.scala
│ ├── Logger.scala
│ ├── MessageBus.scala
│ ├── Messages.scala
│ ├── Node.scala
│ ├── NodeActor.scala
│ ├── Scheduler.scala
│ ├── Storage.scala
│ ├── VertexToWorkerMapper.scala
│ ├── Worker.scala
│ └── WorkerApi.scala
│ ├── loading
│ ├── AdjacencyListLoader.scala
│ └── Loading.scala
│ ├── messaging
│ ├── AbstractMessageBus.scala
│ ├── AkkaProxy.scala
│ ├── BulkMessageBus.scala
│ ├── DefaultMessageBus.scala
│ ├── DefaultVertexToWorkerMapper.scala
│ └── IntIdDoubleSignalMessageBus.scala
│ ├── node
│ ├── BinaryTreeIdleDetector.scala
│ ├── DefaultNodeActor.scala
│ └── WorkersOnNodeIdleDetector.scala
│ ├── nodeprovisioning
│ ├── NodeProvisioner.scala
│ └── cluster
│ │ ├── ClusterNodeEntryPointTemplate.scala
│ │ ├── ClusterNodeProvisioner.scala
│ │ ├── ClusterNodeProvisionerActor.scala
│ │ └── ClusterNodeShutdownHandlerActor.scala
│ ├── scheduler
│ ├── LowLatencyScheduler.scala
│ └── ThroughputScheduler.scala
│ ├── serialization
│ └── DefaultSerializer.scala
│ ├── storage
│ ├── JavaMapVertexStorage.scala
│ ├── JavaVertexMap.scala
│ ├── MixedVertexStorage.scala
│ ├── VertexMap.scala
│ └── VertexMapStorage.scala
│ ├── util
│ ├── AkkaRemoteAddress.scala
│ ├── AkkaUtil.scala
│ ├── BitSet.scala
│ ├── FastInsertIntSet.scala
│ ├── FileReader.scala
│ ├── HashSet.scala
│ ├── IntDoubleHashMap.scala
│ ├── IntHashMap.scala
│ ├── IntHashSet.scala
│ ├── IntIntHashMap.scala
│ ├── IntLongHashMap.scala
│ ├── IntSet.scala
│ ├── IntValueHashMap.scala
│ ├── Ints.scala
│ ├── IteratorConcatenator.scala
│ ├── MemoryEfficientSplayIntSet.scala
│ ├── RandomString.scala
│ ├── SearchableIntSet.scala
│ ├── SplayIntSet.scala
│ └── Verifier.scala
│ └── worker
│ ├── AkkaWorker.scala
│ ├── WorkerGraphEditor.scala
│ ├── WorkerImplementation.scala
│ └── WorkerOperationCounters.scala
└── test
├── resources
└── com
│ └── signalcollect
│ └── loading
│ ├── adjacency-list-format
│ ├── ascii-ints.txt
│ ├── ints-no-newline-at-end.txt
│ └── notredame2
└── scala
└── com
└── signalcollect
├── ActorSystemSpec.scala
├── AkkaProxySpec.scala
├── BeforeRemovalSpec.scala
├── GraphModificationSpec.scala
├── GraphResetSpec.scala
├── IntegrationSpec.scala
├── LatencySpec.scala
├── NonExistentVertexHandlerSpec.scala
├── SerializationIntegrationSpec.scala
├── TestConfig.scala
├── TestGraph.scala
├── VertexSpec.scala
├── console
└── ConsoleServerSpec.scala
├── features
├── AggregationOperationsSpec.scala
├── BulkSignalingSpec.scala
├── ComputationTerminationSpec.scala
├── ErrorSpec.scala
├── GraphLoadingSpec.scala
├── MapperSpec.scala
├── MultipleGraphs.scala
├── MultipleVertexAdditionsSpec.scala
├── SnapshotSpec.scala
└── StoreClosedOnShutdownSpec.scala
├── loading
├── AdjacencyListLoaderSpec.scala
└── LoadingSpec.scala
├── serialization
└── SerializerSpec.scala
├── storage
├── VertexMapSpec.scala
└── VertexStorageSpec.scala
└── util
├── BasicBitSetSpec.scala
├── Benchmark.scala
├── BitSetSpec.scala
├── FastInsertIntSetSpec.scala
├── FileReaderSpec.scala
├── HashSetSpec.scala
├── IntLongHashMapSpec.scala
├── IntsSpec.scala
├── IteratorConcatenatorSpec.scala
├── SplayBench.scala
└── SplayIntSetSpec.scala
/.gitignore:
--------------------------------------------------------------------------------
1 | *.snapshot
2 | !/.gitignore
3 | log_messages.txt
4 | .classpath
5 | .project
6 | .cache*
7 | .settings/
8 | .worksheet/
9 | target/
10 |
--------------------------------------------------------------------------------
/.signed:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/uzh/signal-collect/175be65ee64e67c791b2fb83083852317ffcf3af/.signed
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: scala
2 | jdk:
3 | - oraclejdk8
4 | scala:
5 | - 2.11.7
6 | before_script:
7 | - "export JAVA_OPTS=-Xmx2048m"
8 | sudo: false
9 |
--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
1 | This product includes software developed by the University of Zurich (http://www.uzh.ch) and by iHealth Technologies (http://www.ihealthtechnologies.com).
2 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Signal/Collect
2 | ==============
3 |
4 | Signal/Collect is a framework for computations on large graphs. The model allows to concisely express many iterated and data-flow algorithms, while the framework parallelizes and distributes the computation.
5 |
6 | How to develop in Eclipse
7 | -------------------------
8 | Install the [Typesafe IDE for Scala 2.11](http://scala-ide.org/download/sdk.html).
9 |
10 | Ensure that Eclipse uses a Java 8 library and JVM: Preferences → Java → Installed JREs → JRE/JDK 8 should be installed and selected.
11 |
12 | Import the project into Eclipse: File → Import... → Maven → Existing Maven Projects → select "signal-collect" folder
13 |
14 | Thanks a lot to
15 | ---------------
16 | * [University of Zurich](http://www.ifi.uzh.ch/ddis.html) and the [Hasler Foundation](http://www.haslerstiftung.ch/en/home) have generously funded the research on graph processing and the development of Signal/Collect.
17 | * GitHub helps us by hosting our [code repositories](https://github.com/uzh/signal-collect).
18 | * Travis.CI offers us very convenient [continuous integration](https://travis-ci.org/uzh/signal-collect).
19 | * Codacy gives us automated [code reviews](https://www.codacy.com/public/uzh/signalcollect).
20 |
--------------------------------------------------------------------------------
/src/main/resources/web-data/css/introjs-customizations.css:
--------------------------------------------------------------------------------
1 | /**
2 | * @author Silvan Troxler
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed below the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed below the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations below the License.
17 | */
18 |
19 | .introjs-overlay {
20 | opacity: 0.2 !important;
21 | }
22 |
23 | .introjs-helperLayer {
24 | position: absolute;
25 | z-index: 9999998;
26 | background-color: rgba(224,27,27,0.3);
27 | border: 3px solid red;
28 | }
29 |
30 | .introjs-tooltip {
31 | margin-left:10px;
32 | }
33 |
34 | .introjs-helperNumberLayer {
35 | display: none;
36 | }
--------------------------------------------------------------------------------
/src/main/resources/web-data/html/404.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
21 |
22 |
23 | 404 - Resource not found
24 |
25 |
26 | 404 - Resource not found
27 | The Web site you seek
28 | cannot be located but
29 | endless others exist
30 | Go to Signal/Collect Console
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/src/main/resources/web-data/img/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/uzh/signal-collect/175be65ee64e67c791b2fb83083852317ffcf3af/src/main/resources/web-data/img/favicon.ico
--------------------------------------------------------------------------------
/src/main/resources/web-data/img/loading.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/uzh/signal-collect/175be65ee64e67c791b2fb83083852317ffcf3af/src/main/resources/web-data/img/loading.gif
--------------------------------------------------------------------------------
/src/main/resources/web-data/img/sc.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/uzh/signal-collect/175be65ee64e67c791b2fb83083852317ffcf3af/src/main/resources/web-data/img/sc.png
--------------------------------------------------------------------------------
/src/main/resources/web-data/img/select.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/uzh/signal-collect/175be65ee64e67c791b2fb83083852317ffcf3af/src/main/resources/web-data/img/select.png
--------------------------------------------------------------------------------
/src/main/resources/web-data/js/resources-configuration.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @author Silvan Troxler
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed below the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed below the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations below the License.
17 | */
18 |
19 | /**
20 | * The Configuration module retrieves data about different configuration and
21 | * parameter statistics from the console server. The module shows the JVM
22 | * parameters, the computation statistics as well as the graph statistics.
23 | * @constructor
24 | */
25 | scc.modules.Configuration = function() {
26 | this.requires = ["configuration"];
27 |
28 | /**
29 | * Function that is called by the main module when a new WebSocket connection
30 | * is established. Requests data from the ConfigurationProvider.
31 | */
32 | this.onopen = function () {
33 | scc.order({"requestor": "Resources", "provider": "configuration"});
34 | }
35 |
36 | /**
37 | * Function that is called by the main module when a WebSocket error is
38 | * encountered. Does nothing.
39 | * @param {Event} e - The event that triggered the call.
40 | */
41 | this.onerror = function(e) { }
42 |
43 | /**
44 | * Function that is called by the main module when a requested piece of data
45 | * is not (yet) available from the server. Does nothing.
46 | */
47 | this.notready = function() { }
48 |
49 | /**
50 | * Function that is called by the main module when a new WebSocket connection
51 | * breaks down. Does nothing.
52 | */
53 | this.onclose = function() { }
54 |
55 | /**
56 | * Function that is called by the main module when a message is received from
57 | * the WebSocket. It populates the given information about the JVM,
58 | * computation and the graph in the proper elements.
59 | * @param {object} msg - The message object received from the server.
60 | */
61 | this.onmessage = function(msg) {
62 | if (msg.executionConfiguration !== "unknown") {
63 | $.each(msg.executionConfiguration, function(k,v) {
64 | $("#resStat" + k).html(v);
65 | });
66 | }
67 | var ul = $("#infrastructureStatBox ul").html('');
68 | $.each(msg.systemProperties, function(index) {
69 | $.each(msg.systemProperties[index], function(k, v) {
70 | if (scc.conf.resources.hideInfrastructureItems.indexOf(k) === -1) {
71 | ul.append('' + k + ': ' + v + '');
72 | }
73 | });
74 | });
75 | ul = $("#graphStatBox ul").html('');
76 | $.each(msg.graphConfiguration, function(k, v) {
77 | if (k === "consoleHttpPort" && v[0] === -1) {
78 | v[0] = location.port;
79 | }
80 | ul.append('' + k + ': ' + v[0] + '');
81 | });
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/src/main/resources/web-data/js/scc.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @author Carol Alexandru
3 | * @author Silvan Troxler
4 | *
5 | * Copyright 2013 University of Zurich
6 | *
7 | * Licensed below the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed below the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations below the License.
18 | */
19 |
20 | /**
21 | * The project container variable scc which appears in the global namespace.
22 | * @namespace
23 | * @property {object} modules - The module classes that can be instantiated.
24 | * @property {object} consumers - The instantiated modules
25 | * @property {object} defaults - The default settings of each module
26 | * @property {object} orders - The pending orders, zero or one for each consumer
27 | * @property {object} callbacks - Zero or one callback for each consumer to be
28 | * called once a reply has been received from the server
29 | */
30 | var scc = {"modules": {}, "consumers": {}, "defaults": {}, "orders": {},
31 | "callbacks": {}};
32 |
33 | /**
34 | * Container for needed libraries and variables.
35 | * @property {object} graph - Container for the graph libraries
36 | * @property {object} resources - Container for the resources libraries
37 | */
38 | scc.lib = {"graph": {}, "resources": {}};
39 |
40 | /**
41 | * Container for the configurable parameters.
42 | * @property {object} graph - Container for the configurable graph parameters
43 | * @property {object} resources - Container for the configurable resources parameters
44 | */
45 | scc.conf = {"graph": {}, "resources": {}};
46 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/DataFlowVertex.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | /**
23 | * Vertex implementation that collects all the signals that have arrived since the last
24 | * time this vertex has collected. Users of the framework extend this class to implement
25 | * a specific algorithm by defining a `collect` function.
26 | *
27 | * @note The `collect` function receives all signals that arrived at this vertex but have not
28 | * been collected yet as a parameter.
29 | *
30 | * @param id Unique vertex id.
31 | * @param state The initial state of the vertex.
32 | * @param resetState The state will be set to `resetState` after signaling.
33 | *
34 | * @author Philip Stutz
35 | */
36 | abstract class DataFlowVertex[Id, State](
37 | val id: Id,
38 | var state: State)
39 | extends AbstractVertex[Id, State] {
40 |
41 | type Signal
42 |
43 | def setState(s: State) {
44 | state = s
45 | }
46 |
47 | def deliverSignalWithSourceId(signal: Any, sourceId: Any, graphEditor: GraphEditor[Any, Any]): Boolean = {
48 | deliverSignalWithoutSourceId(signal, graphEditor)
49 | }
50 |
51 | def deliverSignalWithoutSourceId(signal: Any, graphEditor: GraphEditor[Any, Any]): Boolean = {
52 | setState(collect(signal.asInstanceOf[Signal]))
53 | true
54 | }
55 |
56 | /**
57 | * The abstract `collect` function is algorithm specific and calculates the new vertex state.
58 | *
59 | * @param signal a signal that was received by this vertex and has not yet been collected.
60 | *
61 | * @return The new vertex state.
62 | *
63 | * @note Beware of modifying and returning a referenced object,
64 | * default signal scoring and termination detection fail in this case.
65 | */
66 | def collect(signal: Signal): State
67 |
68 | /**
69 | * Function that gets called by the framework whenever this vertex is supposed to collect new signals.
70 | *
71 | * @param graphEditor can be used by this vertex to interact with the graph.
72 | */
73 | override def executeCollectOperation(graphEditor: GraphEditor[Any, Any]) {
74 | super.executeCollectOperation(graphEditor)
75 | }
76 |
77 | /**
78 | * This method is used by the framework in order to decide if the vertex' collect operation
79 | * should be executed.
80 | *
81 | * @return the score value. The meaning of this value depends on the thresholds set in the framework.
82 | */
83 | def scoreCollect: Double = {
84 | if (edgesModifiedSinceCollectOperation) {
85 | 1.0
86 | } else {
87 | 0.0
88 | }
89 | }
90 |
91 | }
92 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/DefaultEdge.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import com.signalcollect.interfaces.EdgeId
23 | import akka.event.Logging
24 | import com.signalcollect.interfaces.SignalMessageWithSourceId
25 |
26 | /**
27 | * Edge that connects a source vertex with a target vertex.
28 | * Users of the framework extend this class to implement a specific algorithm by defining a `signal` function.
29 | *
30 | * @param targetId target vertex id
31 | * @param description an additional description of this edge that would allow to tell apart multiple edges between the source and the target vertex
32 | */
33 | abstract class DefaultEdge[TargetId](val targetId: TargetId) extends Edge[TargetId] {
34 |
35 | /** The type of signals that are sent along this edge. */
36 | type Signal = Any
37 |
38 | var source: Source = _
39 |
40 | /**
41 | * An edge id uniquely identifies an edge in the graph.
42 | */
43 | def id = EdgeId(sourceId, targetId)
44 |
45 | /** The weight of this edge: 1.0 by default, can be overridden. */
46 | def weight: Double = 1
47 |
48 | /**
49 | * The abstract `signal` function is algorithm specific and is implemented by a user of the framework.
50 | * It calculates the signal that is sent from the source vertex to the target vertex.
51 | *
52 | * @param sourceVertex The source vertex to which this edge is currently attached as an outgoing edge.
53 | *
54 | * @return The signal that will be sent along this edge.
55 | */
56 | def signal: Signal
57 |
58 | /**
59 | * The hash code of the target vertex id is cached to speed up signaling.
60 | */
61 | val cachedTargetIdHashCode = targetId.hashCode
62 |
63 | /**
64 | * Function that gets called by the source vertex whenever this edge is supposed to send a signal.
65 | *
66 | * @param sourceVertex The source vertex of this edge.
67 | *
68 | * @param messageBus an instance of MessageBus which can be used by this edge to interact with the graph.
69 | */
70 | def executeSignalOperation(sourceVertex: Vertex[_, _, _, _], graphEditor: GraphEditor[Any, Any]) {
71 | graphEditor.sendToWorkerForVertexIdHash(SignalMessageWithSourceId(targetId, sourceId, signal), cachedTargetIdHashCode)
72 | }
73 |
74 | /** Called when the edge is attached to a source vertex */
75 | def onAttach(sourceVertex: Vertex[_, _, _, _], graphEditor: GraphEditor[Any, Any]) = {
76 | source = sourceVertex.asInstanceOf[Source]
77 | }
78 |
79 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/Edge.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import com.signalcollect.interfaces.EdgeId
23 |
24 | /**
25 | * This abstract class represents the framework's view of an edge.
26 | *
27 | * @author Philip Stutz
28 | */
29 | abstract class Edge[+TargetId] extends Serializable {
30 |
31 | type Source <: Vertex[_, _, _, _]
32 |
33 | /** An edge id uniquely identifies an edge in the graph. */
34 | def id: EdgeId[_]
35 |
36 | def sourceId: Any = {
37 | if (source != null) {
38 | source.id
39 | } else {
40 | null
41 | }
42 | }
43 | def targetId: TargetId
44 | def source: Source
45 |
46 | /** Called when the edge is attached to a source vertex */
47 | def onAttach(source: Vertex[_, _, _, _], graphEditor: GraphEditor[Any, Any])
48 |
49 | /** The weight of this edge. */
50 | def weight: Double
51 |
52 | /** EdgeClassName(id=`edge id`) */
53 | override def toString = getClass.getSimpleName + "(id=" + id + ")"
54 |
55 | /** The edge hashCode is the hashCode of the id */
56 | override def hashCode = id.hashCode
57 |
58 | /** Two edges are equal if their ids are equal */
59 | override def equals(other: Any): Boolean =
60 | other match {
61 | case e: Edge[_] => e.id == id
62 | case _ => false
63 | }
64 |
65 | /**
66 | * Function that gets called by the source vertex whenever this edge is supposed to send a signal.
67 | *
68 | * @param sourceVertex The source vertex of this edge.
69 | *
70 | * @param messageBus an instance of MessageBus which can be used by this edge to interact with the graph.
71 | */
72 | def executeSignalOperation(sourceVertex: Vertex[_, _, _, _], graphEditor: GraphEditor[Any, Any])
73 |
74 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/ExecutionInformation.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import java.util.concurrent.TimeUnit
23 |
24 | import scala.concurrent.duration.Duration
25 |
26 | import com.signalcollect.configuration.GraphConfiguration
27 | import com.signalcollect.configuration.TerminationReason
28 | import com.signalcollect.interfaces.WorkerStatistics
29 |
30 | /**
31 | * An instance of ExecutionInformation reports information such as execution statistics
32 | * and configuration parameters related to an execution of a computation on a graph.
33 | *
34 | * @param config The graph configuration of the graph that executed a computation.
35 | * @param parameters The execution configuration for this particular execution.
36 | * @param executionStatistics Statistics about this execution, such as the computation time,
37 | * @param aggregatedWorkerStatistics Aggregated statistics over all the workers.
38 | * @param individualWorkerStatistics A list of statistics for all workers individually.
39 | *
40 | * @author Philip Stutz
41 | */
42 | case class ExecutionInformation[Id, Signal](
43 | config: GraphConfiguration[Id, Signal],
44 | numberOfWorkers: Int,
45 | parameters: ExecutionConfiguration[Id, Signal],
46 | executionStatistics: ExecutionStatistics,
47 | aggregatedWorkerStatistics: WorkerStatistics,
48 | individualWorkerStatistics: List[WorkerStatistics]) {
49 |
50 | override def toString: String = {
51 | "------------------------\n" +
52 | "- Execution Parameters -\n" +
53 | "------------------------\n" +
54 | parameters.toString + "\n" +
55 | "\n--------------\n" +
56 | "- Execution Statistics -\n" +
57 | "--------------\n" +
58 | executionStatistics.toString + "\n" +
59 | aggregatedWorkerStatistics.toSimpleString + "\n"
60 | }
61 | }
62 |
63 | case class ExecutionStatistics(
64 | var signalSteps: Long = 0,
65 | var collectSteps: Long = 0,
66 | var computationTime: Duration = Duration.create(0, TimeUnit.MILLISECONDS),
67 | var totalExecutionTime: Duration = Duration.create(0, TimeUnit.MILLISECONDS), // should approximately equal computation time + idle waiting + garbage collection
68 | var jvmCpuTime: Duration = Duration.create(0, TimeUnit.MILLISECONDS),
69 | var terminationReason: TerminationReason.Value = TerminationReason.Converged) {
70 |
71 | override def toString: String = {
72 | "# signal steps\t\t" + signalSteps + "\n" +
73 | "# collect steps\t\t" + collectSteps + "\n" +
74 | "Computation time\t\t" + computationTime.toUnit(TimeUnit.MILLISECONDS).toInt + " milliseconds\n" +
75 | "Master JVM CPU time\t" + jvmCpuTime.toUnit(TimeUnit.MILLISECONDS).toInt + " milliseconds\n" +
76 | "Termination reason\t" + terminationReason
77 | }
78 |
79 | }
80 |
81 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/MemoryEfficientDataFlowVertex.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import scala.reflect.ClassTag
23 |
24 | import com.signalcollect.util.MemoryEfficientSplayIntSet
25 | import com.signalcollect.util.SplayIntSet
26 |
27 | /**
28 | * A memory efficient implementation of a data flow vertex
29 | * with an Int id and a memory efficient edge representation
30 | * that can only represent edges without additional attributes
31 | * that point to a vertex with an Int id.
32 | * The signal function is the same for all edges and defined in
33 | * the 'computeSignal' function.
34 | */
35 | abstract class MemoryEfficientDataFlowVertex[State, GraphSignalType: ClassTag](
36 | val id: Int,
37 | var state: State) extends Vertex[Int, State, Int, GraphSignalType] {
38 |
39 | type OutgoingSignalType
40 |
41 | def collect(signal: GraphSignalType): State
42 |
43 | def computeSignal(targetId: Int): GraphSignalType
44 |
45 | var lastSignalState: State = null.asInstanceOf[State]
46 |
47 | def setState(s: State) {
48 | state = s
49 | }
50 |
51 | def targetIds: Traversable[Int] = {
52 | new Traversable[Int] {
53 | def foreach[U](f: Int => U) {
54 | _targetIds.foreach(f(_))
55 | }
56 | }
57 | }
58 |
59 | protected var _targetIds: SplayIntSet = new MemoryEfficientSplayIntSet
60 |
61 | def deliverSignalWithSourceId(signal: GraphSignalType, sourceId: Int, graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = {
62 | deliverSignalWithoutSourceId(signal, graphEditor)
63 | }
64 |
65 | def deliverSignalWithoutSourceId(signal: GraphSignalType, graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = {
66 | setState(collect(signal))
67 | true
68 | }
69 |
70 | /**
71 | * We always collect on delivery.
72 | */
73 | def scoreCollect = 0
74 |
75 | override def executeSignalOperation(graphEditor: GraphEditor[Int, GraphSignalType]) {
76 | _targetIds.foreach { targetId =>
77 | graphEditor.sendSignal(computeSignal(targetId), targetId, id)
78 | }
79 | lastSignalState = state
80 | }
81 |
82 | def edgeCount = _targetIds.size
83 |
84 | override def toString = s"${this.getClass.getSimpleName}(state=$state)"
85 |
86 | def executeCollectOperation(graphEditor: GraphEditor[Int, GraphSignalType]) {
87 | }
88 |
89 | override def addEdge(e: Edge[Int], graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = {
90 | _targetIds.insert(e.targetId.asInstanceOf[Int])
91 | }
92 |
93 | override def removeEdge(targetId: Int, graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = throw new UnsupportedOperationException
94 |
95 | override def removeAllEdges(graphEditor: GraphEditor[Int, GraphSignalType]): Int = throw new UnsupportedOperationException
96 |
97 | def afterInitialization(graphEditor: GraphEditor[Int, GraphSignalType]) = {}
98 |
99 | def beforeRemoval(graphEditor: GraphEditor[Int, GraphSignalType]) = {}
100 | }
101 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/MemoryEfficientDataGraphVertex.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import com.signalcollect.util.SplayIntSet
23 | import com.signalcollect.util.IntHashMap
24 | import scala.reflect.ClassTag
25 | import com.signalcollect.util.MemoryEfficientSplayIntSet
26 |
27 | /**
28 | * A memory efficient implementation of a data graph vertex
29 | * with an Int id and a memory efficient edge representation
30 | * that can only represent edges without additional attributes
31 | * that point to a vertex with an Int id.
32 | * The signal function is the same for all edges and defined in
33 | * the 'computeSignal' function.
34 | */
35 | abstract class MemoryEfficientDataGraphVertex[State, IncomingSignalType: ClassTag, GraphSignalType](
36 | val id: Int,
37 | var state: State) extends Vertex[Int, State, Int, GraphSignalType] {
38 |
39 | def collect: State
40 |
41 | def computeSignal(targetId: Int): GraphSignalType
42 |
43 | var lastSignalState: State = null.asInstanceOf[State]
44 |
45 | def setState(s: State) {
46 | state = s
47 | }
48 |
49 | def targetIds: Traversable[Int] = {
50 | new Traversable[Int] {
51 | def foreach[U](f: Int => U) {
52 | _targetIds.foreach(f(_))
53 | }
54 | }
55 | }
56 |
57 | protected var _targetIds: SplayIntSet = new MemoryEfficientSplayIntSet
58 |
59 | val mostRecentSignalMap = new IntHashMap[IncomingSignalType](1, 0.85f)
60 |
61 | override def deliverSignalWithSourceId(signal: GraphSignalType, sourceId: Int, graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = {
62 | val s = signal.asInstanceOf[IncomingSignalType]
63 | mostRecentSignalMap.put(sourceId, s)
64 | false
65 | }
66 |
67 | override def deliverSignalWithoutSourceId(signal: GraphSignalType, graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = {
68 | throw new Exception(s"A data graph vertex requires the sender ID for signals. Vertex ${this.toString} just received signal ${signal} without a sender ID.")
69 | }
70 |
71 | override def executeSignalOperation(graphEditor: GraphEditor[Int, GraphSignalType]) {
72 | _targetIds.foreach { targetId =>
73 | graphEditor.sendSignal(computeSignal(targetId), targetId, id)
74 | }
75 | lastSignalState = state
76 | }
77 |
78 | def edgeCount = _targetIds.size
79 |
80 | override def toString = s"${this.getClass.getName}(state=$state)"
81 |
82 | def executeCollectOperation(graphEditor: GraphEditor[Int, GraphSignalType]) {
83 | setState(collect)
84 | }
85 |
86 | override def addEdge(e: Edge[Int], graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = {
87 | _targetIds.insert(e.targetId.asInstanceOf[Int])
88 | }
89 |
90 | override def removeEdge(targetId: Int, graphEditor: GraphEditor[Int, GraphSignalType]): Boolean = throw new UnsupportedOperationException
91 |
92 | override def removeAllEdges(graphEditor: GraphEditor[Int, GraphSignalType]): Int = throw new UnsupportedOperationException
93 |
94 | def afterInitialization(graphEditor: GraphEditor[Int, GraphSignalType]) = {}
95 |
96 | def beforeRemoval(graphEditor: GraphEditor[Int, GraphSignalType]) = {}
97 | }
98 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/OnlySignalOnChangeEdge.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import com.signalcollect.interfaces.SignalMessageWithSourceId
23 |
24 | /**
25 | * OnlySignalOnChangeEdge is an edge implementation that only signals
26 | * when the signal has changed compared to the last signal sent.
27 | * For some algorithms this can reduce the number of messages sent.
28 | *
29 | * @param targetId id of this edges's target vertex
30 | * @param description an additional description of this edge that would allow to tell apart multiple edges between the source and the target vertex
31 | *
32 | * @note Beware of modifying and signaling a referenced object, change detection fails in this case.
33 | */
34 | abstract class OnlySignalOnChangeEdge[SourceIdType, TargetIdType](targetId: TargetIdType)
35 | extends DefaultEdge(targetId) {
36 |
37 | /** Last signal sent along this edge */
38 | var lastSignalSent: Option[Signal] = None
39 |
40 | /**
41 | * Function that gets called by the source vertex whenever this edge is supposed to send a signal.
42 | * Compared to the default implementation there is an additional check if the signal has changed
43 | * and no signal is sent if the signal has not changed.
44 | *
45 | * @param sourceVertex The source vertex of this edge.
46 | *
47 | * @param messageBus an instance of MessageBus which can be used by this edge to interact with the graph.
48 | */
49 | override def executeSignalOperation(sourceVertex: Vertex[_, _, _, _], graphEditor: GraphEditor[Any, Any]) {
50 | val newSignal = signal
51 | if (!lastSignalSent.isDefined || !lastSignalSent.get.equals(newSignal)) {
52 | graphEditor.sendToWorkerForVertexIdHash(SignalMessageWithSourceId(targetId, sourceId, newSignal), cachedTargetIdHashCode)
53 | lastSignalSent = Some(newSignal)
54 | }
55 | }
56 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/OptionalSignalEdge.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import com.signalcollect.interfaces.SignalMessageWithSourceId
23 |
24 | /**
25 | * OptionalSignalEdge is an edge implementation that that requires the
26 | * signal function to return an instance of type `Option` and only signals
27 | * if this signal has type `Some`.
28 | *
29 | * @param sourceId id of this edge's source vertex
30 | * @param targetId id of this edges's target vertex
31 | * @param description an additional description of this edge that would allow to tell apart multiple edges between the source and the target vertex
32 | */
33 | abstract class OptionalSignalEdge[TargetIdType](targetId: TargetIdType) extends DefaultEdge(targetId) {
34 |
35 | /**
36 | * Calculates the new signal and potentially sends it. If the return value of the `signal` function is of type `Some`,
37 | * then the encapsulated value is sent. If it is of type `None`, then nothing is sent.
38 | *
39 | * @param sourceVertex The source vertex of this edge.
40 | *
41 | * @param messageBus an instance of MessageBus which can be used by this edge to interact with the graph.
42 | */
43 | override def executeSignalOperation(sourceVertex: Vertex[_, _, _, _], graphEditor: GraphEditor[Any, Any]) {
44 | val optionalSignal = signal.asInstanceOf[Option[_]]
45 | if (optionalSignal.isDefined) {
46 | graphEditor.sendToWorkerForVertexIdHash(SignalMessageWithSourceId(targetId, sourceId, optionalSignal.get), cachedTargetIdHashCode)
47 | }
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/ProcessingVertex.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import scala.collection.mutable.ArrayBuffer
23 |
24 | /**
25 | * Simple data-flow vertex implementation that merely processes messages.
26 | * Edge additions and edge representations are not implemented.
27 | *
28 | * Delivered signals get accumulated inside the state of this vertex. When the vertex signals,
29 | * the `process` function is called for each item in LIFO order (List has better performance
30 | * than ArrayBuffer for this use case).
31 | *
32 | * @param id Unique vertex id.
33 | *
34 | * @author Philip Stutz
35 | */
36 | abstract class ProcessingVertex[Id, SignalType](
37 | val id: Id,
38 | var state: List[SignalType] = List[SignalType]())
39 | extends Vertex[Id, List[SignalType], Any, Any] {
40 |
41 | def process(signal: SignalType, graphEditor: GraphEditor[Any, Any])
42 |
43 | def shouldProcess(signal: SignalType): Boolean = true
44 |
45 | def setState(s: List[SignalType]) {
46 | state = s
47 | }
48 |
49 | def deliverSignal(signal: Any, sourceId: Option[Any], graphEditor: GraphEditor[Any, Any]): Boolean = {
50 | val item = signal.asInstanceOf[SignalType]
51 | if (shouldProcess(item)) {
52 | state = item :: state
53 | }
54 | true
55 | }
56 |
57 | override def executeSignalOperation(graphEditor: GraphEditor[Any, Any]) {
58 | for (item <- state) {
59 | process(item, graphEditor)
60 | }
61 | state = List.empty
62 | }
63 |
64 | override def scoreSignal: Double = if (state.isEmpty) 0 else 1
65 | def scoreCollect = 0 // Because signals are collected upon delivery.
66 | def edgeCount = 0
67 | override def toString = s"${this.getClass.getName}(state=$state)"
68 | def executeCollectOperation(graphEditor: GraphEditor[Any, Any]) {}
69 | def beforeRemoval(graphEditor: GraphEditor[Any, Any]) = {}
70 | override def afterInitialization(graphEditor: GraphEditor[Any, Any]) {}
71 | override def addEdge(e: Edge[Any], graphEditor: GraphEditor[Any, Any]): Boolean = throw new UnsupportedOperationException
72 | override def removeEdge(targetId: Any, graphEditor: GraphEditor[Any, Any]): Boolean = throw new UnsupportedOperationException
73 | override def removeAllEdges(graphEditor: GraphEditor[Any, Any]): Int = 0
74 | }
75 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/ResetStateAfterSignaling.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | trait ResetStateAfterSignaling[Id, State] extends AbstractVertex[Id, State] {
23 |
24 | def resetState: State
25 |
26 | /**
27 | * Delegates to superclass and resets the state to the initial state after signaling.
28 | */
29 | abstract override def executeSignalOperation(graphEditor: GraphEditor[Any, Any]) {
30 | super.executeSignalOperation(graphEditor)
31 | setState(resetState)
32 | }
33 |
34 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/StateForwarderEdge.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | /**
23 | * StateForwarderEdge is an edge implementation that signals
24 | * the state of its source vertex.
25 | *
26 | * @param targetId id of this edges's target vertex
27 | */
28 | class StateForwarderEdge[TargetIdType](targetId: TargetIdType)
29 | extends DefaultEdge(targetId) {
30 |
31 | def signal = source.state.asInstanceOf[Signal]
32 |
33 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/StressTest.scala:
--------------------------------------------------------------------------------
1 | ///*
2 | // * @author Philip Stutz
3 | // *
4 | // * Copyright 2014 University of Zurich
5 | // *
6 | // * Licensed under the Apache License, Version 2.0 (the "License");
7 | // * you may not use this file except in compliance with the License.
8 | // * You may obtain a copy of the License at
9 | // *
10 | // * http://www.apache.org/licenses/LICENSE-2.0
11 | // *
12 | // * Unless required by applicable law or agreed to in writing, software
13 | // * distributed under the License is distributed on an "AS IS" BASIS,
14 | // * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | // * See the License for the specific language governing permissions and
16 | // * limitations under the License.
17 | // *
18 | // */
19 | //
20 | //package com.signalcollect
21 | //
22 | //import org.scalatest.FlatSpec
23 | //import org.scalatest.Matchers
24 | //
25 | //class CrazyLoadVertex(id: Int, initialState: Int) extends MemoryEfficientDataGraphVertex(id, initialState) {
26 | // type Signal = Int
27 | // type OutgoingSignalType = Int
28 | //
29 | // def collect = state + 1
30 | //
31 | // def computeSignal(targetId: Int) = state
32 | //
33 | // def scoreSignal = {
34 | // if (lastSignalState != state && state < 1000) {
35 | // 1
36 | // } else {
37 | // 0
38 | // }
39 | // }
40 | //
41 | // def scoreCollect = 1
42 | //
43 | //}
44 | //
45 | //class StressTest extends FlatSpec with Matchers {
46 | //
47 | // "Signal/Collect" should "correctly work when the system is pushed to the limit" in {
48 | // val g = GraphBuilder.withMessageSerialization(true).withKryoRegistrations(List(
49 | // "com.signalcollect.interfaces.AddVertex",
50 | // "com.signalcollect.StateForwarderEdge",
51 | // "com.signalcollect.CrazyLoadVertex",
52 | // "com.signalcollect.util.IntHashMap",
53 | // "scala.reflect.ManifestFactory$$anon$5",
54 | // "scala.runtime.Nothing$",
55 | // "com.signalcollect.MemoryEfficientSplayIntSet",
56 | // "com.signalcollect.util.SplayNode")).build
57 | // for (id <- 1 until 1000) {
58 | // g.addVertex(new CrazyLoadVertex(id, 1))
59 | // for (to <- 1 until 1000) {
60 | // g.addEdge(id, new StateForwarderEdge(to))
61 | // }
62 | // }
63 | // g.execute
64 | // g.shutdown
65 | // }
66 | //
67 | //}
68 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/SumOfOutWeights.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | trait SumOfOutWeights[Id, State] extends AbstractVertex[Id, State] {
23 |
24 | /**
25 | * @return The sum of the weights of all outgoing edges.
26 | */
27 | var sumOfOutWeights: Double = 0
28 |
29 | abstract override def addEdge(e: Edge[Any], graphEditor: GraphEditor[Any, Any]): Boolean = {
30 | val added = super.addEdge(e, graphEditor)
31 | if (added) {
32 | sumOfOutWeights += e.weight
33 | }
34 | added
35 | }
36 |
37 | abstract override def removeEdge(targetId: Any, graphEditor: GraphEditor[Any, Any]): Boolean = {
38 | val outgoingEdge = outgoingEdges.get(targetId)
39 | val weightToSubtract = outgoingEdge match {
40 | case None => 0
41 | case Some(edge) => edge.weight
42 | }
43 | val removed = super.removeEdge(targetId, graphEditor)
44 | if (removed) {
45 | sumOfOutWeights -= weightToSubtract
46 | }
47 | removed
48 | }
49 |
50 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/Timeable.scala:
--------------------------------------------------------------------------------
1 | package com.signalcollect
2 |
3 | import com.signalcollect.interfaces.AggregationOperation
4 | import scala.collection.immutable.SortedMap
5 |
6 | /** Container for the deliver and collect duration measurements */
7 | case class ActivityTime(signal: Int, deliver: Int, collect: Int) extends Ordered[ActivityTime] {
8 | override def toString: String =
9 | f"signal: ${signal}ns, deliver: ${deliver}ns, collect: ${collect}ns"
10 | def compare(that: ActivityTime) = {
11 | (that.signal + that.deliver + that.collect) - (this.signal + this.deliver + this.collect)
12 | }
13 | }
14 |
15 | /**
16 | * Finds the vertices in the graph which were active for the longest duration
17 | *
18 | * @param n the number of top vertices to find
19 | */
20 | class TopActivityAggregator[Id](n: Int)
21 | extends AggregationOperation[SortedMap[ActivityTime, Id]] {
22 | type ActivityMap = SortedMap[ActivityTime, Id]
23 | def extract(v: Vertex[_, _, _, _]): ActivityMap = v match {
24 | case t: Timeable[Id, _, _, _] =>
25 | SortedMap((ActivityTime(t.signalTime, t.deliverTime, t.collectTime) -> t.id))
26 | case _ =>
27 | SortedMap[ActivityTime, Id]()
28 | }
29 | def reduce(activities: Stream[ActivityMap]): ActivityMap = {
30 | activities.foldLeft(SortedMap[ActivityTime, Id]()) { (acc, m) => acc ++ m }.take(n)
31 | }
32 | }
33 |
34 | /** Allows measuring how long a vertex stays in deliverSignal and collect*/
35 | trait Timeable[Id, State, GraphIdUpperBound, GraphSignalUpperBound] extends Vertex[Id, State, GraphIdUpperBound, GraphSignalUpperBound] {
36 | var signalTime: Int = 0
37 | var deliverTime: Int = 0
38 | var collectTime: Int = 0
39 | def time[R](block: => R): (R, Int) = {
40 | val t0 = System.nanoTime()
41 | val result = block
42 | val t1 = System.nanoTime()
43 | (result, (t1 - t0).toInt)
44 | }
45 | abstract override def executeSignalOperation(graphEditor: GraphEditor[GraphIdUpperBound, GraphSignalUpperBound]): Unit = {
46 | val (_, t) = time(super.executeSignalOperation(graphEditor))
47 | signalTime += t
48 | }
49 | abstract override def deliverSignalWithSourceId(signal: GraphSignalUpperBound, sourceId: GraphIdUpperBound,
50 | graphEditor: GraphEditor[GraphIdUpperBound, GraphSignalUpperBound]): Boolean = {
51 | val (result, t) = time(super.deliverSignalWithSourceId(signal, sourceId, graphEditor))
52 | deliverTime += t
53 | result
54 | }
55 | abstract override def deliverSignalWithoutSourceId(signal: GraphSignalUpperBound,
56 | graphEditor: GraphEditor[GraphIdUpperBound, GraphSignalUpperBound]): Boolean = {
57 | val (result, t) = time(super.deliverSignalWithoutSourceId(signal, graphEditor))
58 | deliverTime += t
59 | result
60 | }
61 | abstract override def executeCollectOperation(graphEditor: GraphEditor[GraphIdUpperBound, GraphSignalUpperBound]): Unit = {
62 | val (_, t) = time(super.executeCollectOperation(graphEditor))
63 | collectTime += t
64 | }
65 | }
66 |
67 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/configuration/GraphConfiguration.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Francisco de Freitas
4 | *
5 | * Copyright 2011 University of Zurich
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.configuration
22 |
23 | import com.signalcollect.interfaces.MessageBusFactory
24 | import com.signalcollect.interfaces.StorageFactory
25 | import com.signalcollect.interfaces.WorkerFactory
26 | import com.signalcollect.nodeprovisioning.NodeProvisioner
27 | import com.signalcollect.factory.messagebus.AkkaMessageBusFactory
28 | import akka.event.Logging.LogLevel
29 | import akka.event.Logging
30 | import com.signalcollect.factory.worker.AkkaWorkerFactory
31 | import com.signalcollect.interfaces.SchedulerFactory
32 | import com.signalcollect.factory.scheduler.Throughput
33 | import com.signalcollect.interfaces.MapperFactory
34 | import com.signalcollect.factory.mapper.DefaultMapperFactory
35 | import com.signalcollect.storage.VertexMapStorage
36 | import com.signalcollect.factory.storage.MemoryEfficientStorage
37 | import akka.actor.ActorRef
38 | import akka.actor.ActorSystem
39 | import com.signalcollect.messaging.BulkMessageBus
40 | import com.signalcollect.factory.messagebus.BulkAkkaMessageBusFactory
41 | import scala.reflect.ClassTag
42 | import com.signalcollect.interfaces.ExistingVertexHandlerFactory
43 | import com.signalcollect.interfaces.UndeliverableSignalHandlerFactory
44 | import com.signalcollect.interfaces.EdgeAddedToNonExistentVertexHandlerFactory
45 |
46 | /**
47 | * All the graph configuration parameters with their defaults.
48 | */
49 | case class GraphConfiguration[@specialized(Int, Long) Id: ClassTag, Signal: ClassTag](
50 | actorSystem: Option[ActorSystem],
51 | actorNamePrefix: String,
52 | eagerIdleDetection: Boolean,
53 | consoleEnabled: Boolean,
54 | throttlingEnabled: Boolean,
55 | throttlingDuringLoadingEnabled: Boolean,
56 | supportBlockingGraphModificationsInVertex: Boolean,
57 | consoleHttpPort: Int,
58 | loggingLevel: Option[LogLevel], // Uses Akka config if undefined
59 | mapperFactory: MapperFactory[Id],
60 | storageFactory: StorageFactory[Id, Signal],
61 | schedulerFactory: SchedulerFactory[Id, Signal],
62 | preallocatedNodes: Option[Array[ActorRef]],
63 | nodeProvisioner: NodeProvisioner[Id, Signal],
64 | statsReportingIntervalInMilliseconds: Int,
65 | kryoRegistrations: List[String],
66 | kryoInitializer: Option[String], // Uses Akka config if undefined
67 | serializeMessages: Option[Boolean], // Uses Akka config if undefined
68 | workerFactory: WorkerFactory[Id, Signal],
69 | messageBusFactory: MessageBusFactory[Id, Signal],
70 | existingVertexHandlerFactory: ExistingVertexHandlerFactory[Id, Signal],
71 | undeliverableSignalHandlerFactory: UndeliverableSignalHandlerFactory[Id, Signal],
72 | edgeAddedToNonExistentVertexHandlerFactory: EdgeAddedToNonExistentVertexHandlerFactory[Id, Signal])
73 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/configuration/TerminationReason.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.configuration
21 |
22 | /**
23 | * Enumeration used to communicate why a computation was terminated.
24 | */
25 | object TerminationReason extends Enumeration with Serializable {
26 |
27 | /**
28 | * Computation terminated because the specified time limit was reached.
29 | */
30 | val TimeLimitReached = Value
31 |
32 | /**
33 | * Computation terminated because all the signalScores and collectScores
34 | * of the vertices were below the respective thresholds.
35 | */
36 | val Converged = Value
37 |
38 | /**
39 | * Computation terminated because the global constraint was met.
40 | */
41 | val GlobalConstraintMet = Value
42 |
43 | /**
44 | * Computation terminated because the step limit was reached.
45 | * This is only relevant for the synchronous execution mode.
46 | */
47 | val ComputationStepLimitReached = Value
48 |
49 | /**
50 | * Computation has not termination. This is used for the continuous
51 | * asynchronous execution mode, which returns immediately, but keeps on processing.
52 | */
53 | val Ongoing = Value
54 |
55 | /**
56 | * Computation was terminated by the user
57 | */
58 | val TerminatedByUser = Value
59 |
60 | }
61 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/ChineseWhispersClustering.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Daniel Strebel
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 | package com.signalcollect.examples
20 |
21 | import com.signalcollect._
22 | import com.signalcollect.configuration._
23 |
24 | /**
25 | * Represents an entity in a Chinese Whispers clustering algorithm.
26 | * Each entity determines its new state i.e. the cluster it belongs to,
27 | * to be the most popular cluster in among its neighbors. Ties are broken randomly.
28 | * Initially each entity assumes it belongs to its own cluster and therefore uses
29 | * its own ID as cluster label.
30 | */
31 | class CWVertex(id: Any, selfPreference: Double = 1.0) extends DataGraphVertex(id, id) {
32 |
33 | type Signal = (Any, Double)
34 |
35 | def collect = {
36 | //group most recent signals by clustering label
37 | val grouped = (((state, selfPreference)) :: signals.toList).groupBy(_._1)
38 | //sort the grouped list by the sum of all clustering label weights
39 | val sorted = grouped.toList sortBy { _._2.foldLeft(0.0)((sum, elem) => sum + elem._2) }
40 | //return the most popular label as new state
41 | sorted.last._1
42 | }
43 | }
44 |
45 | /**
46 | * Connects two entities in a Chinese Whispers algorithm and sets the signal to be
47 | * the source vertex's state plus together with the weight of the connection.
48 | */
49 | class CWEdge(t: Any, weight: Double = 1.0) extends DefaultEdge(t) {
50 | type Source = CWVertex
51 |
52 | def signal = (source.state, weight)
53 |
54 | }
55 |
56 | /**
57 | * Exemplary algorithm of two fully connected clusters containing {0,1,2} and {8,9,10}
58 | * where nodes 2 and 8 are connected via a chain.
59 | */
60 | object ChineseWhispersClustering extends App {
61 |
62 | val graph = GraphBuilder.build
63 |
64 | for (i <- 0 until 11) {
65 | graph.addVertex(new CWVertex(i))
66 | }
67 |
68 | graph.addVertex(new CWVertex(1))
69 | graph.addVertex(new CWVertex(2))
70 | graph.addVertex(new CWVertex(3))
71 | graph.addVertex(new CWVertex(4))
72 | graph.addVertex(new CWVertex(5))
73 |
74 | for (i <- 0 until 2) {
75 | graph.addEdge(i, new CWEdge(i + 1))
76 | graph.addEdge(i + 1, new CWEdge(i))
77 | }
78 |
79 | graph.addEdge(0, new CWEdge(2))
80 | graph.addEdge(2, new CWEdge(0))
81 |
82 | for (i <- 2 until 8) {
83 | graph.addEdge(i, new CWEdge(i + 1))
84 | graph.addEdge(i + 1, new CWEdge(i))
85 | }
86 |
87 | for (i <- 8 until 10) {
88 | graph.addEdge(i, new CWEdge(i + 1))
89 | graph.addEdge(i + 1, new CWEdge(i))
90 | }
91 |
92 | graph.addEdge(10, new CWEdge(8))
93 | graph.addEdge(8, new CWEdge(10))
94 |
95 | val stats = graph.execute
96 | graph.foreachVertex { v => println(v) }
97 |
98 | println(stats)
99 |
100 | graph.shutdown
101 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/ClusteringCoefficient.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.examples
21 |
22 | import com.signalcollect._
23 |
24 | class ClusteringCoefficientVertex(id: Any) extends DataGraphVertex(id, 0.0) {
25 |
26 | type Signal = Set[Any]
27 |
28 | lazy val maxEdges = outgoingEdges.size * (outgoingEdges.size - 1)
29 | lazy val neighbourIds = targetIds.toSet
30 |
31 | def collect = {
32 | if (maxEdges != 0) {
33 | val edgesBetweenNeighbours = signals.map(neighbourIds.intersect(_)).map(_.size).sum
34 | edgesBetweenNeighbours / maxEdges.toDouble
35 | } else {
36 | Double.NaN
37 | }
38 | }
39 |
40 | }
41 |
42 | class ClusteringCoefficientEdge(t: Any) extends DefaultEdge(t) {
43 | type Source = ClusteringCoefficientVertex
44 |
45 | def signal = source.neighbourIds
46 |
47 | }
48 |
49 | object ClusteringCoefficient extends App {
50 | val graph = GraphBuilder.build
51 | graph.addVertex(new ClusteringCoefficientVertex(1))
52 | graph.addVertex(new ClusteringCoefficientVertex(2))
53 | graph.addVertex(new ClusteringCoefficientVertex(3))
54 | graph.addVertex(new ClusteringCoefficientVertex(4))
55 | graph.addVertex(new ClusteringCoefficientVertex(5))
56 | graph.addVertex(new ClusteringCoefficientVertex(6))
57 | graph.addEdge(1, new ClusteringCoefficientEdge(2))
58 | graph.addEdge(2, new ClusteringCoefficientEdge(1))
59 | graph.addEdge(1, new ClusteringCoefficientEdge(3))
60 | graph.addEdge(3, new ClusteringCoefficientEdge(1))
61 | graph.addEdge(1, new ClusteringCoefficientEdge(4))
62 | graph.addEdge(4, new ClusteringCoefficientEdge(1))
63 | graph.addEdge(1, new ClusteringCoefficientEdge(5))
64 | graph.addEdge(5, new ClusteringCoefficientEdge(1))
65 | graph.addEdge(2, new ClusteringCoefficientEdge(3))
66 | graph.addEdge(3, new ClusteringCoefficientEdge(2))
67 | graph.addEdge(3, new ClusteringCoefficientEdge(5))
68 | graph.addEdge(5, new ClusteringCoefficientEdge(3))
69 | graph.addEdge(6, new ClusteringCoefficientEdge(5))
70 | graph.addEdge(5, new ClusteringCoefficientEdge(6))
71 | graph.addEdge(6, new ClusteringCoefficientEdge(1))
72 | graph.addEdge(1, new ClusteringCoefficientEdge(6))
73 |
74 | val stats = graph.execute
75 | println(stats)
76 | graph.foreachVertex(println(_))
77 | graph.shutdown
78 | }
79 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/CompanyValuation.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.examples
21 |
22 | import com.signalcollect._
23 | import com.signalcollect.configuration.ExecutionMode
24 |
25 | class Company(id: String, coreValue: Double) extends DataGraphVertex(id, coreValue) {
26 | type Signal = Double
27 | def collect = coreValue + signals.sum
28 | }
29 |
30 | class OwnedBy(targetId: String, percentage: Double) extends DefaultEdge(targetId) {
31 | type Source = Company
32 | def signal = source.state * percentage
33 | }
34 |
35 | /**
36 | * Example computation of the total value of companies that have mutual stakes in each other.
37 | *
38 | * Renault currently has a 44.4 percent stake in Nissan, and Nissan holds a 15 percent stake in Renault.
39 | * Source: http://en.wikipedia.org/wiki/Renault-Nissan_Alliance.
40 | */
41 | object CaompanyValuation extends App {
42 | val graph = GraphBuilder.withConsole(true).build
43 | graph.addVertex(new Company("Nissan", coreValue = 37)) //market cap: ~ USD 42 billion
44 | graph.addVertex(new Company("Renault", coreValue = 14)) // market cap: ~ USD 33 billion
45 | graph.addEdge("Nissan", new OwnedBy("Renault", 0.444))
46 | graph.addEdge("Renault", new OwnedBy("Nissan", 0.15))
47 | graph.execute(ExecutionConfiguration.withExecutionMode(ExecutionMode.Interactive))
48 | graph.foreachVertex(println(_))
49 | graph.shutdown
50 | }
51 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/EfficientPageRank.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.examples
21 |
22 | import com.signalcollect._
23 | import com.signalcollect.configuration.ExecutionMode
24 | import com.signalcollect.factory.messagebus.IntIdDoubleSignalMessageBusFactory
25 |
26 | /**
27 | * Placeholder edge that gets discarded by memory efficient vertices that
28 | * have their own internal edge representations.
29 | */
30 | class PlaceholderEdge[Id](targetId: Id) extends DefaultEdge(targetId) {
31 | def signal = throw new Exception("This is a placeholder edge, its signal function should never be called.")
32 | }
33 |
34 | class EfficientPageRankVertex(id: Int)
35 | extends MemoryEfficientDataFlowVertex[Double, Double](id = id, state = 0.15) {
36 |
37 | type OutgoingSignalType = Double
38 |
39 | def computeSignal(edgeId: Int) =
40 | throw new Exception("This vertex type computes signals inside of 'executeSignalOperation', " +
41 | "'computeSignal' should never be called.")
42 |
43 | override def executeSignalOperation(graphEditor: GraphEditor[Int, Double]) {
44 | if (edgeCount != 0) {
45 | val signal = (state - lastSignalState) / edgeCount
46 | targetIds.foreach(graphEditor.sendSignal(signal, _))
47 | }
48 | lastSignalState = state
49 | }
50 |
51 | def collect(signal: Double): Double = {
52 | state + 0.85 * signal
53 | }
54 |
55 | def scoreSignal = {
56 | state - lastSignalState
57 | }
58 |
59 | def addTargetId(targetId: Int) {
60 | _targetIds.insert(targetId)
61 | }
62 |
63 | }
64 |
65 | /** Builds a PageRank compute graph and executes the computation */
66 | object MemoryEfficientPageRank extends App {
67 | val graph = new GraphBuilder[Int, Double]().
68 | withMessageBusFactory(new IntIdDoubleSignalMessageBusFactory(10000)).
69 | // withConsole(true).
70 | build
71 | graph.addVertex(new EfficientPageRankVertex(1))
72 | graph.addVertex(new EfficientPageRankVertex(2))
73 | graph.addVertex(new EfficientPageRankVertex(3))
74 | graph.addEdge(1, new PlaceholderEdge(2))
75 | graph.addEdge(2, new PlaceholderEdge(1))
76 | graph.addEdge(2, new PlaceholderEdge(3))
77 | graph.addEdge(3, new PlaceholderEdge(2))
78 | graph.awaitIdle
79 | val stats = graph.execute //(ExecutionConfiguration.withExecutionMode(ExecutionMode.Interactive))
80 | println(stats)
81 |
82 | graph.foreachVertex(println(_))
83 | graph.shutdown
84 | }
85 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/LodNeighbourhoodPageRank.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.examples
21 |
22 | import com.signalcollect._
23 | import scala.io.Source
24 | import scala.io.Codec
25 | import com.signalcollect.TopKFinder
26 | import sun.net.www.http.HttpClient
27 | import java.net.URL
28 | import java.net.UnknownHostException
29 |
30 | class LodNeighbourhoodPageRank(id: String, crawlDepth: Int = 3) extends PageRankVertex[String](id) {
31 |
32 | override def afterInitialization(ge: GraphEditor[Any, Any]) {
33 | println("Added " + id)
34 | if (crawlDepth > 0) {
35 | try {
36 | val dataUrl = id.replace(
37 | "http://dbpedia.org/resource/",
38 | "http://dbpedia.org/data/") +
39 | ".ntriples"
40 | val lodSite = Source.fromURL(dataUrl)
41 | val triples = lodSite.getLines
42 | val urls = triples.map(_.split("\t")).map(_(2)).flatMap(s =>
43 | if (s.startsWith(" println("Problem: " + t.toString) // Parse error or not a LOD link, ignore.
55 | }
56 | }
57 |
58 | }
59 | }
60 |
61 | object LodNeighbourhood extends App {
62 | val graph = GraphBuilder.build
63 | graph.addVertex(
64 | new LodNeighbourhoodPageRank("http://dbpedia.org/resource/Tiger"))
65 | println("Awaiting idle.")
66 | graph.awaitIdle
67 | println("Executing.")
68 | val stats = graph.execute
69 | val top20 = graph.aggregate(new TopKFinder[Double](20))
70 | println(stats)
71 | println(top20.mkString("\n"))
72 | graph.shutdown
73 | }
74 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/PageRank.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.examples
21 |
22 | import com.signalcollect._
23 | import com.signalcollect.configuration.ExecutionMode
24 |
25 | /**
26 | * Represents an edge in a PageRank compute graph
27 | *
28 | * @param s: the identifier of the source vertex
29 | * @param t: the identifier of the target vertex
30 | */
31 | class PageRankEdge[Id](t: Id) extends DefaultEdge(t) {
32 |
33 | type Source = PageRankVertex[Id]
34 |
35 | /**
36 | * The signal function calculates how much rank the source vertex
37 | * transfers to the target vertex.
38 | */
39 | def signal = source.state * weight / source.sumOfOutWeights
40 |
41 | }
42 |
43 | /**
44 | * Represents a page in a PageRank compute graph
45 | *
46 | * @param id: the identifier of this vertex
47 | * @param dampingFactor: @see PageRank algorithm
48 | */
49 | class PageRankVertex[Id](id: Id, dampingFactor: Double = 0.85) extends DataGraphVertex[Id, Double](id, 1 - dampingFactor) {
50 |
51 | type Signal = Double
52 |
53 | /**
54 | * The collect function calculates the rank of this vertex based on the rank
55 | * received from neighbors and the damping factor.
56 | */
57 | def collect: Double = 1 - dampingFactor + dampingFactor * signals.sum
58 |
59 | override def scoreSignal: Double = {
60 | if (edgesModifiedSinceSignalOperation) {
61 | 1
62 | } else {
63 | lastSignalState match {
64 | case None => 1
65 | case Some(oldState) => (state - oldState).abs
66 | }
67 | }
68 | }
69 |
70 | }
71 |
72 | /** Builds a PageRank compute graph and executes the computation */
73 | object PageRank extends App {
74 | val graph = new GraphBuilder[Any, Any]().
75 | withConsole(true).
76 | build
77 |
78 | graph.awaitIdle
79 | graph.addVertex(new PageRankVertex(1))
80 | graph.addVertex(new PageRankVertex(2))
81 | graph.addVertex(new PageRankVertex(3))
82 | graph.addEdge(1, new PageRankEdge(2))
83 | graph.addEdge(2, new PageRankEdge(1))
84 | graph.addEdge(2, new PageRankEdge(3))
85 | graph.addEdge(3, new PageRankEdge(2))
86 |
87 | graph.awaitIdle
88 | val stats = graph.execute(ExecutionConfiguration.withExecutionMode(ExecutionMode.Interactive))
89 | println(stats)
90 |
91 | graph.foreachVertex(println(_))
92 | graph.shutdown
93 | }
94 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/SchellingSegregation.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Daniel Strebel
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.examples
21 |
22 | import com.signalcollect._
23 |
24 | /**
25 | * Represents an agent in a Schelling-Segregation simulation
26 | *
27 | * @param id: the identifier of this vertex
28 | * @param initialState: initial state of the agent
29 | * @param equalityThreshold: minimum required percentage of neighbors with equal state as float value from 0 to 1.
30 | */
31 | class SegregationAgent(id: Any, initialState: Int, equalityThreshold: Float) extends DataGraphVertex(id, initialState) {
32 | type Signal = Int
33 |
34 | def collect = {
35 | val equalNeighborsCount = (signals filter (_ equals this.state)).size
36 | val totalNeighborsCount = signals.size
37 | if (equalNeighborsCount.toFloat / totalNeighborsCount >= equalityThreshold) {
38 | this.state
39 | } else {
40 | (this.state + 1) % 2
41 | }
42 | }
43 |
44 | }
45 |
46 | /**Builds a Schelling-Segregation simulation on a random grid and executes it**/
47 | object SchellingSegregation extends App {
48 | val graph = GraphBuilder.build
49 |
50 | //Dimensions of the grid
51 | val columns = 140
52 | val rows = 40
53 |
54 | println("Adding vertices ...") //Create all cells.
55 | for (column <- 0 to columns; row <- 0 to rows) {
56 | graph.addVertex(new SegregationAgent((column, row), (math.random * 2.0).floor.toInt, 0.45f))
57 | }
58 |
59 | println("Adding edges ...") // Connect the neighboring cells.
60 | for (column <- 0 to columns; row <- 0 to rows) {
61 | for (neighbor <- neighbors(column, row)) {
62 | if (inGrid(neighbor._1, neighbor._2)) {
63 | graph.addEdge((column, row), new StateForwarderEdge((neighbor._1, neighbor._2)))
64 | }
65 | }
66 | }
67 |
68 | println("Grid before:\n" + stringRepresentationOfGraph)
69 | val stats = graph.execute
70 | println(stats) //Print computation statistics
71 | println("Grid after:\n" + stringRepresentationOfGraph)
72 | graph.shutdown
73 |
74 | // Returns all the neighboring cells of the cell with the given row/column
75 | def neighbors(column: Int, row: Int): List[(Int, Int)] = {
76 | List(
77 | (column - 1, row - 1), (column, row - 1), (column + 1, row - 1),
78 | (column - 1, row), (column + 1, row),
79 | (column - 1, row + 1), (column, row + 1), (column + 1, row + 1))
80 | }
81 |
82 | // Tests if a cell is within the grid boundaries
83 | def inGrid(column: Int, row: Int): Boolean = {
84 | column >= 0 && row >= 0 && column <= columns && row <= rows
85 | }
86 |
87 | // Creates a string representation of the graph
88 | def stringRepresentationOfGraph: String = {
89 | val stateMap = graph.aggregate(new IdStateMapAggregator[(Int, Int), Int])
90 | val stringBuilder = new StringBuilder
91 | for (row <- 0 to rows) {
92 | for (column <- 0 to columns) {
93 | stringBuilder.append(stateMap((column, row)))
94 | }
95 | stringBuilder.append("\n")
96 | }
97 | stringBuilder.toString
98 | }
99 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/examples/WebCrawler.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.examples
21 |
22 | import com.signalcollect._
23 |
24 | /**
25 | * Regular expression to match links in Html strings
26 | */
27 | object Regex {
28 | val hyperlink = """""".r
29 | }
30 |
31 | /**
32 | * Proof of concept combination of the PageRank vertices with a WebCrawler
33 | * Do not use on a larger scale:
34 | * - does not respect robots.txt
35 | * - lacks proper user agent string
36 | */
37 | object WebCrawler extends App {
38 | val graph = GraphBuilder.build
39 | graph.addVertex(new Webpage("http://www.ifi.uzh.ch/ddis/", 2))
40 | val stats = graph.execute
41 | graph.foreachVertex(println(_))
42 | println(stats)
43 | graph.shutdown
44 | }
45 |
46 | /**
47 | * Adds linked webpages as vertices to the graph and connects them with a link edge
48 | */
49 | class Webpage(id: String, crawlDepth: Int, dampingFactor: Double = 0.85) extends PageRankVertex(id, dampingFactor) {
50 |
51 | /** This method gets called by the framework after the vertex has been fully initialized. */
52 | override def afterInitialization(graphEditor: GraphEditor[Any, Any]) {
53 | super.afterInitialization(graphEditor)
54 | if (crawlDepth > 0) {
55 | try {
56 | val webpage = io.Source.fromURL(id, "ISO-8859-1").mkString
57 | Regex.hyperlink.findAllIn(webpage).matchData map (_.group(1)) foreach { linked =>
58 | graphEditor.addVertex(new Webpage(linked, crawlDepth - 1))
59 | graphEditor.addEdge(id, new PageRankEdge(linked))
60 | }
61 | } catch {
62 | case _: Throwable =>
63 | }
64 | }
65 | }
66 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/handler/DefaultHandlerFactory.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.handler
21 |
22 | import com.signalcollect._
23 | import com.signalcollect.interfaces._
24 |
25 | class DefaultExistingVertexHandlerFactory[Id, Signal] extends ExistingVertexHandlerFactory[Id, Signal] {
26 | def createInstance: ExistingVertexHandler[Id, Signal] =
27 | new DefaultExistingVertexHandler[Id, Signal]
28 | override def toString = "DefaultExistingVertexHandlerFactory"
29 | }
30 |
31 | class DefaultExistingVertexHandler[Id, Signal] extends ExistingVertexHandler[Id, Signal] {
32 | def mergeVertices(existing: Vertex[Id, _, Id, Signal], failedVertexAddition: Vertex[Id, _, Id, Signal], ge: GraphEditor[Id, Signal]) {
33 | // Do nothing, the second vertex with the same id is silently discarded.
34 | }
35 | }
36 |
37 | class DefaultUndeliverableSignalHandlerFactory[@specialized(Int, Long) Id, Signal] extends UndeliverableSignalHandlerFactory[Id, Signal] {
38 | def createInstance: UndeliverableSignalHandler[Id, Signal] =
39 | new DefaultUndeliverableSignalHandler[Id, Signal]
40 | override def toString = "DefaultUndeliverableSignalHandlerFactory"
41 | }
42 |
43 | class DefaultUndeliverableSignalHandler[@specialized(Int, Long) Id, Signal] extends UndeliverableSignalHandler[Id, Signal] {
44 | def vertexForSignalNotFound(s: Signal, inexistentTargetId: Id, senderId: Option[Id], ge: GraphEditor[Id, Signal]) {
45 | throw new Exception(s"Undeliverable signal: $s from $senderId could not be delivered to $inexistentTargetId, because no vertex with that id exists..")
46 | }
47 | }
48 |
49 | class DefaultEdgeAddedToNonExistentVertexHandlerFactory[@specialized(Int, Long) Id, Signal] extends EdgeAddedToNonExistentVertexHandlerFactory[Id, Signal] {
50 | def createInstance: EdgeAddedToNonExistentVertexHandler[Id, Signal] =
51 | new DefaultEdgeAddedToNonExistentVertexHandler[Id, Signal]
52 | override def toString = "DefaultEdgeAddedToNonExistentVertexHandlerFactory"
53 | }
54 |
55 | class DefaultEdgeAddedToNonExistentVertexHandler[@specialized(Int, Long) Id, Signal] extends EdgeAddedToNonExistentVertexHandler[Id, Signal] {
56 | def handleImpossibleEdgeAddition(edge: Edge[Id], vertexId: Id, graphEditor: GraphEditor[Id, Signal]): Option[Vertex[Id, _, Id, Signal]] = {
57 | throw new Exception(
58 | s"Could not add edge: ${edge.getClass.getSimpleName}(id = $vertexId -> ${edge.targetId}), because vertex with id $vertexId does not exist.")
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/mapper/DefaultMapperFactory.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Mihaela Verman
4 | *
5 | * Copyright 2013 University of Zurich
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.factory.mapper
22 |
23 | import com.signalcollect.interfaces.MapperFactory
24 | import com.signalcollect.interfaces.VertexToWorkerMapper
25 | import com.signalcollect.messaging.DefaultVertexToWorkerMapper
26 |
27 | /**
28 | * Default random hash partitioning mapper.
29 | * Has good load balancing but poor locality.
30 | */
31 | class DefaultMapperFactory[@specialized(Int, Long) Id] extends MapperFactory[Id] {
32 | def createInstance(numberOfNodes: Int, workersPerNode: Int): VertexToWorkerMapper[Id] =
33 | new DefaultVertexToWorkerMapper[Id](numberOfNodes, workersPerNode)
34 | override def toString = "DefaultMapperFactory"
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/messagebus/AkkaMessageBusFactory.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.messagebus
21 |
22 | import scala.reflect.ClassTag
23 | import com.signalcollect.interfaces.MessageBus
24 | import com.signalcollect.interfaces.MessageBusFactory
25 | import com.signalcollect.interfaces.WorkerApiFactory
26 | import com.signalcollect.messaging._
27 | import com.signalcollect.interfaces.VertexToWorkerMapper
28 | import akka.actor.ActorSystem
29 |
30 | class AkkaMessageBusFactory[@specialized(Int, Long) Id: ClassTag, Signal: ClassTag]
31 | extends MessageBusFactory[Id, Signal] {
32 | def createInstance(
33 | system: ActorSystem,
34 | numberOfWorkers: Int,
35 | numberOfNodes: Int,
36 | mapper: VertexToWorkerMapper[Id],
37 | sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
38 | workerApiFactory: WorkerApiFactory[Id, Signal]): MessageBus[Id, Signal] = {
39 | new DefaultMessageBus[Id, Signal](
40 | system,
41 | numberOfWorkers,
42 | numberOfNodes,
43 | mapper,
44 | sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
45 | workerApiFactory)
46 | }
47 | override def toString = "AkkaMessageBusFactory"
48 | }
49 |
50 | /**
51 | * Stores outgoing messages until 'flushThreshold' messages are queued for a worker.
52 | * Combines messages for the same vertex using 'combiner'.
53 | */
54 | class BulkAkkaMessageBusFactory[@specialized(Int, Long) Id: ClassTag, Signal: ClassTag](
55 | flushThreshold: Int,
56 | withSourceIds: Boolean)
57 | extends MessageBusFactory[Id, Signal] {
58 | def createInstance(
59 | system: ActorSystem,
60 | numberOfWorkers: Int,
61 | numberOfNodes: Int,
62 | mapper: VertexToWorkerMapper[Id],
63 | sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
64 | workerApiFactory: WorkerApiFactory[Id, Signal]): MessageBus[Id, Signal] = {
65 | new BulkMessageBus[Id, Signal](
66 | system,
67 | numberOfWorkers,
68 | numberOfNodes,
69 | mapper,
70 | flushThreshold,
71 | withSourceIds,
72 | sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
73 | workerApiFactory)
74 | }
75 | override def toString = "BulkAkkaMessageBusFactory"
76 | }
77 |
78 | class IntIdDoubleSignalMessageBusFactory(flushThreshold: Int)
79 | extends MessageBusFactory[Int, Double] {
80 | def createInstance(
81 | system: ActorSystem,
82 | numberOfWorkers: Int,
83 | numberOfNodes: Int,
84 | mapper: VertexToWorkerMapper[Int],
85 | sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
86 | workerApiFactory: WorkerApiFactory[Int, Double]): MessageBus[Int, Double] = {
87 | new IntIdDoubleSignalMessageBus(
88 | system,
89 | numberOfWorkers,
90 | numberOfNodes,
91 | mapper,
92 | flushThreshold,
93 | sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
94 | workerApiFactory)
95 | }
96 | override def toString = "CombiningMessageBusFactory"
97 | }
98 |
99 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/scheduler/LowLatency.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.scheduler
21 |
22 | import com.signalcollect.interfaces.SchedulerFactory
23 | import com.signalcollect.interfaces.Scheduler
24 | import com.signalcollect.interfaces.Worker
25 | import com.signalcollect.scheduler.LowLatencyScheduler
26 |
27 | /**
28 | * The low-latency scheduler tries to minimize the latency between
29 | * receiving a message, collecting and signaling.
30 | */
31 | class LowLatency[@specialized(Int, Long) Id, Signal] extends SchedulerFactory[Id, Signal] {
32 | def createInstance(worker: Worker[Id, Signal]): Scheduler[Id, Signal] = {
33 | new LowLatencyScheduler[Id, Signal](worker)
34 | }
35 | override def toString: String = "Low-Latency Scheduler Factory"
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/scheduler/Throughput.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.scheduler
21 |
22 | import com.signalcollect.interfaces.SchedulerFactory
23 | import com.signalcollect.interfaces.Scheduler
24 | import com.signalcollect.interfaces.Worker
25 | import com.signalcollect.scheduler.ThroughputScheduler
26 |
27 | /**
28 | * The low-latency scheduler tries to maximize the signaling/collecting
29 | * throughput and prevent excessive memory usage.
30 | */
31 | class Throughput[@specialized(Int, Long) Id, Signal] extends SchedulerFactory[Id, Signal] {
32 | def createInstance(worker: Worker[Id, Signal]): Scheduler[Id, Signal] = {
33 | new ThroughputScheduler[Id, Signal](worker)
34 | }
35 | override def toString: String = "Throughput Scheduler Factory"
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/storage/JavaMapStorage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.storage
21 |
22 | import com.signalcollect.interfaces.Storage
23 | import com.signalcollect.interfaces.StorageFactory
24 | import com.signalcollect.storage.JavaMapVertexStorage
25 |
26 | /**
27 | * Storage backed by Java HashMaps.
28 | */
29 | class JavaMapStorage[@specialized(Int, Long) Id, Signal] extends StorageFactory[Id, Signal] {
30 | def createInstance: Storage[Id, Signal] = new JavaMapVertexStorage[Id, Signal]
31 | override def toString = "JavaMapStorage"
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/storage/MemoryEfficientStorage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.storage
21 |
22 | import com.signalcollect.interfaces.Storage
23 | import com.signalcollect.interfaces.StorageFactory
24 | import com.signalcollect.storage._
25 | import scala.reflect.ClassTag
26 |
27 | /**
28 | * Storage backed by a memory efficient VertexMap.
29 | * Inserts/removals of vertices are slower than with a Java HashMap.
30 | */
31 | class MemoryEfficientStorage[@specialized(Int, Long) Id: ClassTag, Signal: ClassTag] extends StorageFactory[Id, Signal] {
32 | def createInstance: Storage[Id, Signal] = {
33 | new VertexMapStorage[Id, Signal]()
34 | }
35 | override def toString = "MemoryEfficientStorage"
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/storage/MixedStorage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.storage
21 |
22 | import com.signalcollect.interfaces.Storage
23 | import com.signalcollect.interfaces.StorageFactory
24 | import com.signalcollect.storage.MixedVertexStorage
25 |
26 | /**
27 | * Storage backed by a mix between a custom-tailored open hash map implementation for vertices
28 | * and the default Java HashMap for toSignal/toCollect.
29 | */
30 | class MixedStorage[@specialized(Int, Long) Id, Signal] extends StorageFactory[Id, Signal] {
31 | def createInstance: Storage[Id, Signal] = new MixedVertexStorage[Id, Signal]
32 | override def toString = "MixedStorage"
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/worker/Akka.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.worker
21 |
22 | import scala.reflect.ClassTag
23 | import com.signalcollect.configuration.GraphConfiguration
24 | import com.signalcollect.interfaces.WorkerFactory
25 | import com.signalcollect.worker.AkkaWorker
26 | import com.signalcollect.interfaces.MessageBusFactory
27 | import com.signalcollect.interfaces.MapperFactory
28 | import com.signalcollect.interfaces.StorageFactory
29 | import com.signalcollect.interfaces.SchedulerFactory
30 | import com.signalcollect.interfaces.ExistingVertexHandlerFactory
31 | import com.signalcollect.interfaces.UndeliverableSignalHandlerFactory
32 | import com.signalcollect.interfaces.EdgeAddedToNonExistentVertexHandlerFactory
33 |
34 | /**
35 | * The default Akka worker implementation.
36 | */
37 | class AkkaWorkerFactory[Id: ClassTag, Signal: ClassTag] extends WorkerFactory[Id, Signal] {
38 | def createInstance(
39 | workerId: Int,
40 | numberOfWorkers: Int,
41 | numberOfNodes: Int,
42 | messageBusFactory: MessageBusFactory[Id, Signal],
43 | mapperFactory: MapperFactory[Id],
44 | storageFactory: StorageFactory[Id, Signal],
45 | schedulerFactory: SchedulerFactory[Id, Signal],
46 | existingVertexHandlerFactory: ExistingVertexHandlerFactory[Id, Signal],
47 | undeliverableSignalHandlerFactory: UndeliverableSignalHandlerFactory[Id, Signal],
48 | edgeAddedToNonExistentVertexHandlerFactory: EdgeAddedToNonExistentVertexHandlerFactory[Id, Signal],
49 | statsReportingIntervalInMilliseconds: Int,
50 | eagerIdleDetection: Boolean,
51 | throttlingEnabled: Boolean,
52 | throttlingDuringLoadingEnabled: Boolean,
53 | supportBlockingGraphModificationsInVertex: Boolean): AkkaWorker[Id, Signal] = {
54 | new AkkaWorker[Id, Signal](
55 | workerId,
56 | numberOfWorkers,
57 | numberOfNodes,
58 | messageBusFactory,
59 | mapperFactory,
60 | storageFactory,
61 | schedulerFactory,
62 | existingVertexHandlerFactory,
63 | undeliverableSignalHandlerFactory,
64 | edgeAddedToNonExistentVertexHandlerFactory,
65 | statsReportingIntervalInMilliseconds,
66 | eagerIdleDetection,
67 | throttlingEnabled,
68 | throttlingDuringLoadingEnabled,
69 | supportBlockingGraphModificationsInVertex)
70 | }
71 | override def toString: String = "AkkaWorkerFactory"
72 | }
73 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/factory/workerapi/WorkerApiFactory.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.factory.workerapi
21 |
22 | import com.signalcollect.coordinator.DefaultWorkerApi
23 | import com.signalcollect.interfaces.VertexToWorkerMapper
24 | import com.signalcollect.interfaces.WorkerApi
25 | import com.signalcollect.interfaces.WorkerApiFactory
26 |
27 | class DefaultWorkerApiFactory[Id, Signal] extends WorkerApiFactory[Id, Signal] {
28 | override def createInstance(
29 | workerProxies: Array[WorkerApi[Id, Signal]],
30 | mapper: VertexToWorkerMapper[Id]): WorkerApi[Id, Signal] = {
31 | new DefaultWorkerApi(workerProxies, mapper)
32 | }
33 | override def toString = "DefaultWorkerApiFactory"
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/AggregationOperation.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import com.signalcollect.Vertex
23 |
24 | /**
25 | * An aggregation operation aggregates some value of type `ValueType` over all the vertices in a graph.
26 | */
27 | abstract class AggregationOperation[ValueType] extends ComplexAggregation[ValueType, ValueType] {
28 |
29 | /**
30 | * Extracts values of type `ValueType` from vertices.
31 | */
32 | def extract(v: Vertex[_, _, _, _]): ValueType
33 |
34 | /**
35 | * Reduces an arbitrary number of elements to one element.
36 | */
37 | def reduce(elements: Stream[ValueType]): ValueType
38 |
39 | def aggregationOnWorker(vertices: Stream[Vertex[_, _, _, _]]): ValueType = {
40 | reduce(vertices map extract)
41 | }
42 |
43 | def aggregationOnCoordinator(workerResults: Iterable[ValueType]): ValueType = {
44 | reduce(workerResults.toStream)
45 | }
46 |
47 | }
48 |
49 | /**
50 | * Implementation related interface.
51 | * Only use for more complex aggregations or when performance is important.
52 | */
53 | abstract class ComplexAggregation[WorkerResult, EndResult] extends Serializable {
54 |
55 | def aggregationOnWorker(vertices: Stream[Vertex[_, _, _, _]]): WorkerResult
56 |
57 | def aggregationOnCoordinator(workerResults: Iterable[WorkerResult]): EndResult
58 |
59 | }
60 |
61 | /**
62 | * More modular interface for aggregation operations.
63 | */
64 | abstract class ModularAggregationOperation[ValueType] extends AggregationOperation[ValueType] {
65 |
66 | /**
67 | * Reduces an arbitrary number of elements to one element.
68 | */
69 | def reduce(elements: Stream[ValueType]): ValueType = {
70 | elements.foldLeft(neutralElement)(aggregate)
71 | }
72 |
73 | /**
74 | * Aggregates all the values extracted by the `extract` function.
75 | *
76 | * @note There is no guarantee about the order in which this function gets executed on the extracted values.
77 | */
78 | def aggregate(a: ValueType, b: ValueType): ValueType
79 |
80 | /**
81 | * Neutral element of the `aggregate` function:
82 | * `aggregate(x, neutralElement) == x`
83 | */
84 | def neutralElement: ValueType
85 | }
86 |
87 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Coordinator.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import com.signalcollect.GraphEditor
23 |
24 | import akka.actor.Actor
25 | import akka.event.Logging.LogLevel
26 | import akka.event.Logging.LogEvent
27 |
28 | /**
29 | * Required because a Java Dynamic Proxy can only work with interfaces.
30 | */
31 | trait Coordinator[Id, Signal] extends Actor with MessageRecipientRegistry {
32 |
33 | override def toString: String = this.getClass.getSimpleName
34 |
35 | def getWorkerApi(): WorkerApi[Id, Signal]
36 |
37 | def getGraphEditor(): GraphEditor[Id, Signal]
38 |
39 | def getGlobalInboxSize(): Long
40 |
41 | def getWorkerStatuses(): Array[WorkerStatus]
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Factory.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import scala.reflect.ClassTag
23 | import com.signalcollect.factory.workerapi.DefaultWorkerApiFactory
24 | import akka.actor.ActorSystem
25 | import com.signalcollect.worker.AkkaWorker
26 |
27 | abstract class Factory extends Serializable {
28 | override def toString = this.getClass.getSimpleName
29 | }
30 |
31 | abstract class ExistingVertexHandlerFactory[Id, Signal] extends Factory {
32 | def createInstance: ExistingVertexHandler[Id, Signal]
33 | }
34 |
35 | abstract class UndeliverableSignalHandlerFactory[Id, Signal] extends Factory {
36 | def createInstance: UndeliverableSignalHandler[Id, Signal]
37 | }
38 |
39 | abstract class EdgeAddedToNonExistentVertexHandlerFactory[Id, Signal] extends Factory {
40 | def createInstance: EdgeAddedToNonExistentVertexHandler[Id, Signal]
41 | }
42 |
43 | abstract class WorkerFactory[Id: ClassTag, Signal: ClassTag] extends Factory {
44 | def createInstance(
45 | workerId: Int,
46 | numberOfWorkers: Int,
47 | numberOfNodes: Int,
48 | messageBusFactory: MessageBusFactory[Id, Signal],
49 | mapperFactory: MapperFactory[Id],
50 | storageFactory: StorageFactory[Id, Signal],
51 | schedulerFactory: SchedulerFactory[Id, Signal],
52 | existingVertexHandlerFactory: ExistingVertexHandlerFactory[Id, Signal],
53 | undeliverableSignalHandlerFactory: UndeliverableSignalHandlerFactory[Id, Signal],
54 | edgeAddedToNonExistentVertexHandlerFactory: EdgeAddedToNonExistentVertexHandlerFactory[Id, Signal],
55 | statsReportingIntervalInMilliseconds: Int,
56 | eagerIdleDetection: Boolean,
57 | throttlingEnabled: Boolean,
58 | throttlingDuringLoadingEnabled: Boolean,
59 | supportBlockingGraphModificationsInVertex: Boolean): AkkaWorker[Id, Signal]
60 | }
61 |
62 | abstract class MessageBusFactory[Id: ClassTag, Signal: ClassTag] extends Factory {
63 | def createInstance(
64 | system: ActorSystem,
65 | numberOfWorkers: Int,
66 | numberOfNodes: Int,
67 | mapper: VertexToWorkerMapper[Id],
68 | sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
69 | workerApiFactory: WorkerApiFactory[Id, Signal] = new DefaultWorkerApiFactory[Id, Signal]): MessageBus[Id, Signal]
70 | }
71 |
72 | abstract class MapperFactory[Id] extends Factory {
73 | def createInstance(numberOfNodes: Int, workersPerNode: Int): VertexToWorkerMapper[Id]
74 | }
75 |
76 | abstract class StorageFactory[Id, Signal] extends Factory {
77 | def createInstance: Storage[Id, Signal]
78 | }
79 |
80 | abstract class SchedulerFactory[Id, Signal] extends Factory {
81 | def createInstance(worker: Worker[Id, Signal]): Scheduler[Id, Signal]
82 | }
83 |
84 | abstract class WorkerApiFactory[Id, Signal] extends Factory {
85 | def createInstance(
86 | workerProxies: Array[WorkerApi[Id, Signal]],
87 | mapper: VertexToWorkerMapper[Id]): WorkerApi[Id, Signal]
88 | }
89 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Handlers.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import com.signalcollect.Vertex
23 | import com.signalcollect.GraphEditor
24 | import com.signalcollect.Edge
25 |
26 | trait ExistingVertexHandler[Id, Signal] {
27 |
28 | /**
29 | * Sets the function that can intervene when a vertex with the same ID is added
30 | * repeatedly. The new vertex will be thrown out for sure, but some of its
31 | * information might be added to the existing vertex.
32 | *
33 | * @note By default the addition of a vertex is ignored if an existing vertex has the same ID.
34 | */
35 | def mergeVertices(existing: Vertex[Id, _, Id, Signal], failedVertexAddition: Vertex[Id, _, Id, Signal], ge: GraphEditor[Id, Signal])
36 |
37 | }
38 |
39 | trait UndeliverableSignalHandler[@specialized(Int, Long) Id, Signal] {
40 |
41 | /**
42 | * Sets the function that handles signals that could not be delivered to a vertex.
43 | *
44 | * @note By default an exception is thrown when a signal is not deliverable. The handler function
45 | * receives the signal and an instance of GraphEditor as parameters in order to take some
46 | * action that handles this case.
47 | */
48 | def vertexForSignalNotFound(signal: Signal, inexistentTargetId: Id, senderId: Option[Id], graphEditor: GraphEditor[Id, Signal])
49 |
50 | }
51 |
52 | trait EdgeAddedToNonExistentVertexHandler[@specialized(Int, Long) Id, Signal] {
53 |
54 | /**
55 | * Sets the handler that gets triggered, when the vertex to which an edge should be added does not exist.
56 | * Optionally returns the vertex that should be created and to whioch the edge can then be added.
57 | *
58 | * @note By default an exception is thrown when an edge cannot be added. The handler function
59 | * receives the edge, the id of the vertex that does not exist and an instance of GraphEditor as parameters in order to
60 | * potentially create a vertex to which the edge should be added.
61 | */
62 | def handleImpossibleEdgeAddition(edge: Edge[Id], vertexId: Id, graphEditor: GraphEditor[Id, Signal]): Option[Vertex[Id, _, Id, Signal]]
63 |
64 | }
65 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Logger.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import akka.actor.{ Actor, ActorLogging }
23 | import org.json4s.JValue
24 |
25 | trait Logger {
26 | def getLogMessages: List[JValue]
27 | }
28 |
29 | trait ActorRestartLogging {
30 | self: Actor with ActorLogging =>
31 |
32 | override def preRestart(t: Throwable, message: Option[Any]) {
33 | val msg = s"Unhandled error: $message"
34 | log.error(t, msg)
35 | t.printStackTrace
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Node.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Thomas Keller
4 | *
5 | * Copyright 2012 University of Zurich
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.interfaces
22 |
23 | import com.signalcollect.worker.AkkaWorker
24 |
25 | // Has to be a trait to be proxied.
26 | trait Node[Id, Signal] {
27 | def createWorker(workerId: Int, creator: () => AkkaWorker[Id, Signal]): String // string = remote actor address
28 | def initializeMessageBus(numberOfWorkers: Int, numberOfNodes: Int, messageBusFactory: MessageBusFactory[Id, Signal], mapperFactory: MapperFactory[Id])
29 | def initializeIdleDetection(): Unit
30 | def numberOfCores(): Int
31 | def shutdown(): Unit
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/NodeActor.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import akka.actor.Actor
23 |
24 | // Has to be a trait to be proxied.
25 | trait NodeActor[Id, Signal] extends Node[Id, Signal]
26 | with Actor
27 | with MessageRecipientRegistry
28 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Scheduler.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import com.signalcollect.Vertex
23 |
24 | abstract class Scheduler[Id, Signal](val worker: Worker[Id, Signal]) {
25 | def executeOperations(systemOverloaded: Boolean)
26 | def handleCollectOnDelivery(v: Vertex[Id, _, Id, Signal])
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Storage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Daniel Strebel
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.interfaces
20 |
21 | import com.signalcollect.Vertex
22 |
23 | /**
24 | * High level interface to abstract all vertex storage related implementations
25 | */
26 | trait Storage[@specialized(Int, Long) Id, Signal] {
27 | def vertices: VertexStore[Id, Signal]
28 | def toSignal: VertexStore[Id, Signal] //collection of all vertices that need to signal
29 | def toCollect: VertexStore[Id, Signal] //collection of all vertices that need to collect
30 | def updateStateOfVertex(vertex: Vertex[Id, _, Id, Signal]): Unit
31 | def close(): Unit
32 | }
33 |
34 | /**
35 | * Stores vertices and makes them retrievable through their associated id.
36 | */
37 | trait VertexStore[@specialized(Int, Long) Id, Signal] {
38 | def get(id: Id): Vertex[Id, _, Id, Signal]
39 | def put(vertex: Vertex[Id, _, Id, Signal]): Boolean
40 | def updateStateOfVertex(vertex: Vertex[Id, _, Id, Signal]): Unit
41 | def remove(id: Id): Unit
42 | def isEmpty: Boolean
43 | def size: Long
44 | def stream: Stream[Vertex[Id, _, Id, Signal]]
45 | def foreach(f: Vertex[Id, _, Id, Signal] => Unit): Unit
46 | def process(p: Vertex[Id, _, Id, Signal] => Unit, numberOfVertices: Option[Int] = None): Int
47 | def processWithCondition(p: Vertex[Id, _, Id, Signal] => Unit, breakCondition: () => Boolean): Int
48 | def close(): Unit
49 | }
50 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/VertexToWorkerMapper.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.interfaces
20 |
21 | trait VertexToWorkerMapper[@specialized(Int, Long) Id] {
22 | def getWorkerIdForVertexId(vertexId: Id): Int
23 | def getWorkerIdForVertexIdHash(vertexIdHash: Int): Int
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/Worker.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import com.signalcollect.Vertex
23 |
24 | trait Worker[Id, Signal]
25 | extends WorkerApi[Id, Signal]
26 | with MessageRecipientRegistry {
27 | def vertexStore: Storage[Id, Signal]
28 | def scheduler: Scheduler[Id, Signal]
29 | var messageBusFlushed: Boolean
30 | def executeCollectOperationOfVertex(vertex: Vertex[Id, _, Id, Signal], addToSignal: Boolean = true)
31 | def executeSignalOperationOfVertex(vertex: Vertex[Id, _, Id, Signal])
32 | def signalThreshold: Double
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/interfaces/WorkerApi.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.interfaces
21 |
22 | import com.signalcollect.Edge
23 | import com.signalcollect.GraphEditor
24 | import com.signalcollect.Vertex
25 |
26 | trait WorkerApi[Id, Signal] {
27 | def addVertex(vertex: Vertex[Id, _, Id, Signal]): Unit
28 | def addEdge(sourceId: Id, edge: Edge[Id]): Unit
29 | def removeVertex(vertexId: Id): Unit
30 | def removeEdge(edgeId: EdgeId[Id]): Unit
31 | def processSignalWithSourceId(signal: Signal, targetId: Id, sourceId: Id): Unit
32 | def processSignalWithoutSourceId(signal: Signal, targetId: Id): Unit
33 | def modifyGraph(graphModification: GraphEditor[Id, Signal] => Unit, vertexIdHint: Option[Id] = None): Unit
34 | def loadGraph(graphModifications: Iterator[GraphEditor[Id, Signal] => Unit], vertexIdHint: Option[Id] = None): Unit
35 |
36 | def setSignalThreshold(signalThreshold: Double): Unit
37 | def setCollectThreshold(collectThreshold: Double): Unit
38 |
39 | def recalculateScores(): Unit
40 | def recalculateScoresForVertexWithId(vertexId: Id): Unit
41 |
42 | def forVertexWithId[VertexType <: Vertex[Id, _, Id, Signal], ResultType](vertexId: Id, f: VertexType => ResultType): ResultType
43 | def foreachVertex(f: Vertex[Id, _, Id, Signal] => Unit): Unit
44 | def foreachVertexWithGraphEditor(f: GraphEditor[Id, Signal] => Vertex[Id, _, Id, Signal] => Unit): Unit
45 |
46 | def aggregateOnWorker[WorkerResult](aggregationOperation: ComplexAggregation[WorkerResult, _]): WorkerResult
47 | def aggregateAll[WorkerResult, EndResult](aggregationOperation: ComplexAggregation[WorkerResult, EndResult]): EndResult
48 |
49 | def pauseComputation(): Unit
50 | def startComputation(): Unit
51 |
52 | def signalStep(): Boolean // always returns true, just to make it blocking.
53 | def collectStep(): Boolean
54 |
55 | def getWorkerStatistics(): WorkerStatistics
56 | def getIndividualWorkerStatistics(): List[WorkerStatistics]
57 |
58 | def reset(): Unit
59 | def shutdown(): Unit
60 |
61 | def initializeIdleDetection(): Unit
62 |
63 | //TODO: Implement system information accessors on node instead.
64 | def getNodeStatistics(): NodeStatistics
65 | def getIndividualNodeStatistics(): List[NodeStatistics]
66 |
67 | /**
68 | * Creates a snapshot of all the vertices in all workers.
69 | * Does not store the toSignal/toCollect collections or pending messages.
70 | * Should only be used when the workers are idle.
71 | * Overwrites any previous snapshot that might exist.
72 | */
73 | def snapshot(): Unit
74 |
75 | /**
76 | * Restores the last snapshot of all the vertices in all workers.
77 | * Does not store the toSignal/toCollect collections or pending messages.
78 | * Should only be used when the workers are idle.
79 | */
80 | def restore(): Unit
81 |
82 | /**
83 | * Deletes the worker snapshots if they exist.
84 | */
85 | def deleteSnapshot(): Unit
86 |
87 | }
88 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/loading/AdjacencyListLoader.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.loading
20 |
21 | import java.io.InputStream
22 |
23 | import com.signalcollect.{ GraphEditor, Vertex }
24 | import com.signalcollect.util.FileReader
25 |
26 | /**
27 | * Loads a graph from an adjacency list format.
28 | * The data format is: vertexId #outEdges outId1 outId1
29 | *
30 | * @param filePath The path of the file that should be loaded.
31 | * @param combinedVertexBuilder The function that creates a vertex from an ID and an adjacency array.
32 | * @example An example vertex descritpion for a vertex with ID 1 and two out edges
33 | * to vertices with IDs 5 and 6:
34 | * 1 2 5 6
35 | * @example The loader is used like this: graphEditor.loadGraph(AdjacencyListLoader[Double]("./testfile", myVertexBuilder))
36 | * @note All numbers have to be in ASCII/UTF-8 format.
37 | * @note Each vertex is on its own line.
38 | * @note The vertex ids have to be positive Ints.
39 | */
40 | case class AdjacencyListLoader[SignalType](
41 | inputStream: () => InputStream, combinedVertexBuilder: (Int, Array[Int]) => Vertex[Int, _, Int, SignalType])
42 | extends Iterator[GraphEditor[Int, SignalType] => Unit] {
43 |
44 | var intIterator: Iterator[Int] = _
45 |
46 | var isInitialized = false
47 |
48 | protected def readNextVertex(): Vertex[Int, _, Int, SignalType] = {
49 | if (intIterator.hasNext) {
50 | val id = intIterator.next
51 | val numberOfLinks = intIterator.next
52 | val outlinks = new Array[Int](numberOfLinks)
53 | var i = 0
54 | while (i < numberOfLinks) {
55 | outlinks(i) = intIterator.next()
56 | i += 1
57 | }
58 | combinedVertexBuilder(id, outlinks)
59 | } else {
60 | null.asInstanceOf[Vertex[Int, _, Int, SignalType]]
61 | }
62 | }
63 |
64 | var nextVertex: Vertex[Int, _, Int, SignalType] = null
65 |
66 | def initialize(): Unit = {
67 | intIterator = FileReader.intIterator(inputStream())
68 | isInitialized = true
69 | nextVertex = readNextVertex
70 | }
71 |
72 | def hasNext: Boolean = {
73 | if (!isInitialized) {
74 | initialize
75 | }
76 | nextVertex != null
77 | }
78 |
79 | def next(): GraphEditor[Int, SignalType] => Unit = {
80 | if (!isInitialized) {
81 | initialize()
82 | }
83 | if (nextVertex == null) {
84 | throw new Exception("next was called when hasNext is false.")
85 | }
86 | val v = nextVertex // This is actually important, so the closure doesn't capture the mutable var.
87 | val loader: GraphEditor[Int, SignalType] => Unit = { ge =>
88 | ge.addVertex(v)
89 | }
90 | nextVertex = readNextVertex
91 | loader
92 | }
93 |
94 | }
95 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/loading/Loading.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.loading
20 |
21 | import com.signalcollect.GraphEditor
22 | import com.signalcollect.Vertex
23 | import com.signalcollect.util.FileReader
24 | import java.io.FileInputStream
25 | import java.io.InputStream
26 |
27 | object Loading {
28 |
29 | /**
30 | * Returns a modification iterator that can be used with Graph.loadGraph
31 | *
32 | * @param vertexDataIterator Iterator of vertex ID, edges IDs tuples.
33 | * @param creator Vertex creator that creates a vertex from an (ID, target IDs) tuple.
34 | */
35 | @inline def loader[Id, Signal](
36 | vertexDataIterator: Iterator[(Id, List[Id])],
37 | creator: (Id, List[Id]) => Vertex[Id, _, Id, Signal]): Iterator[GraphEditor[Id, Signal] => Unit] = {
38 | vertexDataIterator.map { vData: (Id, List[Id]) =>
39 | (ge: GraphEditor[Id, Signal]) =>
40 | val v = creator(vData._1, vData._2)
41 | ge.addVertex(v)
42 | }
43 | }
44 |
45 | /**
46 | * Returns an iterator that transforms a list of ASCII encoded Int Int tuples that are ordered by source ID into
47 | * a tuple of the source vertex ID with a list of the respective target IDs.
48 | *
49 | * @note Negative numbers are unsupported.
50 | */
51 | @inline def intEdgeIdsOrderedBySourceId(inputStream: InputStream): Iterator[(Int, List[Int])] = {
52 | new VertexTupleIterator(FileReader.intIterator(inputStream))
53 | }
54 |
55 | }
56 |
57 | /**
58 | * This iterator takes an edge ID iterator that is ordered by source ID and
59 | * it allows to iterate over the source (vertex ID, edge list) tuples.
60 | */
61 | final class VertexTupleIterator[Id](val ids: Iterator[Id]) extends Iterator[(Id, List[Id])] {
62 | private var nextSourceId: Option[Id] = None
63 |
64 | @inline def hasNext: Boolean = {
65 | if (nextSourceId.isDefined) {
66 | true
67 | } else if (ids.hasNext) {
68 | val nextId = ids.next
69 | nextSourceId = Some(nextId)
70 | true
71 | } else {
72 | false
73 | }
74 | }
75 |
76 | @inline def next: (Id, List[Id]) = {
77 | val sourceId = nextSourceId.get
78 | nextSourceId = None
79 | var targetIds: List[Id] = Nil
80 | var nextIdIsTargetId = true
81 | var done = false
82 | assert(ids.hasNext, s"There was no target ID for edge with source ID $sourceId")
83 | while (ids.hasNext && !done) {
84 | val id = ids.next
85 | if (nextIdIsTargetId) {
86 | targetIds = id :: targetIds
87 | nextIdIsTargetId = false
88 | } else if (id != sourceId) {
89 | done = true
90 | nextSourceId = Some(id)
91 | } else {
92 | nextIdIsTargetId = true
93 | }
94 | }
95 | (sourceId, targetIds)
96 | }
97 |
98 | }
99 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/messaging/DefaultMessageBus.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.messaging
21 |
22 | import com.signalcollect.interfaces.WorkerApiFactory
23 | import com.signalcollect.interfaces.MessageBus
24 | import com.signalcollect.interfaces.VertexToWorkerMapper
25 | import akka.actor.ActorSystem
26 |
27 | class DefaultMessageBus[Id, Signal](
28 | val system: ActorSystem,
29 | val numberOfWorkers: Int,
30 | val numberOfNodes: Int,
31 | val mapper: VertexToWorkerMapper[Id],
32 | val sendCountIncrementorForRequests: MessageBus[_, _] => Unit,
33 | workerApiFactory: WorkerApiFactory[Id, Signal]) extends AbstractMessageBus[Id, Signal] {
34 | lazy val workerApi = workerApiFactory.createInstance(workerProxies, mapper)
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/messaging/DefaultVertexToWorkerMapper.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2010 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.messaging
21 |
22 | import com.signalcollect.interfaces.VertexToWorkerMapper
23 |
24 | class DefaultVertexToWorkerMapper[Id](numberOfNodes: Int, workersPerNode: Int) extends VertexToWorkerMapper[Id] with Serializable {
25 |
26 | val numberOfWorkers = numberOfNodes * workersPerNode
27 |
28 | @inline final def getWorkerIdForVertexId(vertexId: Id): Int = {
29 | (vertexId.hashCode & Int.MaxValue) % numberOfWorkers
30 | }
31 |
32 | // def getWorkerIdForVertexIdHash(vertexIdHash: Int): Int = (vertexIdHash % numberOfWorkers).abs (BAD PERFORMANCE!)
33 | @inline final def getWorkerIdForVertexIdHash(vertexIdHash: Int): Int = {
34 | (vertexIdHash & Int.MaxValue) % numberOfWorkers
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/node/WorkersOnNodeIdleDetector.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.node
21 |
22 | import com.signalcollect.interfaces.MessageBus
23 | import com.signalcollect.interfaces.BulkStatus
24 | import com.signalcollect.interfaces.WorkerStatus
25 |
26 | class WorkersOnNodeIdleDetector(
27 | val nodeId: Int,
28 | val workersOnNode: Int,
29 | val messageBus: MessageBus[_, _]) {
30 |
31 | val workerStatus = new Array[WorkerStatus](workersOnNode)
32 | val isWorkerIdle = new Array[Boolean](workersOnNode)
33 | val workerStatusAlreadyForwarded = new Array[Boolean](workersOnNode)
34 |
35 | var numberOfIdleWorkers = 0
36 | var numberOfStatsThatNeedForwarding = 0
37 | var allIdleReported = false
38 |
39 | def receivedWorkerStatus(w: WorkerStatus) {
40 | val arrayIndex = w.workerId % workersOnNode
41 | if (isWorkerIdle(arrayIndex)) {
42 | if (!w.isIdle) {
43 | numberOfIdleWorkers -= 1
44 | }
45 | } else {
46 | if (w.isIdle) {
47 | numberOfIdleWorkers += 1
48 | }
49 | }
50 | if (workerStatusAlreadyForwarded(arrayIndex) || workerStatus(arrayIndex) == null) {
51 | // Only increase if there was no message there or if the message that will be replaced had already been forwarded.
52 | numberOfStatsThatNeedForwarding += 1
53 | }
54 | workerStatus(arrayIndex) = w
55 | isWorkerIdle(arrayIndex) = w.isIdle
56 | workerStatusAlreadyForwarded(arrayIndex) = false
57 |
58 | val nodeIsIdle = (numberOfIdleWorkers == workersOnNode)
59 | if (nodeIsIdle || allIdleReported) {
60 | val workerStats = new Array[WorkerStatus](numberOfStatsThatNeedForwarding)
61 | var i = 0
62 | var workerStatsIndex = 0
63 | while (i < workersOnNode) {
64 | if (!workerStatusAlreadyForwarded(i)) {
65 | val status = workerStatus(i)
66 | if (status != null) {
67 | workerStats(workerStatsIndex) = status
68 | workerStatsIndex += 1
69 | }
70 | }
71 | workerStatusAlreadyForwarded(i) = true
72 | i += 1
73 | }
74 | val bulkStatus = BulkStatus(nodeId, workerStats)
75 | messageBus.sendToNodeUncounted(nodeId / 2, bulkStatus)
76 | numberOfStatsThatNeedForwarding = 0
77 | allIdleReported = nodeIsIdle
78 | }
79 | }
80 |
81 | }
82 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/nodeprovisioning/NodeProvisioner.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Thomas Keller
4 | *
5 | * Copyright 2012 University of Zurich
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.nodeprovisioning
22 |
23 | import com.typesafe.config.Config
24 | import akka.actor.ActorRef
25 | import akka.actor.ActorSystem
26 |
27 | trait NodeProvisioner[Id, Signal] extends Serializable {
28 | def numberOfNodes: Int
29 | def getNodes(localSystem: ActorSystem, actorNamePrefix: String, akkaConfig: Config): Array[ActorRef]
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/nodeprovisioning/cluster/ClusterNodeEntryPointTemplate.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Bharath Kumar
4 | *
5 | * Copyright 2015 iHealth Technologies
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.nodeprovisioning.cluster
22 |
23 | import com.typesafe.config.{ Config, ConfigFactory }
24 | import akka.actor.ActorSystem
25 | import com.signalcollect.configuration.Akka
26 | import scala.concurrent.Await
27 | import scala.concurrent.duration.Duration
28 |
29 | object ClusterNodeEntryPointTemplate {
30 |
31 | start()
32 |
33 | def config: Config = Akka.config(
34 | serializeMessages = None,
35 | loggingLevel = None,
36 | kryoRegistrations = List.empty,
37 | kryoInitializer = None)
38 |
39 | def systemName: String = "SignalCollect"
40 |
41 | def start(): Unit = {
42 | val system = ActorSystem(systemName, config)
43 | Await.ready(system.whenTerminated, Duration.Inf)
44 | }
45 |
46 | }
47 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/nodeprovisioning/cluster/ClusterNodeProvisioner.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Bharath Kumar
4 | *
5 | * Copyright 2015 iHealth Technologies
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.nodeprovisioning.cluster
22 |
23 | import scala.concurrent.Await
24 | import scala.concurrent.duration.DurationInt
25 |
26 | import com.signalcollect.nodeprovisioning.NodeProvisioner
27 | import com.typesafe.config.{ Config, ConfigFactory }
28 |
29 | import akka.actor.{ ActorRef, ActorSystem, Props }
30 | import akka.pattern.ask
31 | import akka.util.Timeout
32 |
33 | class ClusterNodeProvisioner[Id, Signal](val numberOfNodes: Int) extends NodeProvisioner[Id, Signal] {
34 |
35 | def getNodes(system: ActorSystem, actorNamePrefix: String, akkaConfig: Config): Array[ActorRef] = {
36 | implicit val timeout = Timeout(300.seconds)
37 | //TODO: Is this coming from some config?
38 | val idleDetectionPropagationDelayInMilliseconds = 500
39 | val provisioner = system.actorOf(Props(classOf[ClusterNodeProvisionerActor],
40 | idleDetectionPropagationDelayInMilliseconds, actorNamePrefix, numberOfNodes),
41 | actorNamePrefix + "ClusterNodeProvisionerActor")
42 | val nodeActorsFuture = provisioner ? RetrieveNodeActors
43 | val nodeActors = Await.result(nodeActorsFuture, timeout.duration)
44 | nodeActors match {
45 | case n: Array[ActorRef] => n
46 | case _ => throw new Exception(s"Expected an array of actor references, instead got $nodeActors.")
47 | }
48 | }
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/nodeprovisioning/cluster/ClusterNodeProvisionerActor.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Bharath Kumar
4 | *
5 | * Copyright 2015 iHealth Technologies
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.nodeprovisioning.cluster
22 |
23 | import com.signalcollect.node.DefaultNodeActor
24 | import akka.actor.{ Actor, ActorLogging, ActorRef, Address, Deploy, Props, actorRef2Scala }
25 | import akka.cluster.Cluster
26 | import akka.cluster.ClusterEvent.{ InitialStateAsEvents, MemberUp }
27 | import akka.remote.RemoteScope
28 | import akka.actor.PoisonPill
29 |
30 | case object RetrieveNodeActors
31 |
32 | class ClusterNodeProvisionerActor(
33 | idleDetectionPropagationDelayInMilliseconds: Int,
34 | actorNamePrefix: String,
35 | numberOfNodes: Int) extends Actor with ActorLogging {
36 |
37 | val cluster = Cluster(context.system)
38 |
39 | val masterNodeActor = startNodeActor(cluster.selfAddress, 0)
40 |
41 | override def preStart(): Unit = {
42 | if (numberOfNodes > 1) {
43 | cluster.subscribe(self, initialStateMode = InitialStateAsEvents, classOf[MemberUp])
44 | }
45 | }
46 |
47 | private[this] var nodeActorArrayRequestor: Option[ActorRef] = None
48 |
49 | private[this] var nodeActors: Map[Int, ActorRef] = Map(0 -> masterNodeActor)
50 |
51 | private[this] var nextNodeActorId = 1
52 |
53 | override def receive: Receive = {
54 | case RetrieveNodeActors =>
55 | assert(nodeActorArrayRequestor.isEmpty, "Cannot have more than one requestor for node actors.")
56 | nodeActorArrayRequestor = Some(sender)
57 | reportNodeActors
58 | case MemberUp(m) =>
59 | log.info(s"Cluster provisioner received member up message for {}.", m)
60 | if (m.address != cluster.selfAddress) {
61 | val nodeActor = startNodeActor(m.address, nextNodeActorId)
62 | nodeActors += nextNodeActorId -> nodeActor
63 | nextNodeActorId += 1
64 | reportNodeActors
65 | }
66 | }
67 |
68 | def reportNodeActors(): Unit = {
69 | if (nodeActors.size == numberOfNodes && nodeActorArrayRequestor.isDefined) {
70 | assert((0 until numberOfNodes).toSet == nodeActors.keys.toSet)
71 | val nodeActorArray = new Array[ActorRef](numberOfNodes)
72 | nodeActors.foreach { case (k, v) => nodeActorArray(k) = v }
73 | nodeActorArrayRequestor.foreach { _ ! nodeActorArray }
74 | log.info(s"All expected {} nodes are up and the respective node actors were created.", numberOfNodes)
75 | cluster.unsubscribe(self)
76 | self ! PoisonPill
77 | }
78 | }
79 |
80 | def startNodeActor(address: Address, nodeActorId: Int): ActorRef = {
81 | val nodeController = context.system.actorOf(
82 | Props(classOf[DefaultNodeActor[Long, Any]],
83 | actorNamePrefix,
84 | nodeActorId,
85 | numberOfNodes,
86 | None,
87 | idleDetectionPropagationDelayInMilliseconds,
88 | None).withDeploy(Deploy(scope = RemoteScope(address))),
89 | name = actorNamePrefix + "DefaultNodeActor" + nodeActorId.toString)
90 | nodeController
91 | }
92 |
93 | }
94 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/nodeprovisioning/cluster/ClusterNodeShutdownHandlerActor.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2015 iHealth Technologies
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.nodeprovisioning.cluster
21 |
22 | import akka.actor.{ Actor, ActorLogging }
23 | import akka.cluster.Cluster
24 | import akka.cluster.ClusterEvent.{ InitialStateAsEvents, MemberExited }
25 |
26 | class ClusterNodeShutdownHandlerActor extends Actor with ActorLogging {
27 |
28 | val cluster = Cluster(context.system)
29 |
30 | override def preStart(): Unit = {
31 | cluster.subscribe(self, initialStateMode = InitialStateAsEvents, classOf[MemberExited])
32 | }
33 |
34 | override def receive: Receive = {
35 | case MemberExited(m) =>
36 | log.info(s"Cluster node received member exited message for {} and is shutting down as well.", m)
37 | cluster.down(cluster.selfAddress)
38 | context.system.terminate()
39 | }
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/scheduler/LowLatencyScheduler.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.scheduler
21 |
22 | import com.signalcollect.Vertex
23 | import com.signalcollect.interfaces.Scheduler
24 | import com.signalcollect.interfaces.Worker
25 |
26 | class LowLatencyScheduler[Id, Signal](w: Worker[Id, Signal]) extends Scheduler[Id, Signal](w) {
27 | override def executeOperations(systemOverloaded: Boolean) {
28 | if (!worker.vertexStore.toCollect.isEmpty) {
29 | val collected = worker.vertexStore.toCollect.process(
30 | vertex => {
31 | worker.executeCollectOperationOfVertex(vertex, addToSignal = false)
32 | if (vertex.scoreSignal > worker.signalThreshold) {
33 | if (systemOverloaded) {
34 | worker.vertexStore.toSignal.put(vertex)
35 | } else {
36 | worker.executeSignalOperationOfVertex(vertex)
37 | }
38 | }
39 | })
40 | worker.messageBusFlushed = false
41 | }
42 | if (!systemOverloaded && !worker.vertexStore.toSignal.isEmpty) {
43 | worker.vertexStore.toSignal.process(worker.executeSignalOperationOfVertex(_))
44 | worker.messageBusFlushed = false
45 | }
46 | }
47 |
48 | // "worker.messageBusFlushed" = false will be called later anyway
49 | override def handleCollectOnDelivery(v: Vertex[Id, _, Id, Signal]) {
50 | worker.executeSignalOperationOfVertex(v)
51 | }
52 |
53 | }
54 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/scheduler/ThroughputScheduler.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.scheduler
21 |
22 | import com.signalcollect.Vertex
23 | import com.signalcollect.interfaces.Scheduler
24 | import com.signalcollect.interfaces.Worker
25 |
26 | class ThroughputScheduler[Id, Signal](
27 | w: Worker[Id, Signal],
28 | val batchSize: Int = 10000)
29 | extends Scheduler[Id, Signal](w) {
30 |
31 | override def executeOperations(systemOverloaded: Boolean) {
32 | if (!worker.vertexStore.toCollect.isEmpty) {
33 | val collected = worker.vertexStore.toCollect.process(
34 | vertex => {
35 | worker.executeCollectOperationOfVertex(vertex, addToSignal = false)
36 | if (vertex.scoreSignal > worker.signalThreshold) {
37 | if (systemOverloaded) {
38 | worker.vertexStore.toSignal.put(vertex)
39 | } else {
40 | worker.executeSignalOperationOfVertex(vertex)
41 | }
42 | }
43 | }, Some(batchSize))
44 | worker.messageBusFlushed = false
45 | }
46 | if (!systemOverloaded && !worker.vertexStore.toSignal.isEmpty) {
47 | worker.vertexStore.toSignal.process(worker.executeSignalOperationOfVertex(_), Some(batchSize))
48 | worker.messageBusFlushed = false
49 | }
50 | }
51 |
52 | override def handleCollectOnDelivery(v: Vertex[Id, _, Id, Signal]) {
53 | worker.vertexStore.toSignal.put(v)
54 | }
55 |
56 | }
57 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/serialization/DefaultSerializer.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Daniel Strebel
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.serialization
20 |
21 | import java.io.ByteArrayInputStream
22 | import java.io.ByteArrayOutputStream
23 | import java.io.ObjectInputStream
24 | import java.io.ObjectOutputStream
25 |
26 | /**
27 | * Default Serializer that uses standard java.io ObjectIn- and ObjectOutputStreams
28 | * and can serialize any object declared as serializable.
29 | */
30 | object DefaultSerializer {
31 |
32 | /**
33 | * Serializes an object.
34 | *
35 | * @param inputObject the object to serialize
36 | * @return serialized object as byte array
37 | */
38 | def write[A](inputObject: A): Array[Byte] = {
39 | val barr = new ByteArrayOutputStream(8192)
40 | val out = new ObjectOutputStream(barr)
41 | out.writeObject(inputObject)
42 | out.close
43 | barr.toByteArray
44 | }
45 |
46 | /**
47 | * Deserialize an object.
48 | *
49 | * @param the serialized object as byte array
50 | * @return the deserialized object
51 | */
52 | def read[A](buffer: Array[Byte]): A = {
53 | val input = new ObjectInputStream(new ByteArrayInputStream(buffer))
54 | val obj = input.readObject
55 | input.close
56 | obj.asInstanceOf[A]
57 | }
58 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/storage/JavaMapVertexStorage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.storage
20 |
21 | import com.signalcollect.interfaces.Storage
22 | import com.signalcollect.interfaces.VertexStore
23 | import com.signalcollect.Vertex
24 |
25 | class JavaMapVertexStorage[Id, Signal] extends Storage[Id, Signal] {
26 | val vertices = vertexStoreFactory
27 | protected def vertexStoreFactory: VertexStore[Id, Signal] = new JavaVertexMap[Id, Signal]
28 | def updateStateOfVertex(vertex: Vertex[Id, _, Id, Signal]): Unit = Unit
29 | def close(): Unit = Unit
30 | val toCollect = vertexSignalFactory //holds all signals that are not collected yet
31 | protected def vertexSignalFactory = new JavaVertexMap[Id, Signal]
32 | val toSignal = vertexSetFactory //holds all vertex ids that need to signal
33 | protected def vertexSetFactory = new JavaVertexMap[Id, Signal]
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/storage/MixedVertexStorage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Daniel Strebel
3 | * @author Philip Stutz
4 | *
5 | * Copyright 2013 University of Zurich
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | */
19 |
20 | package com.signalcollect.storage
21 |
22 | import com.signalcollect.interfaces.Storage
23 | import com.signalcollect.interfaces.VertexStore
24 | import com.signalcollect.Vertex
25 |
26 | /**
27 | * Storage backed by a custom-tailored open hash map implementation for vertices.
28 | */
29 | class MixedVertexStorage[Id, Signal] extends Storage[Id, Signal] {
30 |
31 | def updateStateOfVertex(vertex: Vertex[Id, _, Id, Signal]): Unit = Unit
32 | def close(): Unit = Unit
33 |
34 | val vertices = vertexStoreFactory
35 | protected def vertexStoreFactory: VertexStore[Id, Signal] = new VertexMap[Id, Signal](initialSize = 32768, rehashFraction = .8f)
36 |
37 | val toCollect = vertexSignalFactory //holds all signals that are not collected yet
38 | protected def vertexSignalFactory = new JavaVertexMap[Id, Signal]
39 | val toSignal = vertexSetFactory //holds all vertex ids that need to signal
40 | protected def vertexSetFactory = new JavaVertexMap[Id, Signal]
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/storage/VertexMapStorage.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Daniel Strebel
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.storage
20 |
21 | import com.signalcollect.interfaces.Storage
22 | import com.signalcollect.interfaces.VertexStore
23 | import com.signalcollect.Vertex
24 |
25 | object StorageDefaultValues {
26 | final val defaultInitialSize = 32768
27 | final val defaultRehashFraction = 0.5f
28 | final val defaultShrinkFraction = 0.05f
29 | final val defaultVertexMapRehashFraction = 0.8f
30 | final val minShrinkSize = 16384 // Will not shrink if the map capacity is at or below this.
31 | final val defaultTemporaryMapRehashFraction = 0.5f
32 | final val defaultToSignalInitialSize = 1024
33 | }
34 |
35 | /**
36 | * Storage backed by a custom-tailored open hash map implementation for vertices.
37 | */
38 | class VertexMapStorage[Id, Signal] extends Storage[Id, Signal] {
39 | import StorageDefaultValues._
40 |
41 | def updateStateOfVertex(vertex: Vertex[Id, _, Id, Signal]): Unit = Unit
42 | def close(): Unit = Unit
43 |
44 | val vertices = vertexStoreFactory
45 | protected def vertexStoreFactory: VertexStore[Id, Signal] = new VertexMap[Id, Signal](
46 | initialSize = defaultInitialSize, rehashFraction = defaultVertexMapRehashFraction)
47 |
48 | val toCollect = vertexSignalFactory //holds all signals that are not collected yet
49 | protected def vertexSignalFactory = new VertexMap[Id, Signal](
50 | initialSize = defaultInitialSize, rehashFraction = defaultTemporaryMapRehashFraction)
51 | val toSignal = vertexSetFactory //holds all vertex ids that need to signal
52 | protected def vertexSetFactory = new VertexMap[Id, Signal](
53 | initialSize = defaultToSignalInitialSize, rehashFraction = defaultTemporaryMapRehashFraction)
54 |
55 | }
56 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/util/AkkaRemoteAddress.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.util
21 |
22 | import akka.actor.ActorRef
23 | import akka.actor.ActorSystem
24 | import akka.actor.ExtendedActorSystem
25 |
26 | object AkkaRemoteAddress {
27 | def get(actorRef: ActorRef, system: ActorSystem): String = {
28 | val akkaSystemAddress = system.asInstanceOf[ExtendedActorSystem].provider.getDefaultAddress
29 | val remoteAddress = actorRef.path.toStringWithAddress(akkaSystemAddress)
30 | remoteAddress.toString
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/util/AkkaUtil.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import scala.concurrent.Await
22 | import scala.concurrent.duration.DurationInt
23 | import scala.language.postfixOps
24 |
25 | import akka.actor.ActorSelection
26 | import akka.util.Timeout
27 |
28 | object AkkaUtil {
29 | def getActorRefFromSelection(actorSel: ActorSelection) = {
30 | implicit val timeout = Timeout(30 seconds)
31 | val actorRef = Await.result(actorSel.resolveOne, 30 seconds)
32 | actorRef
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/util/IteratorConcatenator.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.util
21 |
22 | import scala.annotation.tailrec
23 | import scala.collection.mutable.Queue
24 |
25 | /**
26 | * To avoid https://issues.scala-lang.org/browse/SI-8428, which is not really fixed.
27 | *
28 | * Unfortunately I could not reproduce the problem outside of a large and complex TripleRush evaluation.
29 | */
30 | final class IteratorConcatenator[U] extends Iterator[U] {
31 |
32 | val iterators = new Queue[Iterator[U]]()
33 |
34 | def clear(): Unit = {
35 | iterators.clear
36 | }
37 |
38 | def appendIterator(i: Iterator[U]): Unit = {
39 | iterators.enqueue(i)
40 | }
41 |
42 | def next(): U = {
43 | iterators.head.next
44 | }
45 |
46 | @tailrec def hasNext: Boolean = {
47 | if (iterators.isEmpty) {
48 | false
49 | } else {
50 | val headHasNext = iterators.head.hasNext
51 | if (!headHasNext) {
52 | iterators.dequeue
53 | hasNext
54 | } else {
55 | true
56 | }
57 | }
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/util/MemoryEfficientSplayIntSet.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.util
21 |
22 | final class MemoryEfficientSplayIntSet extends SplayIntSet {
23 | def overheadFraction = 0.01f
24 | def maxNodeIntSetSize = 1000
25 | }
26 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/util/RandomString.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import scala.util.Random
22 |
23 | object RandomString {
24 | def alpha = "abcdefghijklmnopqrstuvwxyz"
25 | def numeric = "0123456789"
26 | def alphaNumeric = alpha + numeric
27 |
28 | def generate(length: Int, characters: String = alpha): String = {
29 | def randomIndex = Random.nextInt(characters.length)
30 | def randomChar = characters(randomIndex)
31 | val sb = new StringBuilder(length)
32 | for (i <- 0 until length) {
33 | sb.append(randomChar)
34 | }
35 | sb.toString
36 | }
37 |
38 | }
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/util/SearchableIntSet.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import scala.collection.mutable.Buffer
22 |
23 | class SearchableIntSet(val sorted: Array[Int]) extends AnyVal {
24 |
25 | def toBuffer: Buffer[Int] = sorted.toBuffer
26 | def toList: List[Int] = sorted.toList
27 | def toSet: Set[Int] = sorted.toSet
28 |
29 | @inline def foreach(f: Int => Unit) = {
30 | var i = 0
31 | while (i < sorted.length) {
32 | f(sorted(i))
33 | i += 1
34 | }
35 | }
36 |
37 | /**
38 | * Inserts item into the searchable int set.
39 | * If the item was contained already, returns the same array again (reference equal).
40 | */
41 | def insert(item: Int): Array[Int] = {
42 | val index = insertIndex(item)
43 | if (index < sorted.length && sorted(index) == item) {
44 | return sorted
45 | } else {
46 | val newArray = new Array[Int](sorted.length + 1)
47 | System.arraycopy(sorted, 0, newArray, 0, index)
48 | newArray(index) = item
49 | System.arraycopy(sorted, index, newArray, index + 1, sorted.length - index)
50 | return newArray
51 | }
52 | }
53 |
54 | /**
55 | * Determines the index at which 'item' should be inserted.
56 | */
57 | def insertIndex(item: Int): Int = {
58 | var lower = 0
59 | var upper = sorted.length - 1
60 | while (lower <= upper) {
61 | val mid = lower + (upper - lower) / 2
62 | val midItem = sorted(mid)
63 | if (midItem < item) {
64 | lower = mid + 1
65 | } else if (midItem > item) {
66 | upper = mid - 1
67 | } else {
68 | return mid
69 | }
70 | }
71 | lower + (upper - lower) / 2
72 | }
73 |
74 | /**
75 | * Checks if `item` is contained by using binary search.
76 | */
77 | def contains(item: Int): Boolean = {
78 | // Arrays.binarySearch(childDeltasOptimized, toFind) >= 0
79 | var lower = 0
80 | var upper = sorted.length - 1
81 | while (lower <= upper) {
82 | val mid = lower + (upper - lower) / 2
83 | val midItem = sorted(mid)
84 | if (midItem < item) {
85 | lower = mid + 1
86 | } else if (midItem > item) {
87 | upper = mid - 1
88 | } else {
89 | return true
90 | }
91 | }
92 | false
93 | }
94 |
95 | }
96 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/util/Verifier.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.util
21 |
22 | import java.lang.reflect.InvocationHandler
23 | import java.lang.reflect.Method
24 | import java.lang.reflect.Proxy
25 |
26 | import scala.reflect.ClassTag
27 | import scala.reflect.classTag
28 |
29 | import com.signalcollect.messaging.Command
30 |
31 | object Verifier {
32 | def create[T: ClassTag](reference: T, alternative: T): T = {
33 | val c = classTag[T].runtimeClass
34 | Proxy.newProxyInstance(
35 | c.getClassLoader,
36 | Array[Class[_]](c),
37 | new Verifier[T](reference, alternative)).asInstanceOf[T]
38 | }
39 | }
40 |
41 | /**
42 | * The verifier ensures that an alternative implementation of an
43 | * interface 'I' produces the same results as the reference implementation.
44 | * This class will behave like the reference implementation, but when the
45 | * alternative implementation behaves differently, it will throw an exception
46 | * and print the call history that led to the different behaviour.
47 | *
48 | * @note Create by calling Verifier.create(reference, alternative)
49 | * @note The return values have to be comparabl with a Scala equals comparison.
50 | */
51 | class Verifier[I](referenceImplementation: I, alternativeImplementation: I) extends InvocationHandler {
52 |
53 | var commandHistory = List.empty[Command[I]]
54 |
55 | def invoke(proxy: Object, method: Method, arguments: Array[Object]) = {
56 | val command = new Command[I](method.getDeclaringClass.getName, method.toString, arguments)
57 | commandHistory = command :: commandHistory
58 | val referenceStateBefore = referenceImplementation.toString
59 | val alternativeStateBefore = alternativeImplementation.toString
60 | val referenceResult = command(referenceImplementation)
61 | val alternativeResult = command(alternativeImplementation)
62 | if (referenceResult != alternativeResult) {
63 | throw new Exception(s"Verifier error: Alternative implementation returned $alternativeResult, should have returned $referenceResult.\n" +
64 | s"Reference impl. before call: ${referenceStateBefore}\n" +
65 | s"Alternative impl. before call: ${alternativeStateBefore}\n" +
66 | s"Command history: ${commandHistory.mkString("\n")}")
67 | }
68 | referenceResult
69 | }
70 |
71 | }
72 |
--------------------------------------------------------------------------------
/src/main/scala/com/signalcollect/worker/WorkerOperationCounters.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * @author Francisco de Freitas
5 | * @author Daniel Strebel
6 | *
7 | * Copyright 2012 University of Zurich
8 | *
9 | * Licensed under the Apache License, Version 2.0 (the "License");
10 | * you may not use this file except in compliance with the License.
11 | * You may obtain a copy of the License at
12 | *
13 | * http://www.apache.org/licenses/LICENSE-2.0
14 | *
15 | * Unless required by applicable law or agreed to in writing, software
16 | * distributed under the License is distributed on an "AS IS" BASIS,
17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 | * See the License for the specific language governing permissions and
19 | * limitations under the License.
20 | *
21 | */
22 |
23 | package com.signalcollect.worker
24 |
25 | class WorkerOperationCounters(
26 | var collectOperationsExecuted: Long = 0l,
27 | var signalOperationsExecuted: Long = 0l,
28 | var verticesAdded: Long = 0l,
29 | var verticesRemoved: Long = 0l,
30 | var outgoingEdgesAdded: Long = 0l,
31 | var outgoingEdgesRemoved: Long = 0l,
32 | var signalSteps: Long = 0l,
33 | var collectSteps: Long = 0l,
34 | var signalMessagesReceived: Long = 0l,
35 | var bulkSignalMessagesReceived: Long = 0l,
36 | var continueMessagesReceived: Long = 0l,
37 | var requestMessagesReceived: Long = 0l,
38 | var otherMessagesReceived: Long = 0) {
39 | // Resets operation counters but not messages received/sent counters.
40 | def resetOperationCounters(): Unit = {
41 | collectOperationsExecuted = 0l
42 | signalOperationsExecuted = 0l
43 | verticesAdded = 0l
44 | verticesRemoved = 0l
45 | outgoingEdgesAdded = 0l
46 | outgoingEdgesRemoved = 0l
47 | signalSteps = 0l
48 | collectSteps = 0l
49 | }
50 | // Only these messages are part of termination detection.
51 | def messagesReceived = signalMessagesReceived + bulkSignalMessagesReceived + requestMessagesReceived + otherMessagesReceived
52 | }
53 |
--------------------------------------------------------------------------------
/src/test/resources/com/signalcollect/loading/adjacency-list-format:
--------------------------------------------------------------------------------
1 | # test
2 | 1 0
3 | 4 1 5
4 | 2 3 1 5 4
5 | 5 0
--------------------------------------------------------------------------------
/src/test/resources/com/signalcollect/loading/ascii-ints.txt:
--------------------------------------------------------------------------------
1 | # test
2 | 1234 3525
3 | 123#test test
4 | 436 43663
5 | 33 44
6 | 0
7 |
8 |
--------------------------------------------------------------------------------
/src/test/resources/com/signalcollect/loading/ints-no-newline-at-end.txt:
--------------------------------------------------------------------------------
1 | 1 2 3 0
--------------------------------------------------------------------------------
/src/test/resources/com/signalcollect/loading/notredame2:
--------------------------------------------------------------------------------
1 | # Directed graph: notredame2
2 | # Kronecker Graph: seed matrix [0.999 0.414; 0.453 0.229]
3 | # Nodes: 4 Edges: 4
4 | # FromNodeId ToNodeId
5 | 0 0
6 | 0 1
7 | 1 0
8 | 3 2
9 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/AkkaProxySpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Thomas Keller
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import java.util.concurrent.CountDownLatch
23 |
24 | import org.scalatest.{ Finders, FlatSpec, Matchers }
25 |
26 | import akka.actor.{ Actor, Props, actorRef2Scala }
27 | import interfaces.Request
28 | import messaging.AkkaProxy
29 |
30 | class AkkaProxySpec extends FlatSpec with Matchers {
31 |
32 | "AkkaProxy" should "invoke blocking methods" in {
33 |
34 | object Counter {
35 | val latch = new CountDownLatch(3)
36 | }
37 |
38 | trait CommandExecutor extends Actor {
39 |
40 | def countDown(): Unit = {
41 | Counter.latch.countDown()
42 | }
43 |
44 | def receive = {
45 | case Request(command, reply, incrementor) =>
46 | try {
47 | val result = command.asInstanceOf[CommandExecutor => Any](this)
48 | if (reply) {
49 | if (result == null) {
50 | sender ! None
51 | } else {
52 | sender ! result
53 | }
54 | }
55 | } catch {
56 | case e: Exception =>
57 | throw e
58 | }
59 | }
60 | }
61 |
62 | val system = TestConfig.actorSystem("AkkaProxySpec")
63 | val executor = system.actorOf(Props(new Object with CommandExecutor))
64 | val proxy = AkkaProxy.newInstance[CommandExecutor](executor)
65 | proxy.countDown
66 | proxy.countDown
67 | proxy.countDown
68 | Counter.latch.await()
69 | system.terminate
70 | }
71 |
72 | }
73 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/BeforeRemovalSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2011 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect
21 |
22 | import org.scalatest.Matchers
23 | import org.scalatest.FlatSpec
24 |
25 | class BeforeRemovalSpec extends FlatSpec with Matchers {
26 |
27 | "Framework" should "call the beforeRemoval function of a vertex before removing it" in {
28 | val graph = TestConfig.graphProvider().build
29 | try {
30 | graph.addVertex(new BeforeRemovalVertex)
31 | graph.removeVertex(1)
32 | graph.execute
33 | RemovalDetector.beforeRemovalWorked should be(true)
34 | } finally {
35 | graph.shutdown
36 | }
37 | }
38 |
39 | }
40 |
41 | object RemovalDetector {
42 | var beforeRemovalWorked = false
43 | }
44 |
45 | class BeforeRemovalVertex extends DataGraphVertex(1, 0) {
46 | def collect = 0
47 | override def beforeRemoval(ge: GraphEditor[Any, Any]) = RemovalDetector.beforeRemovalWorked = true
48 | }
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/LatencySpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect
20 |
21 | import org.scalatest.FlatSpec
22 | import org.scalatest.ShouldMatchers
23 | import com.signalcollect.examples.Location
24 | import com.signalcollect.examples.Path
25 | import com.signalcollect.configuration.ExecutionMode
26 |
27 | class LatencySpec extends FlatSpec with ShouldMatchers {
28 |
29 | "Signal/Collect" should "terminate with a very low latency when an execution does nothing" in {
30 | val startTime = System.currentTimeMillis
31 | val g = TestConfig.graphProvider().withStatsReportingInterval(10000).build
32 | try {
33 | (1 to 50).foreach { i =>
34 | g.awaitIdle
35 | val v1 = new Location(1, Some(0))
36 | val v2 = new Location(2, None)
37 | g.addVertex(v1)
38 | g.addVertex(v2)
39 | g.addEdge(1, new Path(2))
40 | g.awaitIdle
41 | //println(g.execute(ExecutionConfiguration.withExecutionMode(ExecutionMode.Synchronous)))
42 | g.execute
43 | assert(v2.state == Some(1))
44 | g.reset
45 | }
46 | } finally {
47 | g.shutdown
48 | }
49 | val stopTime = System.currentTimeMillis
50 | val t = stopTime - startTime
51 | assert(t < 8000, s"Execution took $t milliseconds, should be less than 8 seconds.")
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/NonExistentVertexHandlerSpec.scala:
--------------------------------------------------------------------------------
1 | package com.signalcollect
2 |
3 | import org.scalatest.{ Finders, FlatSpec, Matchers }
4 |
5 | import com.signalcollect.examples.{ PageRankEdge, PageRankVertex }
6 | import com.signalcollect.interfaces.{ EdgeAddedToNonExistentVertexHandler, EdgeAddedToNonExistentVertexHandlerFactory }
7 |
8 | class TestEdgeAddedToNonExistentVertexHandlerFactory extends EdgeAddedToNonExistentVertexHandlerFactory[Any, Any] {
9 | def createInstance: EdgeAddedToNonExistentVertexHandler[Any, Any] =
10 | new TestEdgeAddedToNonExistentVertexHandler
11 | override def toString = "TestEdgeAddedToNonExistentVertexHandlerFactory"
12 | }
13 |
14 | class TestEdgeAddedToNonExistentVertexHandler extends EdgeAddedToNonExistentVertexHandler[Any, Any] {
15 | def handleImpossibleEdgeAddition(edge: Edge[Any], vertexId: Any, graphEditor: GraphEditor[Any, Any]): Option[Vertex[Any, _, Any, Any]] = {
16 | val v = new PageRankVertex[Any](vertexId)
17 | Some(v)
18 | }
19 | }
20 |
21 | class NonExistentVertexHandlerSpec extends FlatSpec with Matchers {
22 |
23 | "Handler for adding an edge to a nonexistent vertex" should "correctly create vertices if needed" in {
24 | val g = TestConfig.graphProvider()
25 | .withEdgeAddedToNonExistentVertexHandlerFactory(new TestEdgeAddedToNonExistentVertexHandlerFactory)
26 | .build
27 | try {
28 | g.addEdge(1, new PageRankEdge(2))
29 | g.addEdge(2, new PageRankEdge(1))
30 | val stats = g.execute
31 | assert(stats.aggregatedWorkerStatistics.numberOfVertices == 2)
32 | } finally {
33 | g.shutdown
34 | }
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/SerializationIntegrationSpec.scala:
--------------------------------------------------------------------------------
1 | package com.signalcollect
2 |
3 | import org.scalatest.Finders
4 | import org.scalatest.FlatSpec
5 | import org.scalatest.ShouldMatchers
6 | import org.scalatest.prop.Checkers
7 | import com.signalcollect.examples.PageRankEdge
8 | import com.signalcollect.examples.PageRankVertex
9 | import com.signalcollect.examples.EfficientPageRankVertex
10 | import com.signalcollect.examples.PlaceholderEdge
11 |
12 | class SerializationIntegrationSpec extends FlatSpec with ShouldMatchers with Checkers {
13 |
14 | "Kryo serialization" should "support running PageRank with message serialization" in {
15 | val system = TestConfig.actorSystem()
16 | val graph = new GraphBuilder[Int, Double]().
17 | withMessageSerialization(true).
18 | withActorSystem(system).withActorNamePrefix(TestConfig.prefix).
19 | build
20 | try {
21 | graph.addVertex(new EfficientPageRankVertex(1))
22 | graph.addVertex(new EfficientPageRankVertex(2))
23 | graph.addVertex(new EfficientPageRankVertex(3))
24 | graph.addEdge(1, new PlaceholderEdge(2))
25 | graph.addEdge(2, new PlaceholderEdge(1))
26 | graph.addEdge(2, new PlaceholderEdge(3))
27 | graph.addEdge(3, new PlaceholderEdge(2))
28 | val stats = graph.execute
29 | } finally {
30 | graph.shutdown
31 | }
32 | }
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/TestConfig.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Bharath Kumar
3 | *
4 | * Copyright 2015 iHealth Technologies
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect
20 |
21 | import java.net.ServerSocket
22 | import java.util.UUID
23 |
24 | import akka.actor.ActorSystem
25 | import com.typesafe.config.ConfigFactory
26 |
27 | object TestConfig {
28 | private[this] def seed(actorSystemName: String, seedPort: Int, seedIp: String = "127.0.0.1") = ConfigFactory.parseString(
29 | s"""akka.log-dead-letters-during-shutdown=off
30 | |akka.clustering.name=$actorSystemName
31 | |akka.clustering.seed-ip=$seedIp
32 | |akka.clustering.seed-port=$seedPort
33 | |akka.remote.netty.tcp.port=$seedPort
34 | |akka.cluster.seed-nodes=["akka.tcp://"${actorSystemName}"@"${seedIp}":"${seedPort}]""".stripMargin)
35 |
36 | def randomPort: Int = {
37 | val socket = new ServerSocket(0)
38 | val port = socket.getLocalPort
39 | socket.close()
40 | port
41 | }
42 |
43 | def actorSystem(name: String = UUID.randomUUID().toString.replace("-", ""), port: Int = randomPort) = {
44 | ActorSystem(name, seed(name, port).withFallback(ConfigFactory.load().getConfig("signalcollect")))
45 | }
46 |
47 | def prefix = UUID.randomUUID().toString.replace("-", "")
48 |
49 | def graphProvider(systemName: String = UUID.randomUUID().toString.replace("-", "")) = GraphBuilder
50 | .withActorSystem(actorSystem(systemName))
51 | .withActorNamePrefix(prefix)
52 | }
53 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/VertexSpec.scala:
--------------------------------------------------------------------------------
1 | package com.signalcollect
2 |
3 | import language.higherKinds
4 | import org.scalacheck.Arbitrary
5 | import org.scalacheck.Arbitrary._
6 | import org.scalacheck.Gen
7 | import org.scalacheck.Gen._
8 | import org.scalatest.FlatSpec
9 | import org.scalatest.ShouldMatchers
10 | import org.scalatest.prop.Checkers
11 | import org.scalatest.mock.EasyMockSugar
12 | import com.signalcollect.examples.PageRankVertex
13 | import com.signalcollect.examples.PageRankEdge
14 | import com.signalcollect.interfaces.SignalMessageWithSourceId
15 |
16 | class VertexSpec extends FlatSpec with ShouldMatchers with Checkers with EasyMockSugar {
17 |
18 | lazy val smallInt = Gen.chooseNum(0, 100)
19 | lazy val smallDouble = Gen.chooseNum(0.0, 10.0)
20 |
21 | lazy val signalMapEntry = for {
22 | k <- smallInt
23 | v <- smallDouble
24 | } yield (k, v)
25 |
26 | lazy val signalMap = containerOf[List, (Int, Double)](signalMapEntry).map(_.toMap)
27 |
28 | lazy val outEdgeIds = containerOf[Set, Int](smallInt)
29 |
30 | implicit def arbSignalMap = Arbitrary(signalMap)
31 |
32 | implicit def arbEdgeIds = Arbitrary(outEdgeIds)
33 |
34 | "PageRankVertex" should "correctly collect and signal" in {
35 | check(
36 | (incomingSignals: Map[Int, Double], outgoingEdges: Set[Int]) => {
37 | val id = "test"
38 | val mockGraphEditor = mock[GraphEditor[Any, Any]]
39 | val v = new PageRankVertex(id)
40 | for (id <- outgoingEdges) {
41 | v.addEdge(new PageRankEdge(id), mockGraphEditor)
42 | }
43 | v.afterInitialization(mockGraphEditor)
44 | for ((sourceId, signal) <- incomingSignals) {
45 | v.deliverSignalWithSourceId(signal, sourceId, mockGraphEditor)
46 | }
47 | if (!incomingSignals.isEmpty) {
48 | assert(v.scoreCollect > 0, "vertex received messages, should want to collect")
49 | v.executeCollectOperation(mockGraphEditor)
50 | v.state should equal(0.15 + 0.85 * incomingSignals.values.sum +- 0.0001)
51 | if (!outgoingEdges.isEmpty) {
52 | assert(v.scoreSignal > 0, "vertex updated state, should want to signal")
53 | expecting {
54 | for (targetId <- outgoingEdges) {
55 | call(mockGraphEditor.sendToWorkerForVertexIdHash(
56 | SignalMessageWithSourceId(targetId, id, v.state / outgoingEdges.size), targetId.hashCode))
57 | }
58 | }
59 | whenExecuting(mockGraphEditor) {
60 | v.executeSignalOperation(mockGraphEditor)
61 | }
62 | }
63 | }
64 | true
65 | }, minSuccessful(1000))
66 | }
67 |
68 | }
69 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/features/ErrorSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2015 Cotiviti
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.features
21 |
22 | import akka.testkit.EventFilter
23 | import com.signalcollect.TestGraph
24 | import org.scalatest.fixture.FlatSpec
25 | import org.scalatest.fixture.UnitFixture
26 | import com.signalcollect.DataGraphVertex
27 | import com.signalcollect.GraphEditor
28 | import com.signalcollect.TestEventGraph
29 |
30 | class TestError() extends Exception("Just a test exception.")
31 |
32 | class ErrorSpec extends FlatSpec with UnitFixture {
33 |
34 | "Signal/Collect" should "log errors during vertex initialization" in new TestEventGraph {
35 | EventFilter[TestError](occurrences = 1) intercept {
36 | g.addVertex(new DataGraphVertex(1, 1) {
37 | override def afterInitialization(graphEditor: GraphEditor[Any, Any]) = throw new TestError
38 | def collect: Int = 1
39 | })
40 | }
41 | }
42 |
43 | }
44 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/features/GraphLoadingSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.features
21 |
22 | import com.signalcollect._
23 | import com.signalcollect.examples.PageRankEdge
24 | import com.signalcollect.examples.PageRankVertex
25 | import com.signalcollect.examples.SudokuCell
26 | import org.scalatest.Matchers
27 | import org.scalatest.FlatSpec
28 |
29 | class GraphLoadingSpec extends FlatSpec with Matchers {
30 |
31 | "Graph" should "support the `loadGraph` command" in {
32 | val graph = TestConfig.graphProvider().build
33 | try {
34 | val graphLoaders = (1 to 100).map(x => (10 * x until ((10 * x) + 10)).toIterator.map(y => new PageRankVertex(y)).map(z => {
35 | ge: GraphEditor[Any, Any] =>
36 | ge.addVertex(z)
37 | }))
38 | for (loader <- graphLoaders) {
39 | graph.loadGraph(loader, Some(0))
40 | }
41 | graph.awaitIdle
42 | val stats = graph.execute(ExecutionConfiguration.withSignalThreshold(0.01))
43 | assert(stats.aggregatedWorkerStatistics.numberOfVertices == 1000, s"Only ${stats.aggregatedWorkerStatistics.numberOfVertices} vertices were added, instead of 1000.")
44 | } finally {
45 | graph.shutdown
46 | }
47 | }
48 |
49 | it should "support using the `loadGraph` command after the loading phase" in {
50 | val graph = TestConfig.graphProvider().build
51 | try {
52 | val graphLoaders = (1 to 100).map(x => (10 * x until ((10 * x) + 10)).toIterator.map(y => new PageRankVertex(y)).map(z => {
53 | ge: GraphEditor[Any, Any] =>
54 | ge.addVertex(z)
55 | }))
56 | for (loader <- graphLoaders.take(50)) {
57 | graph.loadGraph(loader, Some(0))
58 | }
59 | graph.awaitIdle
60 | graph.execute(ExecutionConfiguration.withSignalThreshold(0.01))
61 | for (loader <- graphLoaders) {
62 | graph.loadGraph(loader, Some(0))
63 | }
64 | graph.awaitIdle
65 | val stats = graph.execute(ExecutionConfiguration.withSignalThreshold(0.01))
66 | assert(stats.aggregatedWorkerStatistics.numberOfVertices == 1000, s"Only ${stats.aggregatedWorkerStatistics.numberOfVertices} vertices were added, instead of 1000.")
67 | } finally {
68 | graph.shutdown
69 | }
70 | }
71 |
72 | }
73 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/features/MapperSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | * @author Mihaela Verman
4 | *
5 | * Copyright 2013 University of Zurich
6 | *
7 | * Licensed under the Apache License, Version 2.0 (the "License");
8 | * you may not use this file except in compliance with the License.
9 | * You may obtain a copy of the License at
10 | *
11 | * http://www.apache.org/licenses/LICENSE-2.0
12 | *
13 | * Unless required by applicable law or agreed to in writing, software
14 | * distributed under the License is distributed on an "AS IS" BASIS,
15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | * See the License for the specific language governing permissions and
17 | * limitations under the License.
18 | *
19 | */
20 |
21 | package com.signalcollect.features
22 |
23 | import com.signalcollect.{TestConfig, ExecutionConfiguration, GraphBuilder, Vertex}
24 | import com.signalcollect.configuration.ExecutionMode
25 | import com.signalcollect.examples.PageRankEdge
26 | import com.signalcollect.examples.PageRankVertex
27 | import com.signalcollect.interfaces.ModularAggregationOperation
28 | import com.signalcollect.messaging.DefaultVertexToWorkerMapper
29 | import com.signalcollect.interfaces.VertexToWorkerMapper
30 | import com.signalcollect.interfaces.MapperFactory
31 | import org.scalatest.Matchers
32 | import org.scalatest.FlatSpec
33 |
34 | class Worker0Mapper[Id] extends VertexToWorkerMapper[Id] {
35 | def getWorkerIdForVertexId(vertexId: Id): Int = 0
36 | def getWorkerIdForVertexIdHash(vertexIdHash: Int): Int = 0
37 | }
38 |
39 | class Worker0MapperFactory[Id] extends MapperFactory[Id] {
40 | def createInstance(numberOfNodes: Int, workersPerNode: Int) = new Worker0Mapper
41 | }
42 |
43 | /**
44 | * Unit and integration tests for vertex mappers.
45 | */
46 | class MapperSpec extends FlatSpec with Matchers {
47 |
48 | val defaultMapper = new DefaultVertexToWorkerMapper[Int](1, 10)
49 |
50 | "Default mapper" should "correctly map a vertex to a worker" in {
51 | defaultMapper.getWorkerIdForVertexId(13) === 3
52 | }
53 |
54 | it should "correctly map a vertex hash to a worker" in {
55 | defaultMapper.getWorkerIdForVertexIdHash(13) === 3
56 | }
57 |
58 | "Custom mapper" should "correctly support PageRank computation" in {
59 | def verify(v: Vertex[_, _, _, _], expectedState: Double): Boolean = {
60 | val state = v.state.asInstanceOf[Double]
61 | val correct = (state - expectedState).abs < 0.0001
62 | if (!correct) {
63 | System.err.println("Problematic vertex: id=" + v.id + ", expected state=" + expectedState + " actual state=" + state)
64 | }
65 | correct
66 | }
67 | val graph = TestConfig.graphProvider()
68 | .withMapperFactory(new Worker0MapperFactory[Any]).build
69 | try {
70 | for (i <- 0 until 5) {
71 | val v = new PageRankVertex(i)
72 | graph.addVertex(v)
73 | graph.addEdge(i, new PageRankEdge((i + 1) % 5))
74 | }
75 | graph.execute(ExecutionConfiguration.
76 | withExecutionMode(ExecutionMode.PureAsynchronous).
77 | withCollectThreshold(0).
78 | withSignalThreshold(0.00001))
79 | var allcorrect = graph.aggregate(new ModularAggregationOperation[Boolean] {
80 | val neutralElement = true
81 | def aggregate(a: Boolean, b: Boolean): Boolean = a && b
82 | def extract(v: Vertex[_, _, _, _]): Boolean = verify(v, 1.0)
83 | })
84 | allcorrect
85 | } finally {
86 | graph.shutdown
87 | }
88 | }
89 |
90 | }
91 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/features/MultipleGraphs.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Carol Alexandru
3 | *
4 | * Copyright 2015 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.features
21 |
22 | import org.scalatest.Matchers
23 | import org.scalatest.FlatSpec
24 | import com.signalcollect._
25 | import com.signalcollect.examples.PageRankEdge
26 | import com.signalcollect.examples.PageRankVertex
27 |
28 | class MultipleGraphsSpec extends FlatSpec with Matchers {
29 |
30 | def createComputation(): Graph[_, _] = {
31 | val graph = TestConfig.graphProvider().build
32 | graph.addVertex(new PageRankVertex(1))
33 | graph.addVertex(new PageRankVertex(2))
34 | graph.addEdge(1, new PageRankEdge(2))
35 | graph.addEdge(2, new PageRankEdge(1))
36 | graph
37 | }
38 |
39 | "Signal/Collect" should "support running multiple graph instances on the same actor system" in {
40 | val graph1 = createComputation()
41 | val graph2 = createComputation()
42 | val graph3 = createComputation()
43 | graph1.execute
44 | graph2.execute
45 | graph3.execute
46 | graph1.awaitIdle
47 | graph2.awaitIdle
48 | graph3.awaitIdle
49 | graph1.shutdown
50 | graph2.shutdown
51 | graph3.shutdown
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/features/MultipleVertexAdditionsSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.features
21 |
22 | import org.scalatest.FlatSpec
23 | import org.scalatest.prop.Checkers
24 | import com.signalcollect.util.IntSet
25 | import com.signalcollect.util.Ints
26 | import com.signalcollect._
27 | import com.signalcollect.examples.PageRankVertex
28 | import org.scalatest.Matchers
29 | import akka.event.Logging
30 | import com.signalcollect.interfaces.ExistingVertexHandlerFactory
31 | import com.signalcollect.interfaces.ExistingVertexHandler
32 |
33 | class DummyVertex(id: Int) extends PageRankVertex(id) {
34 | state = 1
35 | }
36 |
37 | class TestExistingVertexHandlerFactory[Id, Signal] extends ExistingVertexHandlerFactory[Id, Signal] {
38 | def createInstance: ExistingVertexHandler[Id, Signal] =
39 | new TestExistingVertexHandler[Id, Signal]
40 | override def toString = "TestExistingVertexHandlerFactory"
41 | }
42 |
43 | class TestExistingVertexHandler[Id, Signal] extends ExistingVertexHandler[Id, Signal] {
44 | def mergeVertices(existing: Vertex[Id, _, Id, Signal], failedVertexAddition: Vertex[Id, _, Id, Signal], ge: GraphEditor[Id, Signal]) {
45 | existing.asInstanceOf[DummyVertex].state += 1.0
46 | }
47 | }
48 |
49 | class MultipleVertexAdditionsSpec extends FlatSpec with Matchers {
50 |
51 | "Adding the same vertex multiple times" should "be ignored" in {
52 | val g = TestConfig.graphProvider().build //.withLoggingLevel(Logging.DebugLevel)
53 | try {
54 | g.addVertex(new DummyVertex(133))
55 | g.addVertex(new DummyVertex(134))
56 | g.addVertex(new DummyVertex(133))
57 | val numberOfDummies = g.aggregate(SumOfStates[Double])
58 | numberOfDummies.get should equal(2.0)
59 | } finally {
60 | g.shutdown
61 | }
62 | }
63 |
64 | it should "support merges via handler" in {
65 | val g = TestConfig.graphProvider()
66 | .withExistingVertexHandlerFactory(new TestExistingVertexHandlerFactory[Any, Any]).build
67 | try {
68 | g.addVertex(new DummyVertex(133))
69 | g.addVertex(new DummyVertex(134))
70 | g.addVertex(new DummyVertex(133))
71 | val stateSum = g.aggregate(SumOfStates[Double])
72 | stateSum.get should equal(3.0)
73 | } finally {
74 | g.shutdown
75 | }
76 | }
77 |
78 | }
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/features/SnapshotSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2012 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.features
21 |
22 | import com.signalcollect.{TestConfig, ExecutionConfiguration, GraphBuilder, Vertex}
23 | import com.signalcollect.configuration.ExecutionMode
24 | import com.signalcollect.examples.PageRankEdge
25 | import com.signalcollect.examples.PageRankVertex
26 | import com.signalcollect.interfaces.ModularAggregationOperation
27 | import org.scalatest.Matchers
28 | import org.scalatest.FlatSpec
29 |
30 | class SnapshotSpec extends FlatSpec with Matchers {
31 |
32 | "Snapshots" should "correctly store and load a small graph" in {
33 |
34 | def verify(v: Vertex[_, _, _, _], expectedState: Double): Boolean = {
35 | val state = v.state.asInstanceOf[Double]
36 | val correct = (state - expectedState).abs < 0.0001
37 | if (!correct) {
38 | System.err.println("Problematic vertex: id=" + v.id + ", expected state=" + expectedState + " actual state=" + state)
39 | }
40 | correct
41 | }
42 | val graph = TestConfig.graphProvider().build
43 | try {
44 | graph.deleteSnapshot
45 | graph.restore
46 | for (i <- 0 until 5) {
47 | val v = new PageRankVertex(i)
48 | graph.addVertex(v)
49 | graph.addEdge(i, new PageRankEdge((i + 1) % 5))
50 | }
51 | graph.snapshot
52 | graph.execute(ExecutionConfiguration.
53 | withExecutionMode(ExecutionMode.PureAsynchronous).
54 | withCollectThreshold(0).
55 | withSignalThreshold(0.00001))
56 | graph.restore
57 | var allcorrect = graph.aggregate(new ModularAggregationOperation[Boolean] {
58 | val neutralElement = true
59 | def aggregate(a: Boolean, b: Boolean): Boolean = a && b
60 | def extract(v: Vertex[_, _, _, _]): Boolean = verify(v, 0.15)
61 | })
62 | graph.deleteSnapshot
63 | allcorrect
64 | } finally {
65 | graph.shutdown
66 | }
67 | }
68 |
69 | }
70 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/features/StoreClosedOnShutdownSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2015 iHealth Technologies
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | *
18 | */
19 |
20 | package com.signalcollect.features
21 |
22 | import org.scalatest.{ FlatSpec, Matchers }
23 |
24 | import com.signalcollect.GraphBuilder
25 | import com.signalcollect.interfaces.{ Storage, StorageFactory }
26 | import com.signalcollect.storage.VertexMapStorage
27 |
28 | object TestStorageFactory extends StorageFactory[Any, Any] {
29 | def createInstance: Storage[Any, Any] = TestStorage
30 | }
31 |
32 | object TestStorage extends VertexMapStorage[Any, Any] {
33 | var closed = false
34 | override def close(): Unit = {
35 | closed = true
36 | }
37 | }
38 |
39 | class StoreClosedOnShutdownSpec extends FlatSpec with Matchers {
40 |
41 | "Store" should "be closed on shutdown" in {
42 | val graph = GraphBuilder.withStorageFactory(TestStorageFactory).build
43 | graph.awaitIdle
44 | graph.shutdown
45 | assert(TestStorage.closed == true)
46 | }
47 |
48 | }
49 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/loading/AdjacencyListLoaderSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.loading
20 |
21 | import scala.collection.mutable.ArrayBuffer
22 | import org.scalatest.prop.Checkers
23 | import org.scalatest.ShouldMatchers
24 | import org.scalatest.FlatSpec
25 | import java.io.File
26 | import com.signalcollect.util.FileReader
27 | import com.signalcollect.examples.PageRankVertex
28 | import com.signalcollect.examples.EfficientPageRankVertex
29 | import com.signalcollect.util.AsciiIntIterator
30 | import com.signalcollect.util.AsciiIntIterator
31 | import java.io.FileInputStream
32 |
33 | class AdjacencyListLoaderSpec extends FlatSpec with ShouldMatchers with Checkers {
34 |
35 | val testFilePath = "adjacency-list-format"
36 |
37 | "AsciiIntIterator" should "correctly parse the integers from an adjacency list file" in {
38 | val l = FileReader.intIterator(getClass.getResourceAsStream(testFilePath)).toList
39 | val correct = List(1, 0, 4, 1, 5, 2, 3, 1, 5, 4, 5, 0)
40 | assert(l == correct,
41 | s"Read Ints were $l instead of the correct $correct.")
42 | }
43 |
44 | "AdjacencyListLoader" should "correctly parse vertex data from a file" in {
45 | var count = 0
46 | var parsed = List.empty[(Int, List[Int])]
47 | AdjacencyListLoader[Double](() => getClass.getResourceAsStream(testFilePath), {
48 | case (id, array) =>
49 | val dummy = new EfficientPageRankVertex(1)
50 | parsed = (id, array.toList) :: parsed
51 | dummy
52 | }).foreach(x => count += 1)
53 | val correct = List(
54 | (5, List.empty[Int]),
55 | (2, List(1, 5, 4)),
56 | (4, List(5)),
57 | (1, List.empty[Int]))
58 | assert(parsed == correct,
59 | s"Parsed was $parsed instead of the correct $correct.")
60 | }
61 |
62 | }
63 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/loading/LoadingSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.loading
20 |
21 | import scala.collection.mutable.ArrayBuffer
22 | import org.scalatest.prop.Checkers
23 | import org.scalatest.ShouldMatchers
24 | import org.scalatest.FlatSpec
25 | import java.io.File
26 | import com.signalcollect.util.FileReader
27 |
28 | class LoadingSpec extends FlatSpec with ShouldMatchers with Checkers {
29 |
30 | val testFile = s"notredame2"
31 |
32 | "Loader" should "correctly parse vertex data from an ordered integer edge list" in {
33 | val d = Loading.intEdgeIdsOrderedBySourceId(getClass.getResourceAsStream(testFile)).toList
34 | assert(d == List((0, List(1, 0)), (1, List(0)), (3, List(2))))
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/util/BasicBitSetSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import org.scalacheck.Gen
22 | import org.scalacheck.Gen._
23 | import org.scalacheck.Arbitrary._
24 | import org.scalatest.FlatSpec
25 | import org.scalatest.ShouldMatchers
26 | import org.scalatest.prop.Checkers
27 | import java.io.DataOutputStream
28 | import java.io.ByteArrayOutputStream
29 | import org.scalacheck.Arbitrary
30 |
31 | class BasicBitSetSpec extends FlatSpec with ShouldMatchers with Checkers {
32 |
33 | "BasicBitSet" should "correctly retrieve all entries when all bits are set" in {
34 | val longBitSet = new BasicBitSet(-1l) // -1 has all bits set (2's complement)
35 | val toSet = longBitSet.toSet
36 | assert(toSet == (0 to 63).toSet)
37 | assert(longBitSet.size == 64)
38 | }
39 |
40 | it should "should be empty before inserts" in {
41 | val longBitSet = new BasicBitSet(0l)
42 | val toSet = longBitSet.toSet
43 | assert(toSet == Set.empty[Int])
44 | assert(longBitSet.size == 0)
45 | }
46 |
47 | it should "support an insert into an empty set" in {
48 | check(
49 | (item: Int) => {
50 | val insertItem = (item & Int.MaxValue) % 64
51 | val longBitSet = new BasicBitSet(new BasicBitSet(0l).set(insertItem))
52 | assert(longBitSet.toSet === Set(insertItem))
53 | assert(longBitSet.min === insertItem)
54 | assert(longBitSet.max === insertItem)
55 | longBitSet.size === 1
56 | },
57 | minSuccessful(100))
58 | }
59 |
60 | it should "support multiple inserts into an empty set" in {
61 | check(
62 | (is: Array[Int]) => {
63 | val insertItems = is.map(i => (i & Int.MaxValue) % 64)
64 | var longBitSet = new BasicBitSet(0l)
65 | var standardSet = Set.empty[Int]
66 | for (i <- insertItems) {
67 | longBitSet = new BasicBitSet(longBitSet.set(i))
68 | standardSet += i
69 | }
70 | assert(longBitSet.toSet === standardSet)
71 | if (is.nonEmpty) {
72 | assert(longBitSet.min === standardSet.min)
73 | assert(longBitSet.max === standardSet.max)
74 | }
75 | longBitSet.size === standardSet.size
76 | },
77 | minSuccessful(100))
78 | }
79 |
80 | it should "support base values A" in {
81 | val longBitSet = new BasicBitSet(-1l)
82 | assert(longBitSet.toSetWithBaseValue(10) === (10 to 73).toSet)
83 | }
84 |
85 | it should "support base values B" in {
86 | val longBitSet = new BasicBitSet(1l)
87 | assert(longBitSet.toSetWithBaseValue(0) === Set(0))
88 | }
89 |
90 | }
91 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/util/Benchmark.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import java.lang.management.ManagementFactory
22 | import scala.collection.JavaConversions._
23 | import java.lang.reflect.Modifier
24 | import java.lang.reflect.Field
25 | import sun.misc.Unsafe
26 |
27 | case class Stats(
28 | description: String,
29 | nanosecondsPerExecution: Long)
30 |
31 | object Benchmark {
32 | def apply(f: () => Unit, description: String = "", warmups: Int = 1000, measuredRepetitions: Int = 10000): Stats = {
33 | if (description != "") {
34 | println(s"Warming up $description.")
35 | }
36 | var i = 0
37 | while (i < warmups) {
38 | f()
39 | i += 1
40 | }
41 | i = 0
42 | sleepUntilGcInactiveForXSeconds(3)
43 | if (description != "") {
44 | println(s"Benchmarking $description.")
45 | }
46 | val startTime = System.nanoTime
47 | while (i < measuredRepetitions) {
48 | f()
49 | i += 1
50 | }
51 | val finishTime = System.nanoTime
52 | val totalNanoTime = finishTime - startTime
53 | val nanosecondsPerExecution = totalNanoTime.toDouble / measuredRepetitions.toDouble
54 | println(s"$description took $nanosecondsPerExecution nanoseconds per execution")
55 | Stats(description,
56 | nanosecondsPerExecution.toLong)
57 | }
58 |
59 | def sleepUntilGcInactiveForXSeconds(x: Int) {
60 | val gcs = ManagementFactory.getGarbageCollectorMXBeans
61 | val sunGcs = gcs.map(_.asInstanceOf[com.sun.management.GarbageCollectorMXBean])
62 | def collectionTime = sunGcs.map(_.getCollectionTime).sum
63 | def collectionDelta(oldGcTime: Long) = collectionTime - oldGcTime
64 | var secondsWithoutGc = 0
65 | var lastGcTime = collectionTime
66 | while (secondsWithoutGc < x) {
67 | Thread.sleep(1000)
68 | val delta = collectionDelta(lastGcTime)
69 | if (delta > 0) {
70 | secondsWithoutGc = 0
71 | lastGcTime = collectionTime
72 | } else {
73 | secondsWithoutGc += 1
74 | }
75 | }
76 | }
77 |
78 | }
79 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/util/FileReaderSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import scala.collection.mutable.ArrayBuffer
22 | import org.scalatest.prop.Checkers
23 | import org.scalatest.ShouldMatchers
24 | import org.scalatest.FlatSpec
25 | import java.io.File
26 | import com.signalcollect.loading.LoadingSpec
27 |
28 | class FileReaderSpec extends FlatSpec with ShouldMatchers with Checkers {
29 |
30 | val sep = File.separator
31 | val testFilePath1 = s"ascii-ints.txt"
32 | val testFilePath2 = s"ints-no-newline-at-end.txt"
33 |
34 | "FileReader" should "correctly parse a file with an iterator" in {
35 | val asList = FileReader.intIterator(classOf[LoadingSpec].getResourceAsStream(testFilePath1)).toList
36 | assert(asList == List(1234, 3525, 123, 436, 43663, 33, 44, 0))
37 | }
38 |
39 | it should "correctly parse a file with the processing function" in {
40 | val buffer = new ArrayBuffer[Int]
41 | FileReader.processInts(classOf[LoadingSpec].getResourceAsStream(testFilePath1), buffer.append(_))
42 | val asList = buffer.toList
43 | assert(asList == List(1234, 3525, 123, 436, 43663, 33, 44, 0))
44 | }
45 |
46 | it should "correctly iterate over ints when there is no newline at the end" in {
47 | val asList = FileReader.intIterator(classOf[LoadingSpec].getResourceAsStream(testFilePath2)).toList
48 | assert(asList == List(1, 2, 3, 0))
49 | }
50 |
51 | }
52 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/util/HashSetSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import org.scalacheck.Arbitrary
22 | import org.scalacheck.Arbitrary.arbContainer
23 | import org.scalacheck.Gen
24 | import org.scalatest.FlatSpec
25 | import org.scalatest.prop.Checkers
26 |
27 | class HasSetSpec extends FlatSpec with Checkers {
28 |
29 | "HasSet" should "support inserts" in {
30 | check(
31 | (setEntries: Array[String]) => {
32 | var wasEqual = false
33 | val s = new HashSet[String](8, 0.5f)
34 | try {
35 | val sCorrect = setEntries.toSet
36 | for (e <- setEntries) {
37 | s.add(e)
38 | }
39 | wasEqual = s.toScalaSet == sCorrect
40 | if (!wasEqual) {
41 | println("Problematic set: " + s.toScalaSet +
42 | "\nShould have been: " + sCorrect.toString)
43 | }
44 | } catch {
45 | case t: Throwable =>
46 | t.printStackTrace
47 | }
48 | wasEqual
49 | },
50 | minSuccessful(1000))
51 | }
52 |
53 | it should "support removals" in {
54 | check(
55 | (toAdd: Array[String], toAddThenRemove: Array[String]) => {
56 | val setEntries = toAdd.toSet ++ toAddThenRemove.toSet
57 | var wasEqual = false
58 | var sCorrect = setEntries.toSet
59 | val s = new HashSet[String](8, 0.5f)
60 | try {
61 | for (e <- setEntries) {
62 | s.add(e)
63 | }
64 | for (e <- toAddThenRemove) {
65 | s.remove(e)
66 | sCorrect -= e
67 | }
68 | wasEqual = s.toScalaSet == sCorrect
69 | if (!wasEqual) {
70 | println("Problematic set: " + s.toScalaSet +
71 | "\nShould have been: " + sCorrect.toString)
72 | }
73 | } catch {
74 | case t: Throwable =>
75 | t.printStackTrace
76 | }
77 | wasEqual
78 | },
79 | minSuccessful(1000))
80 | }
81 |
82 | }
83 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/util/IntLongHashMapSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2013 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import org.scalacheck.Arbitrary
22 | import org.scalacheck.Arbitrary.arbContainer
23 | import org.scalacheck.Gen
24 | import org.scalatest.FlatSpec
25 | import org.scalatest.prop.Checkers
26 |
27 | class IntLongHashMapSpec extends FlatSpec with Checkers {
28 |
29 | "IntLongHashMap" should "support inserts" in {
30 | check(
31 | (keyValuePairs: Array[(Int, Long)]) => {
32 | val validPairs = keyValuePairs.filter(_._1 != 0)
33 | var wasEqual = false
34 | val m = new IntLongHashMap(8, 0.5f)
35 | try {
36 | val mCorrect = validPairs.toMap
37 | for (p <- validPairs) {
38 | m.put(p._1, p._2)
39 | }
40 | wasEqual = m.toScalaMap == mCorrect
41 | if (!wasEqual) {
42 | println("Problematic map: " + m.toScalaMap +
43 | "\nShould have been: " + mCorrect.toString)
44 | }
45 | } catch {
46 | case t: Throwable =>
47 | t.printStackTrace
48 | }
49 | wasEqual
50 | },
51 | minSuccessful(1000))
52 | }
53 |
54 | it should "support removals" in {
55 | check(
56 | (keyValuePairs1: Array[(Int, Long)], keyValuePairs2: Array[(Int, Long)]) => {
57 | val keys1 = keyValuePairs1.map(_._1)
58 | val keyValuePairs = keyValuePairs1.toMap ++ keyValuePairs2.toMap
59 | val validPairs = keyValuePairs.filter(_._1 != 0)
60 | var wasEqual = false
61 | var mCorrect = validPairs.toMap
62 | val m = new IntLongHashMap(8, 0.5f)
63 | try {
64 | for (p <- validPairs) {
65 | m.put(p._1, p._2)
66 | }
67 | for (key <- keys1) {
68 | m.remove(key)
69 | mCorrect -= key
70 | }
71 | wasEqual = m.toScalaMap == mCorrect
72 | if (!wasEqual) {
73 | println("Problematic map: " + m.toScalaMap +
74 | "\nShould have been: " + mCorrect.toString)
75 | }
76 | } catch {
77 | case t: Throwable =>
78 | t.printStackTrace
79 | }
80 | wasEqual
81 | },
82 | minSuccessful(1000))
83 | }
84 |
85 | }
86 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/util/IteratorConcatenatorSpec.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import org.scalacheck.Gen
22 | import org.scalacheck.Gen._
23 | import org.scalacheck.Arbitrary._
24 | import org.scalatest.FlatSpec
25 | import org.scalatest.ShouldMatchers
26 | import org.scalatest.prop.Checkers
27 | import java.io.DataOutputStream
28 | import java.io.ByteArrayOutputStream
29 | import org.scalacheck.Arbitrary
30 |
31 | class IteratorConcatenatorSpec extends FlatSpec with ShouldMatchers with Checkers {
32 |
33 | "IteratorConcatenator" should "correctly concatenate multiple iterators" in {
34 | val c = new IteratorConcatenator[Int]
35 | for (i <- 1 to 1000 by 10) {
36 | c.appendIterator((i until i + 10).iterator)
37 | }
38 | c.appendIterator(Seq(1001, 1002, 1003).iterator)
39 | c.appendIterator(Set(1004).iterator)
40 | assert(c.toList == (1 to 1004).toList)
41 | }
42 |
43 | it should "should work with random appends" in {
44 | check(
45 | (is: List[List[Int]]) => {
46 | val c = new IteratorConcatenator[Int]
47 | is.foreach { list => c.appendIterator(list.iterator) }
48 | assert(c.toList == is.flatMap(_.iterator).toList)
49 | true
50 | },
51 | minSuccessful(100))
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/src/test/scala/com/signalcollect/util/SplayBench.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * @author Philip Stutz
3 | *
4 | * Copyright 2014 University of Zurich
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package com.signalcollect.util
20 |
21 | import scala.util.Random
22 |
23 | object SplayBench extends App {
24 | val overheadFractions = List(0.0f, 0.05f, 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 1.0f)
25 | val splitSize = List(10, 100, 1000, 10000)//, 100000)
26 | val splayIntSets = for {
27 | overhead <- overheadFractions
28 | split <- splitSize
29 | } yield new SimpleSplayIntSet(overhead, split)
30 |
31 | def operation(s: SplayIntSet)(): Unit = {
32 | s.insert(Random.nextInt)
33 | }
34 | val stats = splayIntSets.map(
35 | splay => (operation(splay) _, splay.toString)).map {
36 | case (op, desc) => Benchmark(op, desc, measuredRepetitions = 1000000)
37 | }
38 | stats.sortBy(_.nanosecondsPerExecution).foreach(println)
39 | }
40 |
--------------------------------------------------------------------------------