├── dolphin-bsp └── src │ ├── test │ ├── resources │ │ ├── result │ │ │ ├── linearreg_loss │ │ │ ├── logisticreg_accuracy │ │ │ ├── linearreg_model │ │ │ ├── logisticreg_model │ │ │ └── pagerank │ │ └── data │ │ │ └── pagerank │ └── java │ │ └── edu │ │ └── snu │ │ └── dolphin │ │ └── bsp │ │ └── examples │ │ └── ml │ │ └── algorithms │ │ ├── graph │ │ ├── package-info.java │ │ └── GraphTestSuite.java │ │ ├── regression │ │ ├── package-info.java │ │ └── RegressionTestSuite.java │ │ └── classification │ │ ├── package-info.java │ │ └── ClassificationTestSuite.java │ └── main │ └── java │ └── edu │ └── snu │ └── dolphin │ └── bsp │ ├── core │ ├── Key.java │ ├── OptimizeScheme.java │ ├── Optimizer.java │ ├── package-info.java │ ├── metric │ │ ├── package-info.java │ │ ├── MetricCodec.java │ │ ├── MetricTracker.java │ │ ├── MetricTrackers.java │ │ └── MetricException.java │ ├── ParseException.java │ ├── CommunicationGroup.java │ ├── CtrlMessage.java │ ├── UserJobInfo.java │ ├── DataParser.java │ ├── UserParameters.java │ ├── KeyValueStore.java │ ├── OutputStreamProvider.java │ └── KeyValueStoreService.java │ ├── scheduling │ ├── package-info.java │ ├── SchedulabilityAnalyzer.java │ └── YarnSchedulabilityAnalyzer.java │ ├── groupcomm │ ├── names │ │ ├── package-info.java │ │ ├── DataGather.java │ │ ├── DataReduce.java │ │ ├── DataScatter.java │ │ ├── DataBroadcast.java │ │ └── CtrlMsgBroadcast.java │ └── interfaces │ │ ├── package-info.java │ │ ├── DataGatherSender.java │ │ ├── DataReduceSender.java │ │ ├── DataBroadcastSender.java │ │ ├── DataReduceReceiver.java │ │ ├── DataScatterReceiver.java │ │ ├── DataBroadcastReceiver.java │ │ ├── DataScatterSender.java │ │ └── DataGatherReceiver.java │ ├── parameters │ ├── package-info.java │ ├── JobIdentifier.java │ ├── OutputDir.java │ ├── InputDir.java │ ├── Timeout.java │ ├── OnLocal.java │ ├── DesiredSplits.java │ ├── EvaluatorSize.java │ └── LocalRuntimeMaxNumEvaluators.java │ └── examples │ ├── ml │ ├── parameters │ │ ├── package-info.java │ │ ├── CommunicationGroup.java │ │ ├── Dimension.java │ │ ├── NumberOfClusters.java │ │ ├── Lambda.java │ │ ├── StepSize.java │ │ ├── DampingFactor.java │ │ ├── IsCovarianceShared.java │ │ ├── ConvergenceThreshold.java │ │ ├── IsCovarianceDiagonal.java │ │ └── MaxIterations.java │ ├── key │ │ ├── package-info.java │ │ └── Centroids.java │ ├── loss │ │ ├── package-info.java │ │ ├── SquareLoss.java │ │ ├── HingeLoss.java │ │ └── Loss.java │ ├── converge │ │ ├── package-info.java │ │ ├── PageRankConvCond.java │ │ ├── ClusteringConvCond.java │ │ └── LinearModelConvCond.java │ ├── data │ │ ├── package-info.java │ │ ├── Row.java │ │ ├── VectorDistanceMeasure.java │ │ ├── EuclideanDistance.java │ │ ├── Model.java │ │ ├── LinearRegSummary.java │ │ └── PageRankSummary.java │ ├── sub │ │ ├── package-info.java │ │ ├── PageRankReduceFunction.java │ │ ├── LinearRegReduceFunction.java │ │ └── LogisticRegReduceFunction.java │ ├── regularization │ │ ├── package-info.java │ │ ├── L2Regularization.java │ │ └── Regularization.java │ └── algorithms │ │ ├── graph │ │ └── package-info.java │ │ ├── clustering │ │ ├── package-info.java │ │ ├── em │ │ │ ├── package-info.java │ │ │ └── EMMainCommGroup.java │ │ ├── kmeans │ │ │ ├── package-info.java │ │ │ └── KMeansMainCommGroup.java │ │ ├── ClusteringPreCommGroup.java │ │ └── ClusteringPreStageBuilder.java │ │ ├── regression │ │ └── package-info.java │ │ └── classification │ │ └── package-info.java │ └── simple │ ├── package-info.java │ ├── SimpleCommGroup.java │ ├── SimpleReduceFunction.java │ ├── SimpleJobInfo.java │ ├── SimpleREEF.java │ └── SimpleCmpTask.java ├── dolphin-ps └── src │ ├── main │ └── java │ │ └── edu │ │ └── snu │ │ └── dolphin │ │ ├── util │ │ └── package-info.java │ │ └── ps │ │ ├── server │ │ ├── package-info.java │ │ ├── partitioned │ │ │ ├── parameters │ │ │ │ ├── package-info.java │ │ │ │ └── ServerQueueSize.java │ │ │ ├── package-info.java │ │ │ └── PartitionedServerSideReplySender.java │ │ ├── api │ │ │ └── package-info.java │ │ └── concurrent │ │ │ ├── api │ │ │ └── package-info.java │ │ │ └── impl │ │ │ └── package-info.java │ │ ├── ns │ │ ├── package-info.java │ │ ├── EndpointId.java │ │ ├── PSMessageHandler.java │ │ └── ParameterServerMsgCodec.java │ │ ├── package-info.java │ │ ├── driver │ │ ├── package-info.java │ │ ├── api │ │ │ ├── package-info.java │ │ │ └── ParameterServerManager.java │ │ └── impl │ │ │ ├── package-info.java │ │ │ └── ServerId.java │ │ ├── examples │ │ └── add │ │ │ ├── parameters │ │ │ ├── package-info.java │ │ │ ├── JobTimeout.java │ │ │ ├── NumUpdates.java │ │ │ ├── NumWorkers.java │ │ │ ├── StartKey.java │ │ │ └── NumKeys.java │ │ │ ├── package-info.java │ │ │ ├── IntegerCodec.java │ │ │ └── AddUpdater.java │ │ ├── common │ │ ├── package-info.java │ │ ├── partitioned │ │ │ ├── resolver │ │ │ │ └── package-info.java │ │ │ └── parameters │ │ │ │ ├── package-info.java │ │ │ │ ├── NumServers.java │ │ │ │ └── NumPartitions.java │ │ └── Constants.java │ │ └── worker │ │ ├── partitioned │ │ ├── parameters │ │ │ ├── package-info.java │ │ │ ├── WorkerNumPartitions.java │ │ │ ├── WorkerExpireTimeout.java │ │ │ ├── WorkerKeyCacheSize.java │ │ │ └── WorkerQueueSize.java │ │ ├── package-info.java │ │ └── PartitionedWorkerHandler.java │ │ ├── package-info.java │ │ ├── api │ │ └── package-info.java │ │ ├── concurrent │ │ ├── package-info.java │ │ └── ConcurrentWorkerHandler.java │ │ └── AsyncWorkerHandler.java │ └── test │ └── java │ └── edu │ └── snu │ └── dolphin │ └── ps │ ├── package-info.java │ ├── server │ └── package-info.java │ ├── worker │ └── package-info.java │ ├── common │ └── partitioned │ │ └── resolver │ │ └── package-info.java │ └── TestUtils.java ├── dolphin-dnn └── src │ ├── test │ ├── java │ │ └── edu │ │ │ └── snu │ │ │ └── dolphin │ │ │ └── dnn │ │ │ ├── package-info.java │ │ │ ├── layers │ │ │ └── package-info.java │ │ │ ├── data │ │ │ └── package-info.java │ │ │ └── blas │ │ │ └── function │ │ │ └── package-info.java │ └── resources │ │ └── configuration │ │ ├── neuralnet │ │ ├── neuralnet_ps │ │ └── neuralnet_groupcomm │ └── main │ └── java │ └── edu │ └── snu │ └── dolphin │ └── dnn │ ├── package-info.java │ ├── util │ └── package-info.java │ ├── conf │ └── package-info.java │ ├── layers │ └── package-info.java │ ├── blas │ ├── function │ │ ├── package-info.java │ │ └── Function.java │ ├── jblas │ │ └── package-info.java │ └── package-info.java │ ├── data │ ├── package-info.java │ └── ListReduceFunction.java │ └── layerparam │ ├── provider │ ├── package-info.java │ └── ParameterProvider.java │ └── initializer │ ├── package-info.java │ └── LayerParameterInitializer.java ├── bin ├── run_ps.sh ├── run.sh ├── run_neuralnet.sh ├── run_em.sh ├── run_kmeans.sh ├── run_pagerank.sh ├── run_regression.sh └── run_logistic.sh ├── README.md └── .gitignore /dolphin-bsp/src/test/resources/result/linearreg_loss: -------------------------------------------------------------------------------- 1 | 3.433606e+02 -------------------------------------------------------------------------------- /dolphin-bsp/src/test/resources/result/logisticreg_accuracy: -------------------------------------------------------------------------------- 1 | 8.700000e-01 -------------------------------------------------------------------------------- /dolphin-bsp/src/test/resources/result/linearreg_model: -------------------------------------------------------------------------------- 1 | -2.567485e+00 * x1 + 1.652592e+00 * x2 + -1.224536e+00 * x3 + 5.160130e-01 -------------------------------------------------------------------------------- /dolphin-bsp/src/test/resources/result/logisticreg_model: -------------------------------------------------------------------------------- 1 | -6.098949e-03 * x1 + 3.581356e-03 * x2 + -2.669779e-03 * x3 + 7.465738e-04 -------------------------------------------------------------------------------- /dolphin-bsp/src/test/resources/result/pagerank: -------------------------------------------------------------------------------- 1 | node_id,rank 2 | 1,0.304112 3 | 2,3.339811 4 | 3,3.407025 5 | 4,0.362767 6 | 5,0.750362 7 | 6,0.362767 8 | 7,0.150000 9 | 8,0.150000 10 | 9,0.150000 11 | 10,0.150000 12 | 11,0.150000 13 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/Key.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | public interface Key { 19 | } 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/util/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Utility classes for Dolphin. 18 | */ 19 | package edu.snu.dolphin.util; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/test/java/edu/snu/dolphin/dnn/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for neural network tests. 18 | */ 19 | package edu.snu.dolphin.dnn; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/test/java/edu/snu/dolphin/ps/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for parameter server tests. 18 | */ 19 | package edu.snu.dolphin.ps; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/OptimizeScheme.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | public interface OptimizeScheme { 19 | } 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes of distributed neural network on REEF. 18 | */ 19 | package edu.snu.dolphin.dnn; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/util/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Utility classes for neural network. 18 | */ 19 | package edu.snu.dolphin.dnn.util; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/conf/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Configuration classes for neural network. 18 | */ 19 | package edu.snu.dolphin.dnn.conf; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/layers/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Layer classes for neural network. 18 | */ 19 | package edu.snu.dolphin.dnn.layers; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Server-side Parameter Server classes. 18 | */ 19 | package edu.snu.dolphin.ps.server; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/test/java/edu/snu/dolphin/dnn/layers/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for testing neural network layers. 18 | */ 19 | package edu.snu.dolphin.dnn.layers; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/ns/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Network service related classes for Parameter Server. 18 | */ 19 | package edu.snu.dolphin.ps.ns; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for providing Parameter Server as a REEF service. 18 | */ 19 | package edu.snu.dolphin.ps; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/Optimizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | public interface Optimizer { 19 | OptimizeScheme optimize(); 20 | } 21 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/scheduling/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Support for gang scheduling in Dolphin. 18 | */ 19 | package edu.snu.dolphin.bsp.scheduling; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/blas/function/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for activation functions. 18 | */ 19 | package edu.snu.dolphin.dnn.blas.function; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/blas/jblas/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * BLAS implementation based on JBLAS. 18 | */ 19 | package edu.snu.dolphin.dnn.blas.jblas; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/data/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Data representation classes for neural network. 18 | */ 19 | package edu.snu.dolphin.dnn.data; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/test/java/edu/snu/dolphin/ps/server/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for server-side parameter server tests. 18 | */ 19 | package edu.snu.dolphin.ps.server; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/test/java/edu/snu/dolphin/ps/worker/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for worker-side parameter server tests. 18 | */ 19 | package edu.snu.dolphin.ps.worker; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Dolphin core classes that implement the internal flow of Dolphin. 18 | */ 19 | package edu.snu.dolphin.bsp.core; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/names/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Name classes for REEF group communication. 18 | */ 19 | package edu.snu.dolphin.bsp.groupcomm.names; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Common parameter classes for all Dolphin applications. 18 | */ 19 | package edu.snu.dolphin.bsp.parameters; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/test/java/edu/snu/dolphin/bsp/examples/ml/algorithms/graph/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Tests for Graph. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.graph; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/driver/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Parameter Server classes that are instantiated at the driver. 18 | */ 19 | package edu.snu.dolphin.ps.driver; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/blas/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Interfaces and classes for BLAS (Basic Linear Algebra Subprograms). 18 | */ 19 | package edu.snu.dolphin.dnn.blas; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/parameters/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Parameters related to PS examples. 18 | */ 19 | package edu.snu.dolphin.ps.examples.add.parameters; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Algorithm-specific parameter classes. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.parameters; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/test/java/edu/snu/dolphin/bsp/examples/ml/algorithms/regression/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Tests for Graph. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.regression; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/layerparam/provider/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Parameter provider classes for neural network. 18 | */ 19 | package edu.snu.dolphin.dnn.layerparam.provider; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/test/java/edu/snu/dolphin/dnn/data/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for neural network tests related to data serialization and parsing. 18 | */ 19 | package edu.snu.dolphin.dnn.data; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/common/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Common Parameter Server classes used across Driver, Server, and Worker. 18 | */ 19 | package edu.snu.dolphin.ps.common; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/key/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Dolphin classes that implement the {@code Key} interface. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.key; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/loss/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Loss function classes for machine learning algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.loss; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/metric/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes that are related to the Metrics Collection Service of Dolphin. 18 | */ 19 | package edu.snu.dolphin.bsp.core.metric; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/converge/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Convergence checks for machine learning algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.converge; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/data/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Data representation classes for machine learning algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.data; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/test/java/edu/snu/dolphin/bsp/examples/ml/algorithms/classification/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Tests for Graph. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.classification; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/common/partitioned/resolver/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Resolve Server name based on a hashed key. 18 | */ 19 | package edu.snu.dolphin.ps.common.partitioned.resolver; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/partitioned/parameters/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Parameters related to Partitioned PS. 18 | */ 19 | package edu.snu.dolphin.ps.server.partitioned.parameters; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/test/java/edu/snu/dolphin/ps/common/partitioned/resolver/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for common parameter server tests. 18 | */ 19 | package edu.snu.dolphin.ps.common.partitioned.resolver; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/simple/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Simple REEF application for demonstrating Dolphin's functionalities. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.simple; 20 | -------------------------------------------------------------------------------- /dolphin-dnn/src/test/java/edu/snu/dolphin/dnn/blas/function/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes for testing {@link edu.snu.dolphin.dnn.blas.function.Function}. 18 | */ 19 | package edu.snu.dolphin.dnn.blas.function; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/common/partitioned/parameters/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Common parameters related to Partitioned PS. 18 | */ 19 | package edu.snu.dolphin.ps.common.partitioned.parameters; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/partitioned/parameters/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Parameters related to Partitioned PS Worker. 18 | */ 19 | package edu.snu.dolphin.ps.worker.partitioned.parameters; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Interfaces for representing MPI operation senders and receivers. 18 | */ 19 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Classes that are instantiated at an evaluator that acts as a Parameter Server worker (client). 18 | */ 19 | package edu.snu.dolphin.ps.worker; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/sub/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Codecs, reduce functions, and other classes that support REEF group communication. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.sub; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataGatherSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | public interface DataGatherSender { 19 | T sendGatherData(int iteration); 20 | } 21 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataReduceSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | public interface DataReduceSender { 19 | T sendReduceData(int iteration); 20 | } 21 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/layerparam/initializer/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Parameter initializer classes for each type of neural network layer. 18 | */ 19 | package edu.snu.dolphin.dnn.layerparam.initializer; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/regularization/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Regularization function classes for machine learning algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.regularization; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/driver/api/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Interfaces for driver-side Parameter Server classes that can have several pluggable implementations. 18 | */ 19 | package edu.snu.dolphin.ps.driver.api; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/driver/impl/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Implementations for driver-side Parameter Server classes in {@link edu.snu.dolphin.ps.driver.api}. 18 | */ 19 | package edu.snu.dolphin.ps.driver.impl; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/api/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Interfaces for server-side Parameter Server classes that can have several pluggable implementations. 18 | */ 19 | package edu.snu.dolphin.ps.server.api; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/api/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Interfaces for worker-side Parameter Server classes that can have several pluggable implementations. 18 | */ 19 | package edu.snu.dolphin.ps.worker.api; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/graph/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Machine learning Task classes and launchers for graph algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.graph; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataBroadcastSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | public interface DataBroadcastSender { 19 | T sendBroadcastData(int iteration); 20 | } 21 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataReduceReceiver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | public interface DataReduceReceiver { 19 | void receiveReduceData(int iteration, T data); 20 | } 21 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/clustering/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Machine learning Task classes and launchers for clustering algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.clustering; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/regression/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Machine learning Task classes and launchers for regression algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.regression; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataScatterReceiver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | public interface DataScatterReceiver { 19 | void receiveScatterData(int iteration, T data); 20 | } 21 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/concurrent/api/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Interfaces for server-side Parameter Server classes that can have several pluggable implementations. 18 | */ 19 | package edu.snu.dolphin.ps.server.concurrent.api; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/simple/SimpleCommGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.simple; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | 20 | public final class SimpleCommGroup implements Name { 21 | } 22 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataBroadcastReceiver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | public interface DataBroadcastReceiver { 19 | void receiveBroadcastData(int iteration, T data); 20 | } 21 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/classification/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Machine learning Task classes and launchers for classification algorithms. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.classification; 20 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/concurrent/impl/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Implementations for server-side Parameter Server classes in {@link edu.snu.dolphin.ps.server.concurrent.api}. 18 | */ 19 | package edu.snu.dolphin.ps.server.concurrent.impl; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/clustering/em/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Machine learning Task classes and launchers for the expectation–maximization algorithm. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.clustering.em; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/CommunicationGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | 20 | public class CommunicationGroup implements Name { 21 | } 22 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataScatterSender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | import java.util.List; 19 | 20 | public interface DataScatterSender { 21 | List sendScatterData(int iteration); 22 | } 23 | -------------------------------------------------------------------------------- /dolphin-bsp/src/test/resources/data/pagerank: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Seoul National University 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # name: pagerank 16 | # description: an example in https://en.wikipedia.org/wiki/PageRank 17 | 1 18 | 2 3 19 | 3 2 20 | 4 1 2 21 | 5 2 4 6 22 | 6 2 5 23 | 7 2 5 24 | 8 2 5 25 | 9 2 5 26 | 10 5 27 | 11 5 28 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/clustering/kmeans/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Machine learning Task classes and launchers for the k-means clustering algorithm. 18 | */ 19 | package edu.snu.dolphin.bsp.examples.ml.algorithms.clustering.kmeans; 20 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/interfaces/DataGatherReceiver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.interfaces; 17 | 18 | import java.util.List; 19 | 20 | public interface DataGatherReceiver { 21 | void receiveGatherData(int iteration, List data); 22 | } 23 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/concurrent/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Concurrent implementation of Parameter Server worker (client). 18 | * For use with server in {@link edu.snu.dolphin.ps.server.concurrent}. 19 | */ 20 | package edu.snu.dolphin.ps.worker.concurrent; 21 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/partitioned/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * Partitioned implementation of Parameter Server worker (client). 18 | * For use with server in {@link edu.snu.dolphin.ps.server.partitioned}. 19 | */ 20 | package edu.snu.dolphin.ps.worker.partitioned; 21 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/clustering/em/EMMainCommGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.algorithms.clustering.em; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | 20 | public final class EMMainCommGroup implements Name { 21 | } 22 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/ParseException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | /** 19 | * Exception used by DataParser. 20 | */ 21 | public final class ParseException extends Exception { 22 | public ParseException(final String msg) { 23 | super(msg); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/clustering/ClusteringPreCommGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.algorithms.clustering; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | 20 | public final class ClusteringPreCommGroup implements Name { 21 | } 22 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/clustering/kmeans/KMeansMainCommGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.algorithms.clustering.kmeans; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | 20 | public final class KMeansMainCommGroup implements Name { 21 | } 22 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/key/Centroids.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.key; 17 | 18 | import edu.snu.dolphin.bsp.core.Key; 19 | import org.apache.mahout.math.Vector; 20 | 21 | import java.util.List; 22 | 23 | public final class Centroids implements Key> { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/CommunicationGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Name of the single communication group") 22 | public final class CommunicationGroup implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/driver/impl/ServerId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.driver.impl; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "server identifier for Network Connection Service") 22 | public final class ServerId implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/test/java/edu/snu/dolphin/bsp/examples/ml/algorithms/graph/GraphTestSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.algorithms.graph; 17 | 18 | import org.junit.runner.RunWith; 19 | import org.junit.runners.Suite; 20 | 21 | @RunWith(Suite.class) 22 | @Suite.SuiteClasses({ 23 | PageRankTest.class 24 | }) 25 | public final class GraphTestSuite { 26 | } 27 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/ns/EndpointId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.ns; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Network Connection Service identifier of a Parameter Server evaluator") 22 | public final class EndpointId implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/names/DataGather.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.names; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Name for the operation used to reduce data") 22 | public final class DataGather implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/names/DataReduce.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.names; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Name for the operation used to reduce data") 22 | public final class DataReduce implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/names/DataScatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.names; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Name for the operation used to broadcast data") 22 | public final class DataScatter implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/names/DataBroadcast.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.names; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Name for the operation used to broadcast data") 22 | public final class DataBroadcast implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/JobIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Name of job", 22 | short_name = "jobName") 23 | public final class JobIdentifier implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/CtrlMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | import java.io.Serializable; 19 | 20 | public enum CtrlMessage implements Serializable { 21 | 22 | // run the next iteration of the main loop 23 | RUN, 24 | 25 | // break the main loop of @ComputeTask and run cleanup before terminating the current task 26 | TERMINATE 27 | } 28 | 29 | -------------------------------------------------------------------------------- /dolphin-bsp/src/test/java/edu/snu/dolphin/bsp/examples/ml/algorithms/regression/RegressionTestSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.algorithms.regression; 17 | 18 | import org.junit.runner.RunWith; 19 | import org.junit.runners.Suite; 20 | 21 | @RunWith(Suite.class) 22 | @Suite.SuiteClasses({ 23 | LinearRegTest.class 24 | }) 25 | public final class RegressionTestSuite { 26 | } 27 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/OutputDir.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Directory to write output data to", 22 | short_name = "output") 23 | public final class OutputDir implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/groupcomm/names/CtrlMsgBroadcast.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.groupcomm.names; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Name for the operation used to manage control messages") 22 | public final class CtrlMsgBroadcast implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/InputDir.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "File or directory to read input data from", 22 | short_name = "input") 23 | public final class InputDir implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-bsp/src/test/java/edu/snu/dolphin/bsp/examples/ml/algorithms/classification/ClassificationTestSuite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.algorithms.classification; 17 | 18 | import org.junit.runner.RunWith; 19 | import org.junit.runners.Suite; 20 | 21 | @RunWith(Suite.class) 22 | @Suite.SuiteClasses({ 23 | LogisticRegTest.class 24 | }) 25 | public final class ClassificationTestSuite { 26 | } 27 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/blas/function/Function.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.dnn.blas.function; 17 | 18 | import edu.snu.dolphin.dnn.blas.Matrix; 19 | 20 | /** 21 | * Function interface. 22 | */ 23 | public interface Function { 24 | 25 | Matrix apply(Matrix m); 26 | 27 | Matrix applyi(Matrix m); 28 | 29 | Matrix derivative(Matrix m); 30 | 31 | Matrix derivativei(Matrix m); 32 | } 33 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/parameters/JobTimeout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.examples.add.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Job Timeout in ms", default_value = "15000", short_name = "timeout") 22 | public final class JobTimeout implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/Dimension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "feature dimension of the data set", 22 | short_name = "dim") 23 | public final class Dimension implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/parameters/NumUpdates.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.examples.add.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Total number of updates", default_value = "100", short_name = "numUpdates") 22 | public final class NumUpdates implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/common/partitioned/parameters/NumServers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.common.partitioned.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Number of servers", default_value = "1", short_name = "numServers") 22 | public final class NumServers implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/Timeout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Time allowed until job ends", 22 | short_name = "timeout", 23 | default_value = "100000") 24 | public final class Timeout implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/scheduling/SchedulabilityAnalyzer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.scheduling; 17 | 18 | /** 19 | * Determines whether dolphin's Gang Scheduling is possible for the Runtime and its current resources. 20 | */ 21 | public interface SchedulabilityAnalyzer { 22 | /** 23 | * @return whether scheduling is possible under current resource conditions. 24 | */ 25 | boolean isSchedulable(); 26 | } 27 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/parameters/NumWorkers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.examples.add.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Number of workers to run updater tasks on", default_value = "1", short_name = "numWorkers") 22 | public final class NumWorkers implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/ns/PSMessageHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.ns; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | import org.apache.reef.wake.EventHandler; 21 | 22 | @NamedParameter(doc = "Network message receive handler for Parameter Servers and Workers") 23 | public final class PSMessageHandler implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/NumberOfClusters.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Number of clusters", 22 | short_name = "numCls", 23 | default_value = "5") 24 | public final class NumberOfClusters implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/OnLocal.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Whether or not to run on local machine", 22 | short_name = "local", 23 | default_value = "false") 24 | public final class OnLocal implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/DesiredSplits.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Number of evaluators to run job with", 22 | short_name = "split", 23 | default_value = "1") 24 | public final class DesiredSplits implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/metric/MetricCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core.metric; 17 | 18 | import org.apache.reef.tang.annotations.DefaultImplementation; 19 | import org.apache.reef.wake.remote.Codec; 20 | 21 | import java.util.Map; 22 | 23 | /** 24 | * Interface of codecs for metrics. 25 | */ 26 | @DefaultImplementation(DefaultMetricCodecImpl.class) 27 | public interface MetricCodec extends Codec> { 28 | } 29 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/Lambda.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "value for the lambda constant", 22 | short_name = "lambda", 23 | default_value = "1") 24 | public final class Lambda implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/common/partitioned/parameters/NumPartitions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.common.partitioned.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Number of partitions across all servers", default_value = "2", short_name = "numPartitions") 22 | public final class NumPartitions implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/partitioned/parameters/WorkerNumPartitions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.worker.partitioned.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Number of partitions", default_value = "2", short_name = "workerNumPartitions") 22 | public final class WorkerNumPartitions implements Name { 23 | } 24 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/StepSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "initial value of the step size", 22 | short_name = "stepSize", 23 | default_value = "1e-2") 24 | public final class StepSize implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/EvaluatorSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Evaluators with this size will be requested (in MBs)", 22 | short_name = "evalSize", 23 | default_value = "128") 24 | public final class EvaluatorSize implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/partitioned/parameters/WorkerExpireTimeout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.worker.partitioned.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Time in ms to expire local cache", default_value = "1000", 22 | short_name = "workerExpireTimeout") 23 | public final class WorkerExpireTimeout implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/partitioned/parameters/WorkerKeyCacheSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.worker.partitioned.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Max number of encoded keys to cache", default_value = "100", 22 | short_name = "workerKeyCacheSize") 23 | public final class WorkerKeyCacheSize implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/DampingFactor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Damping factor value for PageRank algorithm", 22 | short_name = "dampingFactor", 23 | default_value = "0.85") 24 | public final class DampingFactor implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/partitioned/parameters/ServerQueueSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.server.partitioned.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Max number of items that can be queued for each partition", default_value = "1000", 22 | short_name = "serverQueueSize") 23 | public final class ServerQueueSize implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/partitioned/parameters/WorkerQueueSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.worker.partitioned.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Max number of items that can be queued for each partition", default_value = "100", 22 | short_name = "workerQueueSize") 23 | public final class WorkerQueueSize implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/UserJobInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * Interface for a user-defined job, which is a unit of work in Dolphin. 22 | * This class should be implemented by a user-defined job 23 | * which specify a data parser and stages composing the job 24 | */ 25 | public interface UserJobInfo { 26 | List getStageInfoList(); 27 | 28 | Class getDataParser(); 29 | } 30 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/IsCovarianceShared.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Whether to share a covariance matrix among clusters or not", 22 | short_name = "isCovShared", 23 | default_value = "false") 24 | public class IsCovarianceShared implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/common/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.common; 17 | 18 | /** 19 | * Constants used across Driver/Server/Worker boundaries. 20 | */ 21 | public final class Constants { 22 | /** 23 | * Empty private constructor to prohibit instantiation of utility class. 24 | */ 25 | private Constants() { 26 | } 27 | 28 | public static final String SERVER_ID_PREFIX = "SERVER_ID_"; 29 | public static final String WORKER_ID_PREFIX = "WORKER_ID_"; 30 | } 31 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/ConvergenceThreshold.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Threshold value for algorithm convergence", 22 | short_name = "convThr", 23 | default_value = "0.0001") 24 | public final class ConvergenceThreshold implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/IsCovarianceDiagonal.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Whether to assume that covariance matrices are diagonal or not", 22 | short_name = "isCovDiag", 23 | default_value = "true") 24 | public class IsCovarianceDiagonal implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/parameters/MaxIterations.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Maximum number of iterations allowed before algorithm stops", 22 | short_name = "maxIter", 23 | default_value = "20") 24 | public final class MaxIterations implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/parameters/LocalRuntimeMaxNumEvaluators.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Maximum number of local runtime evaluators, must be at least Data Loading Splits + 1", 22 | short_name = "maxNumEvalLocal", default_value = "2") 23 | public final class LocalRuntimeMaxNumEvaluators implements Name { 24 | } 25 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/partitioned/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * A Partitioned Parameter Server, that supports atomic, in-order processing of push and pull operations. 18 | * Partitions are based on the hash of the key. 19 | * Each partition consists of a queue, kvStore, and thread. 20 | * Operations are immediately added to the queue, so the message processing threads will not block, 21 | * except when the queue is full. 22 | */ 23 | package edu.snu.dolphin.ps.server.partitioned; 24 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/parameters/StartKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.examples.add.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Start key to apply updates on: updates are done on [KeyStart, KeyStart + NumKeys)", 22 | default_value = "0", 23 | short_name = "startKey") 24 | public final class StartKey implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/parameters/NumKeys.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.examples.add.parameters; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | @NamedParameter(doc = "Number of keys to apply updates on: updates are done on [KeyStart, KeyStart + NumKeys)", 22 | default_value = "8", 23 | short_name = "numKeys") 24 | public final class NumKeys implements Name { 25 | } 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/DataParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | /** 19 | * Interface for parse function class to be used with DataParseService. 20 | */ 21 | public interface DataParser { 22 | 23 | /** 24 | * @return parsed data as format T 25 | * @throws ParseException parsing has failed due to incorrect input format 26 | */ 27 | T get() throws ParseException; 28 | 29 | /** 30 | * Parse data input and keep it for later get() calls. 31 | */ 32 | void parse(); 33 | } 34 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/metric/MetricTracker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core.metric; 17 | 18 | import java.util.Map; 19 | 20 | /** 21 | * Interface that metric trackers implement. 22 | */ 23 | public interface MetricTracker extends AutoCloseable { 24 | 25 | /** 26 | * start tracking metrics. 27 | */ 28 | void start(); 29 | 30 | /** 31 | * stop tracking metrics and return currently tracked metrics. 32 | * @return key and value of metrics 33 | */ 34 | Map stop(); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /** 17 | * A simple example that uses the Parameter Server and AddUpdater. 18 | * It pushes updates from multiple workers by running UpdaterTask, 19 | * then pulls and validates the resulting values by running ValidatorTask. 20 | * 21 | * The example has two main purposes: 22 | * 1. It shows how a PS implementation can be configured. 23 | * 2. It can also work as a simple load generator to test PS implementations. 24 | */ 25 | package edu.snu.dolphin.ps.examples.add; 26 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/data/Row.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.data; 17 | 18 | import org.apache.mahout.math.Vector; 19 | 20 | public final class Row { 21 | private final double output; 22 | private final Vector feature; 23 | 24 | public Row(final double output, final Vector feature) { 25 | this.output = output; 26 | this.feature = feature; 27 | } 28 | 29 | public double getOutput() { 30 | return output; 31 | } 32 | 33 | public Vector getFeature() { 34 | return feature; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/metric/MetricTrackers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core.metric; 17 | 18 | import org.apache.reef.tang.annotations.Name; 19 | import org.apache.reef.tang.annotations.NamedParameter; 20 | 21 | import java.util.Set; 22 | 23 | /** 24 | * The set of MetricTracker implementations registered to MetricManager. 25 | */ 26 | @NamedParameter(doc = "The set of MetricTracker implementations registered to MetricManager.", 27 | default_classes = {}) 28 | public final class MetricTrackers implements Name> { 29 | } 30 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/data/VectorDistanceMeasure.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.data; 17 | 18 | import org.apache.mahout.math.Vector; 19 | import org.apache.reef.tang.annotations.DefaultImplementation; 20 | 21 | /** 22 | * Interface for a distance measure used to compute distance between points. 23 | */ 24 | @DefaultImplementation(EuclideanDistance.class) 25 | public interface VectorDistanceMeasure { 26 | 27 | /** 28 | * Compute distance between @param v1 and @param v2. 29 | */ 30 | double distance(Vector v1, Vector v2); 31 | } 32 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/converge/PageRankConvCond.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.converge; 17 | 18 | import org.apache.reef.tang.annotations.DefaultImplementation; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * Interface for a check function that decides whether 24 | * PageRank algorithm has converged or not. 25 | */ 26 | @DefaultImplementation(PageRankConvEuclidean.class) 27 | public interface PageRankConvCond { 28 | 29 | /** 30 | * Check convergence conditions. 31 | */ 32 | boolean checkConvergence(Map rank); 33 | } 34 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/converge/ClusteringConvCond.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.converge; 17 | 18 | import org.apache.mahout.math.Vector; 19 | import org.apache.reef.tang.annotations.DefaultImplementation; 20 | 21 | /** 22 | * Interface for a check function that decides whether 23 | * a clustering algorithm has converged or not. 24 | */ 25 | @DefaultImplementation(ClusteringConvEuclidean.class) 26 | public interface ClusteringConvCond { 27 | 28 | /** 29 | * Check convergence conditions. 30 | */ 31 | boolean checkConvergence(Iterable centroids); 32 | } 33 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/simple/SimpleReduceFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.simple; 17 | 18 | import org.apache.reef.io.network.group.api.operators.Reduce; 19 | 20 | import javax.inject.Inject; 21 | 22 | public class SimpleReduceFunction implements Reduce.ReduceFunction { 23 | 24 | @Inject 25 | public SimpleReduceFunction() { 26 | } 27 | 28 | @Override 29 | public final Integer apply(final Iterable dataList) { 30 | Integer sum = 0; 31 | for (final Integer data : dataList) { 32 | sum += data; 33 | } 34 | return sum; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/UserParameters.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | import org.apache.reef.tang.Configuration; 19 | import org.apache.reef.tang.annotations.DefaultImplementation; 20 | 21 | /** 22 | * Interface for providing configurations setting user-defined (algorithmic-specific, application-specific) parameters. 23 | */ 24 | @DefaultImplementation(UserParametersImpl.class) 25 | public interface UserParameters { 26 | Configuration getDriverConf(); 27 | 28 | Configuration getServiceConf(); 29 | 30 | Configuration getUserCmpTaskConf(); 31 | 32 | Configuration getUserCtrlTaskConf(); 33 | } 34 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/converge/LinearModelConvCond.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.converge; 17 | 18 | import edu.snu.dolphin.bsp.examples.ml.data.LinearModel; 19 | import org.apache.reef.tang.annotations.DefaultImplementation; 20 | 21 | /** 22 | * Interface for a check function that decides whether 23 | * a linear classification/regression algorithm has converged or not. 24 | */ 25 | @DefaultImplementation(LinearModelConvEuclidean.class) 26 | public interface LinearModelConvCond { 27 | 28 | /** 29 | * Check convergence conditions. 30 | */ 31 | boolean checkConvergence(LinearModel model); 32 | } 33 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/server/partitioned/PartitionedServerSideReplySender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.server.partitioned; 17 | 18 | /** 19 | * Sender for PartitionedParameterServer. 20 | */ 21 | public interface PartitionedServerSideReplySender { 22 | /** 23 | * Implementing classes must serialize K, V immediately within the calling thread, 24 | * to ensure atomicity of updates. 25 | * @param destId the destination's network address 26 | * @param key key, to be serialized immediately 27 | * @param value value, to be serialized immediately 28 | */ 29 | void sendReplyMsg(String destId, K key, V value); 30 | } 31 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/AsyncWorkerHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.worker; 17 | 18 | /** 19 | * Process a pull reply message received from the server. 20 | * This is an internal interface, to be used to connect the {@link WorkerSideMsgHandler} 21 | * to a {@link edu.snu.dolphin.ps.worker.api.ParameterWorker}. 22 | */ 23 | public interface AsyncWorkerHandler { 24 | /** 25 | * Reply to the worker with a {@code value} that was previously requested by {@code pull}. 26 | * @param key key object representing what was sent 27 | * @param value value sent from the server 28 | */ 29 | void processReply(K key, V value); 30 | } 31 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/metric/MetricException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core.metric; 17 | 18 | /** 19 | * Thrown when metric tracking fails. Metric tracking commonly fails for two reasons. 20 | * The first is that the tracking is started again before the previous tracking is stopped. 21 | * The second is that tracking is stopped before being started. 22 | */ 23 | public class MetricException extends Exception { 24 | 25 | /** 26 | * Constructs a new exception with the specified detail message. 27 | * @param message the detail message. 28 | */ 29 | public MetricException(final String message) { 30 | super(message); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/algorithms/clustering/ClusteringPreStageBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.algorithms.clustering; 17 | 18 | import edu.snu.dolphin.bsp.core.StageInfo; 19 | import edu.snu.dolphin.bsp.examples.ml.sub.VectorListCodec; 20 | 21 | public final class ClusteringPreStageBuilder { 22 | 23 | /** 24 | * Should not be instantiated. 25 | */ 26 | private ClusteringPreStageBuilder() { 27 | } 28 | 29 | public static StageInfo build() { 30 | return StageInfo.newBuilder(ClusteringPreCmpTask.class, ClusteringPreCtrlTask.class, ClusteringPreCommGroup.class) 31 | .setGather(VectorListCodec.class).build(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /bin/run_ps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2016 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run_ps.sh edu.snu.dolphin.ps.examples.add.ConcurrentPSExampleREEF -numUpdates 960 -timeout 360000 -numWorkers 4 18 | 19 | # RUNTIME 20 | SELF_JAR=`echo dolphin-ps/target/dolphin-ps-*-shaded.jar` 21 | 22 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 23 | 24 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 25 | 26 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 27 | 28 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOCAL_RUNTIME_TMP $LOGGING_CONFIG $*" 29 | echo $CMD 30 | $CMD # 2> /dev/null 31 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/KeyValueStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | 19 | import javax.inject.Inject; 20 | import java.util.HashMap; 21 | 22 | /** 23 | * Simple Key-value store used by key-value store service. 24 | */ 25 | public final class KeyValueStore { 26 | private final HashMap, Object> hashMap; 27 | 28 | @Inject 29 | public KeyValueStore() { 30 | hashMap = new HashMap<>(); 31 | } 32 | 33 | public void put(final Class> key, final T value) { 34 | hashMap.put(key, value); 35 | } 36 | 37 | @SuppressWarnings("unchecked") 38 | public T get(final Class> key) { 39 | return (T) hashMap.get(key); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2015 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run.sh -local true -split 4 -input dolphin-bsp/src/test/resources/data/sample -output output_simple -maxNumEvalLocal 5 18 | 19 | SELF_JAR=`echo dolphin-bsp/target/dolphin-bsp-*-shaded.jar` 20 | 21 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 22 | 23 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 24 | 25 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 26 | 27 | ALG=edu.snu.dolphin.bsp.examples.simple.SimpleREEF 28 | 29 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOGGING_CONFIG $ALG $*" 30 | echo $CMD 31 | $CMD 32 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/loss/SquareLoss.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.loss; 17 | 18 | import org.apache.mahout.math.Vector; 19 | 20 | import javax.inject.Inject; 21 | 22 | /** 23 | * Represents the regularize for linear regression (least mean squares). 24 | */ 25 | public final class SquareLoss implements Loss { 26 | 27 | @Inject 28 | public SquareLoss() { 29 | } 30 | 31 | @Override 32 | public double loss(final double predict, final double output) { 33 | return Math.pow(predict - output, 2) / 2; 34 | } 35 | 36 | @Override 37 | public Vector gradient(final Vector feature, final double predict, final double output) { 38 | return feature.times(predict - output); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Dolphin 2 | ======= 3 | 4 | Dolphin is a machine learning platform built on top of [Apache REEF](http://reef.apache.org/). Dolphin consists of a BSP-style machine learning framework (`dolphin-bsp`), a deep learning framework (`dolphin-dnn`), and a parameter server module (`dolphin-ps`). 5 | 6 | ### Submodules 7 | 8 | * [`dolphin-bsp`](dolphin-bsp/README.md): BSP-style framework for multi-staged processing. 9 | * [`dolphin-dnn`](dolphin-dnn/README.md): Deep learning framework for training deep neural network models on large datasets. 10 | * [`dolphin-ps`](dolphin-ps/README.md): Parameter server module for asynchronous machine learning algorithms. 11 | 12 | ### How to build and run Dolphin 13 | 14 | 1. Build REEF: check https://cwiki.apache.org/confluence/display/REEF/Compiling+REEF 15 | Currently, Dolphin depends on REEF `0.14.0-SNAPSHOT`, which means you must build the current snapshot of REEF before building Dolphin. We will move to the release version `0.14.0` once it's out. 16 | 17 | 2. Build Dolphin: 18 | ``` 19 | git clone https://github.com/cmssnu/dolphin 20 | cd dolphin 21 | mvn clean install 22 | ``` 23 | 24 | 25 | 3. Run Dolphin: check the READMEs in the submodules for more details. 26 | 27 | ### Dolphin Mailing List 28 | Bug reports and feature requests are welcome, as well as even simple questions! 29 | Contact us and share your thoughts by subscribing to dolphin-discussion@googlegroups.com. 30 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/concurrent/ConcurrentWorkerHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.worker.concurrent; 17 | 18 | import edu.snu.dolphin.ps.worker.AsyncWorkerHandler; 19 | 20 | import javax.inject.Inject; 21 | 22 | public final class ConcurrentWorkerHandler implements AsyncWorkerHandler { 23 | private final ConcurrentParameterWorker concurrentParameterWorker; 24 | 25 | @Inject 26 | private ConcurrentWorkerHandler(final ConcurrentParameterWorker concurrentParameterWorker) { 27 | this.concurrentParameterWorker = concurrentParameterWorker; 28 | } 29 | 30 | @Override 31 | public void processReply(final K key, final V value) { 32 | concurrentParameterWorker.processReply(key, value); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/loss/HingeLoss.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.loss; 17 | 18 | import org.apache.mahout.math.Vector; 19 | 20 | import javax.inject.Inject; 21 | 22 | /** 23 | * Represents regularize function for hinge regularize (mainly used in Support Vector Machines). 24 | */ 25 | public final class HingeLoss implements Loss { 26 | 27 | @Inject 28 | public HingeLoss() { 29 | } 30 | 31 | @Override 32 | public double loss(final double predict, final double output) { 33 | return Math.max(0, 1 - output * predict); 34 | } 35 | 36 | @Override 37 | public Vector gradient(final Vector feature, final double predict, final double output) { 38 | return feature.times(predict * output >= 1 ? 0 : -output); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/layerparam/provider/ParameterProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.dnn.layerparam.provider; 17 | 18 | import edu.snu.dolphin.dnn.layers.LayerParameter; 19 | 20 | /** 21 | * Interface for parameter provider that gathers parameter gradients and provides updated parameters. 22 | */ 23 | public interface ParameterProvider { 24 | 25 | /** 26 | * Pushes parameter gradients for an input batch. 27 | * @param batchSize the size of an input batch 28 | * @param parameterGradients parameter gradients sums for an input batch 29 | */ 30 | void push(final int batchSize, final LayerParameter[] parameterGradients); 31 | 32 | /** 33 | * @return the updated parameters of the whole network. 34 | */ 35 | LayerParameter[] pull(); 36 | } 37 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/worker/partitioned/PartitionedWorkerHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.worker.partitioned; 17 | 18 | import edu.snu.dolphin.ps.worker.AsyncWorkerHandler; 19 | 20 | import javax.inject.Inject; 21 | 22 | public final class PartitionedWorkerHandler implements AsyncWorkerHandler { 23 | private final PartitionedParameterWorker partitionedParameterWorker; 24 | 25 | @Inject 26 | private PartitionedWorkerHandler(final PartitionedParameterWorker partitionedParameterWorker) { 27 | this.partitionedParameterWorker = partitionedParameterWorker; 28 | } 29 | 30 | @Override 31 | public void processReply(final K key, final V value) { 32 | partitionedParameterWorker.processReply(key, value); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /bin/run_neuralnet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2015 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run_neuralnet.sh -local true -maxIter 100 -conf dolphin-dnn/src/test/resources/configuration/neuralnet -input dolphin-dnn/src/test/resources/data/neuralnet -timeout 800000 18 | 19 | SELF_JAR=`echo dolphin-dnn/target/dolphin-dnn-*-shaded.jar` 20 | 21 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 22 | 23 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 24 | 25 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 26 | 27 | ALG=edu.snu.dolphin.dnn.NeuralNetworkREEF 28 | 29 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOGGING_CONFIG $ALG $*" 30 | echo $CMD 31 | $CMD 32 | -------------------------------------------------------------------------------- /bin/run_em.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2015 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run_em.sh -numCls 4 -convThr 0.01 -maxIter 20 -local true -split 4 -input dolphin-bsp/src/test/resources/data/clustering -output output_em -maxNumEvalLocal 5 18 | 19 | SELF_JAR=`echo dolphin-bsp/target/dolphin-bsp-*-shaded.jar` 20 | 21 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 22 | 23 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 24 | 25 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 26 | 27 | ALG=edu.snu.dolphin.bsp.examples.ml.algorithms.clustering.em.EMREEF 28 | 29 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOGGING_CONFIG $ALG $*" 30 | echo $CMD 31 | $CMD 32 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/data/ListReduceFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.dnn.data; 17 | 18 | import org.apache.reef.io.network.group.api.operators.Reduce; 19 | 20 | import javax.inject.Inject; 21 | import java.util.LinkedList; 22 | import java.util.List; 23 | 24 | /** 25 | * MPI Reduce function for a generic Java List. 26 | * Aggregates several lists into one big list. 27 | */ 28 | public final class ListReduceFunction implements Reduce.ReduceFunction { 29 | 30 | @Inject 31 | private ListReduceFunction() { 32 | } 33 | 34 | @Override 35 | public List apply(final Iterable elements) { 36 | final List retList = new LinkedList(); 37 | for (final List element : elements) { 38 | retList.addAll(element); 39 | } 40 | return retList; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /dolphin-ps/src/test/java/edu/snu/dolphin/ps/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps; 17 | 18 | import java.util.concurrent.ExecutorService; 19 | import java.util.concurrent.Executors; 20 | 21 | public final class TestUtils { 22 | 23 | /** 24 | * Should not be instantiated. 25 | */ 26 | private TestUtils() { 27 | } 28 | 29 | /** 30 | * Use a thread pool to concurrently execute threads. 31 | * Note that this method does NOT wait for the termination of all threads before returning. 32 | */ 33 | public static void runConcurrently(final Runnable[] threads) throws InterruptedException { 34 | final ExecutorService pool = Executors.newFixedThreadPool(threads.length); 35 | for (final Runnable thread : threads) { 36 | pool.submit(thread); 37 | } 38 | pool.shutdown(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /bin/run_kmeans.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2015 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run_kmeans.sh -numCls 4 -convThr 0.01 -maxIter 20 -local true -split 4 -input dolphin-bsp/src/test/resources/data/clustering -output output_kmeans -maxNumEvalLocal 5 18 | 19 | SELF_JAR=`echo dolphin-bsp/target/dolphin-bsp-*-shaded.jar` 20 | 21 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 22 | 23 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 24 | 25 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 26 | 27 | ALG=edu.snu.dolphin.bsp.examples.ml.algorithms.clustering.kmeans.KMeansREEF 28 | 29 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOGGING_CONFIG $ALG $*" 30 | echo $CMD 31 | $CMD 32 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/scheduling/YarnSchedulabilityAnalyzer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.scheduling; 17 | 18 | import javax.inject.Inject; 19 | 20 | // TODO: This class should be implemented and used as a part of changes to support Gang Scheduling in Yarn. 21 | /** 22 | * Not yet implemented, so always returns isSchedulable as true. 23 | * A future implementation should take into account total Yarn resources and 24 | * queue quotas to determine if Gang Scheduling for dolphin is feasible. 25 | */ 26 | public final class YarnSchedulabilityAnalyzer implements SchedulabilityAnalyzer { 27 | 28 | @Inject 29 | private YarnSchedulabilityAnalyzer() { 30 | } 31 | 32 | /** 33 | * @return true 34 | */ 35 | @Override 36 | public boolean isSchedulable() { 37 | return true; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /bin/run_pagerank.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2015 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run_pagerank.sh -convThr 0.01 -maxIter 10 -dampingFactor 0.85 -local true -split 1 -input dolphin-bsp/src/test/resources/data/pagerank -output output_pagerank -maxNumEvalLocal 2 18 | 19 | SELF_JAR=`echo dolphin-bsp/target/dolphin-bsp-*-shaded.jar` 20 | 21 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 22 | 23 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 24 | 25 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 26 | 27 | ALG=edu.snu.dolphin.bsp.examples.ml.algorithms.graph.PageRankREEF 28 | 29 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOGGING_CONFIG $ALG $*" 30 | echo $CMD 31 | $CMD 32 | -------------------------------------------------------------------------------- /bin/run_regression.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2015 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run_regression.sh -dim 3 -maxIter 20 -stepSize 0.001 -lambda 0.1 -local true -split 4 -input dolphin-bsp/src/test/resources/data/regression -output output_regression -maxNumEvalLocal 5 18 | 19 | SELF_JAR=`echo dolphin-bsp/target/dolphin-bsp-*-shaded.jar` 20 | 21 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 22 | 23 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 24 | 25 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 26 | 27 | ALG=edu.snu.dolphin.bsp.examples.ml.algorithms.regression.LinearRegREEF 28 | 29 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOGGING_CONFIG $ALG $*" 30 | echo $CMD 31 | $CMD 32 | -------------------------------------------------------------------------------- /bin/run_logistic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright (C) 2015 Seoul National University 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # EXAMPLE USAGE 17 | # bin/run_logistic.sh -dim 3 -maxIter 20 -stepSize 0.00001 -lambda 0.1 -local true -split 4 -input dolphin-bsp/src/test/resources/data/classification -output output_logistic -maxNumEvalLocal 5 18 | 19 | SELF_JAR=`echo dolphin-bsp/target/dolphin-bsp-*-shaded.jar` 20 | 21 | LOGGING_CONFIG='-Djava.util.logging.config.class=org.apache.reef.util.logging.Config' 22 | 23 | CLASSPATH=$YARN_HOME/share/hadoop/common/*:$YARN_HOME/share/hadoop/common/lib/*:$YARN_HOME/share/hadoop/yarn/*:$YARN_HOME/share/hadoop/hdfs/*:$YARN_HOME/share/hadoop/mapreduce/lib/*:$YARN_HOME/share/hadoop/mapreduce/* 24 | 25 | YARN_CONF_DIR=$YARN_HOME/etc/hadoop 26 | 27 | ALG=edu.snu.dolphin.bsp.examples.ml.algorithms.classification.LogisticRegREEF 28 | 29 | CMD="java -cp $YARN_CONF_DIR:$SELF_JAR:$CLASSPATH $LOGGING_CONFIG $ALG $*" 30 | echo $CMD 31 | $CMD 32 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/data/EuclideanDistance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.data; 17 | 18 | import org.apache.mahout.math.Vector; 19 | 20 | import javax.inject.Inject; 21 | 22 | /** 23 | * Implements DistanceMeasure (Default). 24 | */ 25 | public final class EuclideanDistance implements VectorDistanceMeasure { 26 | 27 | @Inject 28 | public EuclideanDistance() { 29 | } 30 | 31 | @Override 32 | public double distance(final Vector v1, final Vector v2) { 33 | if (v1.size() != v2.size()) { 34 | throw new IllegalArgumentException("Vector dimensions are not consistent"); 35 | } 36 | 37 | double distance = 0; 38 | for (int i = 0; i < v1.size(); i++) { 39 | distance += (v1.get(i) - v2.get(i)) * (v1.get(i) - v2.get(i)); 40 | } 41 | return Math.sqrt(distance); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ---------------------------------------------------------------------- 2 | # Folders created by the various REEF runtimes during testing 3 | # ---------------------------------------------------------------------- 4 | REEF_LOCAL_RUNTIME 5 | REEF_MESOS_RUNTIME 6 | # 7 | # ---------------------------------------------------------------------- 8 | # Files generated during builds 9 | # ---------------------------------------------------------------------- 10 | target 11 | generated 12 | build 13 | # 14 | # ---------------------------------------------------------------------- 15 | # IDE settings 16 | # ---------------------------------------------------------------------- 17 | .externalToolBuilders 18 | nbactions*.xml 19 | nb-configuration.xml 20 | .settings 21 | .classpath 22 | .project 23 | *.iml 24 | .idea 25 | atlassian-ide-plugin.xml 26 | *.sln.DotSettings.user 27 | # 28 | # ---------------------------------------------------------------------- 29 | # OS Files 30 | # ---------------------------------------------------------------------- 31 | .DS_Store 32 | # 33 | # ---------------------------------------------------------------------- 34 | # Temporary Files 35 | # ---------------------------------------------------------------------- 36 | tmp 37 | *~ 38 | \#* 39 | # 40 | # ---------------------------------------------------------------------- 41 | # Unknown Files. Please clean up over time 42 | # ---------------------------------------------------------------------- 43 | ml-data 44 | tang.conf 45 | *.sw[op] 46 | profile-*.json 47 | -------------------------------------------------------------------------------- /dolphin-dnn/src/test/resources/configuration/neuralnet: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Seoul National University 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # name: neuralnet 16 | # description: neural network configuration for MNIST database 17 | batch_size: 10 18 | stepsize: 1e-3 19 | input_shape { 20 | dim: 28 21 | dim: 28 22 | } 23 | parameter_provider { 24 | type: "local" 25 | } 26 | layer { 27 | type: "FullyConnected" 28 | fully_connected_param { 29 | init_weight: 1e-4 30 | init_bias: 2e-4 31 | num_output: 50 32 | } 33 | } 34 | layer { 35 | type: "Activation" 36 | activation_param { 37 | activation_function: "relu" 38 | } 39 | } 40 | layer { 41 | type: "FullyConnected" 42 | fully_connected_param { 43 | init_weight: 1e-2 44 | init_bias: 2e-2 45 | num_output: 10 46 | } 47 | } 48 | layer { 49 | type: "ActivationWithLoss" 50 | activation_with_loss_param { 51 | activation_function: "softmax" 52 | loss_function: "crossEntropy" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/IntegerCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.examples.add; 17 | 18 | import org.apache.reef.wake.remote.Codec; 19 | 20 | import javax.inject.Inject; 21 | import java.nio.ByteBuffer; 22 | 23 | /** 24 | * A simple codec for Integer. 25 | */ 26 | public final class IntegerCodec implements Codec, org.apache.reef.io.serialization.Codec { 27 | 28 | @Inject 29 | private IntegerCodec() { 30 | } 31 | 32 | @Override 33 | public Integer decode(final byte[] bytes) { 34 | final ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); 35 | return byteBuffer.getInt(); 36 | } 37 | 38 | @Override 39 | public byte[] encode(final Integer integer) { 40 | final ByteBuffer byteBuffer = ByteBuffer.allocate(Integer.SIZE / Byte.SIZE); 41 | byteBuffer.putInt(integer); 42 | return byteBuffer.array(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /dolphin-dnn/src/main/java/edu/snu/dolphin/dnn/layerparam/initializer/LayerParameterInitializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.dnn.layerparam.initializer; 17 | 18 | import edu.snu.dolphin.dnn.layers.LayerParameter; 19 | import org.apache.reef.tang.annotations.DefaultImplementation; 20 | 21 | /** 22 | * Interface for parameter initializer. 23 | * 24 | * The parameter initializer generates the initial parameter of the layer by the layer configuration. 25 | */ 26 | @DefaultImplementation(DefaultLayerParameterInitializer.class) 27 | public interface LayerParameterInitializer { 28 | 29 | /** 30 | * @return the initial parameter of the layer. 31 | */ 32 | LayerParameter generateInitialParameter(); 33 | 34 | /** 35 | * @return the index of the layer. 36 | */ 37 | int getIndex(); 38 | 39 | /** 40 | * @return the shape of output. 41 | */ 42 | int[] getOutputShape(); 43 | } 44 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/data/Model.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.data; 17 | 18 | import org.apache.mahout.math.Vector; 19 | import org.apache.reef.tang.annotations.DefaultImplementation; 20 | 21 | import java.io.Serializable; 22 | 23 | /** 24 | * Interface for representations of models used in ML job. 25 | */ 26 | @DefaultImplementation(LinearModel.class) 27 | public interface Model extends Serializable { 28 | 29 | /** 30 | * @return a Vector containing values that represent the model 31 | */ 32 | Vector getParameters(); 33 | 34 | /** 35 | * @param parameters set this model using these values 36 | */ 37 | void setParameters(Vector parameters); 38 | 39 | /** 40 | * @param point the Vector to perform prediction on 41 | * @return a predicted class value using the model 42 | */ 43 | double predict(Vector point); 44 | } 45 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/sub/PageRankReduceFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.sub; 17 | 18 | import edu.snu.dolphin.bsp.examples.ml.data.PageRankSummary; 19 | import org.apache.reef.io.network.group.api.operators.Reduce; 20 | 21 | import javax.inject.Inject; 22 | 23 | public class PageRankReduceFunction implements Reduce.ReduceFunction { 24 | 25 | @Inject 26 | public PageRankReduceFunction() { 27 | } 28 | 29 | @Override 30 | public final PageRankSummary apply(final Iterable summaryList) { 31 | PageRankSummary reducedSummary = null; 32 | for (final PageRankSummary summary : summaryList) { 33 | if (reducedSummary == null) { 34 | reducedSummary = summary; 35 | } else { 36 | reducedSummary.plus(summary); 37 | } 38 | } 39 | return reducedSummary; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/examples/add/AddUpdater.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.examples.add; 17 | 18 | import edu.snu.dolphin.ps.server.api.ParameterUpdater; 19 | 20 | import javax.inject.Inject; 21 | 22 | /** 23 | * A ParameterUpdater that add integers. The initial value is 0, and updates 24 | * are applied as oldValue + deltaValue. 25 | */ 26 | public final class AddUpdater implements ParameterUpdater { 27 | 28 | @Inject 29 | private AddUpdater() { 30 | } 31 | 32 | @Override 33 | public Integer process(final Integer key, final Integer preValue) { 34 | return preValue; 35 | } 36 | 37 | @Override 38 | public Integer update(final Integer oldValue, final Integer deltaValue) { 39 | return oldValue + deltaValue; 40 | } 41 | 42 | @Override 43 | public Integer initValue(final Integer key) { 44 | return 0; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/sub/LinearRegReduceFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.sub; 17 | 18 | import edu.snu.dolphin.bsp.examples.ml.data.LinearRegSummary; 19 | import org.apache.reef.io.network.group.api.operators.Reduce; 20 | 21 | import javax.inject.Inject; 22 | 23 | public class LinearRegReduceFunction implements Reduce.ReduceFunction { 24 | 25 | @Inject 26 | public LinearRegReduceFunction() { 27 | } 28 | 29 | @Override 30 | public final LinearRegSummary apply(final Iterable summaryList) { 31 | LinearRegSummary reducedSummary = null; 32 | for (final LinearRegSummary summary : summaryList) { 33 | if (reducedSummary == null) { 34 | reducedSummary = summary; 35 | } else { 36 | reducedSummary.plus(summary); 37 | } 38 | } 39 | return reducedSummary; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dolphin-dnn/src/test/resources/configuration/neuralnet_ps: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Seoul National University 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # name: neuralnet_ps 16 | # description: neural network configuration for MNIST database using a parameter server 17 | batch_size: 10 18 | stepsize: 1e-3 19 | input_shape { 20 | dim: 28 21 | dim: 28 22 | } 23 | parameter_provider { 24 | type: "parameterserver" 25 | } 26 | layer { 27 | type: "FullyConnected" 28 | fully_connected_param { 29 | init_weight: 1e-4 30 | init_bias: 2e-4 31 | num_output: 50 32 | } 33 | } 34 | layer { 35 | type: "Activation" 36 | activation_param { 37 | activation_function: "relu" 38 | } 39 | } 40 | layer { 41 | type: "FullyConnected" 42 | fully_connected_param { 43 | init_weight: 1e-2 44 | init_bias: 2e-2 45 | num_output: 10 46 | } 47 | } 48 | layer { 49 | type: "ActivationWithLoss" 50 | activation_with_loss_param { 51 | activation_function: "softmax" 52 | loss_function: "crossEntropy" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/data/LinearRegSummary.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.data; 17 | 18 | public class LinearRegSummary { 19 | private final LinearModel model; 20 | private int count = 0; 21 | private double loss = 0; 22 | 23 | public LinearRegSummary(final LinearModel model, final int count, final double loss) { 24 | this.model = model; 25 | this.count = count; 26 | this.loss = loss; 27 | } 28 | 29 | public void plus(final LinearRegSummary summary) { 30 | this.model.setParameters(this.model.getParameters().plus(summary.getModel().getParameters())); 31 | this.count += summary.count; 32 | this.loss += summary.loss; 33 | } 34 | 35 | public LinearModel getModel() { 36 | return model; 37 | } 38 | 39 | public int getCount() { 40 | return count; 41 | } 42 | 43 | public double getLoss() { 44 | return loss; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/sub/LogisticRegReduceFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.sub; 17 | 18 | import edu.snu.dolphin.bsp.examples.ml.data.LogisticRegSummary; 19 | import org.apache.reef.io.network.group.api.operators.Reduce; 20 | 21 | import javax.inject.Inject; 22 | 23 | public class LogisticRegReduceFunction implements Reduce.ReduceFunction { 24 | 25 | @Inject 26 | public LogisticRegReduceFunction() { 27 | } 28 | 29 | @Override 30 | public final LogisticRegSummary apply(final Iterable summaryList) { 31 | LogisticRegSummary reducedSummary = null; 32 | for (final LogisticRegSummary summary : summaryList) { 33 | if (reducedSummary == null) { 34 | reducedSummary = summary; 35 | } else { 36 | reducedSummary.plus(summary); 37 | } 38 | } 39 | return reducedSummary; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dolphin-dnn/src/test/resources/configuration/neuralnet_groupcomm: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2015 Seoul National University 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # name: neuralnet_groupcomm 16 | # description: neural network configuration for MNIST database using REEF's group communication 17 | batch_size: 3 18 | stepsize: 1e-3 19 | input_shape { 20 | dim: 28 21 | dim: 28 22 | } 23 | parameter_provider { 24 | type: "groupcomm" 25 | } 26 | layer { 27 | type: "FullyConnected" 28 | fully_connected_param { 29 | init_weight: 1e-4 30 | init_bias: 2e-4 31 | num_output: 50 32 | } 33 | } 34 | layer { 35 | type: "Activation" 36 | activation_param { 37 | activation_function: "relu" 38 | } 39 | } 40 | layer { 41 | type: "FullyConnected" 42 | fully_connected_param { 43 | init_weight: 1e-2 44 | init_bias: 2e-2 45 | num_output: 10 46 | } 47 | } 48 | layer { 49 | type: "ActivationWithLoss" 50 | activation_with_loss_param { 51 | activation_function: "softmax" 52 | loss_function: "crossEntropy" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/ns/ParameterServerMsgCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.ns; 17 | 18 | import edu.snu.dolphin.ps.avro.AvroParameterServerMsg; 19 | import edu.snu.dolphin.util.AvroUtils; 20 | import org.apache.reef.wake.remote.Codec; 21 | 22 | import javax.inject.Inject; 23 | 24 | /** 25 | * Codec for {@link AvroParameterServerMsg}s. 26 | * Simply uses {@link AvroUtils} to encode and decode messages. 27 | */ 28 | public final class ParameterServerMsgCodec implements Codec { 29 | 30 | @Inject 31 | private ParameterServerMsgCodec() { 32 | } 33 | 34 | @Override 35 | public byte[] encode(final AvroParameterServerMsg msg) { 36 | return AvroUtils.toBytes(msg, AvroParameterServerMsg.class); 37 | } 38 | 39 | @Override 40 | public AvroParameterServerMsg decode(final byte[] data) { 41 | return AvroUtils.fromBytes(data, AvroParameterServerMsg.class); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /dolphin-ps/src/main/java/edu/snu/dolphin/ps/driver/api/ParameterServerManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.ps.driver.api; 17 | 18 | import org.apache.reef.annotations.audience.DriverSide; 19 | import org.apache.reef.tang.Configuration; 20 | 21 | /** 22 | * Driver-side manager for the Parameter Server. 23 | * Service configuration and PS-related evaluator management is done by this class. 24 | * Although public, the methods should not be called by the user directly because 25 | * {@code ParameterServerDriver} calls them internally. 26 | */ 27 | @DriverSide 28 | public interface ParameterServerManager { 29 | 30 | /** 31 | * @return service configuration for an Evaluator that uses a {@code ParameterWorker} 32 | */ 33 | Configuration getWorkerServiceConfiguration(); 34 | 35 | /** 36 | * @return service configuration for an Evaluator that uses a {@code ParameterServer} 37 | */ 38 | Configuration getServerServiceConfiguration(); 39 | } 40 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/OutputStreamProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | import java.io.DataOutputStream; 19 | import java.io.IOException; 20 | 21 | /** 22 | * A provider through which users create output streams. 23 | */ 24 | public interface OutputStreamProvider { 25 | 26 | /** 27 | * create an output stream using the given name. 28 | * @param name name of the created output stream. 29 | * it is used as the name of the directory if the created output stream is a file output stream 30 | * @throws java.io.IOException 31 | */ 32 | DataOutputStream create(final String name) throws IOException; 33 | 34 | /** 35 | * set the id of the current task. 36 | * @param taskId id of the current task 37 | */ 38 | void setTaskId(final String taskId); 39 | 40 | /** 41 | * release unused resources. 42 | * @throws IOException 43 | */ 44 | void close() throws IOException; 45 | } 46 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/data/PageRankSummary.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.data; 17 | 18 | import java.util.Map; 19 | 20 | public class PageRankSummary { 21 | private final Map contributionSum; 22 | 23 | public PageRankSummary(final Map contributionSum) { 24 | this.contributionSum = contributionSum; 25 | } 26 | 27 | public void plus(final PageRankSummary summary) { 28 | for (final Map.Entry entry : summary.getModel().entrySet()) { 29 | final Integer nodeId = entry.getKey(); 30 | final Double contribution = entry.getValue(); 31 | 32 | if (contributionSum.containsKey(nodeId)) { 33 | contributionSum.put(nodeId, contributionSum.get(nodeId) + contribution); 34 | } else { 35 | contributionSum.put(nodeId, contribution); 36 | } 37 | } 38 | } 39 | 40 | public Map getModel() { 41 | return contributionSum; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/regularization/L2Regularization.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.regularization; 17 | 18 | import edu.snu.dolphin.bsp.examples.ml.data.Model; 19 | import edu.snu.dolphin.bsp.examples.ml.parameters.Lambda; 20 | import org.apache.mahout.math.Vector; 21 | import org.apache.reef.tang.annotations.Parameter; 22 | 23 | import javax.inject.Inject; 24 | 25 | /** 26 | * Class that represents ||w||_2^2. 27 | */ 28 | public final class L2Regularization implements Regularization { 29 | private double lambda; 30 | 31 | @Inject 32 | public L2Regularization(@Parameter(Lambda.class) final double lambda) { 33 | this.lambda = lambda; 34 | } 35 | 36 | @Override 37 | public double regularize(final Model model) { 38 | return lambda * model.getParameters().getLengthSquared() / 2; 39 | } 40 | 41 | @Override 42 | public Vector gradient(final Model model) { 43 | return model.getParameters().times(lambda); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/loss/Loss.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.loss; 17 | 18 | import org.apache.mahout.math.Vector; 19 | import org.apache.reef.tang.annotations.DefaultImplementation; 20 | 21 | /** 22 | * Interface for regularize function used in SGD jobs. 23 | */ 24 | @DefaultImplementation(SquareLoss.class) 25 | public interface Loss { 26 | 27 | /** 28 | * @param predict estimated value of the dependent variable 29 | * @param output value of dependent variable 30 | * @return loss computed by predict and output 31 | */ 32 | double loss(final double predict, final double output); 33 | 34 | /** 35 | * @param feature values of independent variables 36 | * @param predict estimated value of the dependent variable 37 | * @param output value of dependent variable 38 | * @return gradient of the loss function at the given feature 39 | */ 40 | Vector gradient(final Vector feature, final double predict, final double output); 41 | } 42 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/ml/regularization/Regularization.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.ml.regularization; 17 | 18 | import edu.snu.dolphin.bsp.examples.ml.data.Model; 19 | import org.apache.mahout.math.Vector; 20 | import org.apache.reef.tang.annotations.DefaultImplementation; 21 | 22 | /** 23 | * Regularization factor to be used with the regularize function. 24 | */ 25 | @DefaultImplementation(L2Regularization.class) 26 | public interface Regularization { 27 | 28 | /** 29 | * @param model model to perform computation on 30 | * @return value of regularization term for a certain model 31 | */ 32 | double regularize(final Model model); 33 | 34 | /** 35 | * calculate the gradient of the regularization term for a certain model, 36 | * to use with gradient descent. 37 | * 38 | * @param model model to perform computation on 39 | * @return value of the gradient of the regularization term for a certain model 40 | */ 41 | Vector gradient(final Model model); 42 | } 43 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/simple/SimpleJobInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.simple; 17 | 18 | import edu.snu.dolphin.bsp.core.*; 19 | import org.apache.reef.io.serialization.SerializableCodec; 20 | 21 | import javax.inject.Inject; 22 | import java.util.LinkedList; 23 | import java.util.List; 24 | 25 | public class SimpleJobInfo implements UserJobInfo { 26 | 27 | @Inject 28 | public SimpleJobInfo() { 29 | } 30 | 31 | @Override 32 | public List getStageInfoList() { 33 | final List stageInfoList = new LinkedList<>(); 34 | stageInfoList.add( 35 | StageInfo.newBuilder(SimpleCmpTask.class, SimpleCtrlTask.class, SimpleCommGroup.class) 36 | .setBroadcast(SerializableCodec.class) 37 | .setReduce(SerializableCodec.class, SimpleReduceFunction.class) 38 | .build()); 39 | return stageInfoList; 40 | } 41 | 42 | @Override 43 | public Class getDataParser() { 44 | return SimpleDataParser.class; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/simple/SimpleREEF.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.simple; 17 | 18 | import edu.snu.dolphin.bsp.core.DolphinConfiguration; 19 | import edu.snu.dolphin.bsp.core.DolphinLauncher; 20 | import edu.snu.dolphin.bsp.core.UserJobInfo; 21 | import edu.snu.dolphin.bsp.parameters.JobIdentifier; 22 | import org.apache.reef.tang.Configurations; 23 | import org.apache.reef.tang.Tang; 24 | 25 | public final class SimpleREEF { 26 | 27 | /** 28 | * Should not be instantiated. 29 | */ 30 | private SimpleREEF() { 31 | } 32 | 33 | public static void main(final String[] args) throws Exception { 34 | DolphinLauncher.run( 35 | Configurations.merge( 36 | DolphinConfiguration.getConfiguration(args), 37 | Tang.Factory.getTang().newConfigurationBuilder() 38 | .bindNamedParameter(JobIdentifier.class, "Simple REEF") 39 | .bindImplementation(UserJobInfo.class, SimpleJobInfo.class) 40 | .build() 41 | ) 42 | ); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/examples/simple/SimpleCmpTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.examples.simple; 17 | 18 | import edu.snu.dolphin.bsp.core.UserComputeTask; 19 | import edu.snu.dolphin.bsp.groupcomm.interfaces.DataBroadcastReceiver; 20 | import edu.snu.dolphin.bsp.groupcomm.interfaces.DataReduceSender; 21 | 22 | import javax.inject.Inject; 23 | 24 | public final class SimpleCmpTask extends UserComputeTask 25 | implements DataBroadcastReceiver, DataReduceSender { 26 | private String message = null; 27 | private Integer count = 0; 28 | 29 | @Inject 30 | private SimpleCmpTask() { 31 | } 32 | 33 | @Override 34 | public void run(final int iteration) { 35 | System.out.println(message); 36 | count++; 37 | 38 | } 39 | 40 | @Override 41 | public void receiveBroadcastData(final int iteration, final String data) { 42 | message = data; 43 | count = 0; 44 | } 45 | 46 | @Override 47 | public Integer sendReduceData(final int iteration) { 48 | return count; 49 | } 50 | 51 | } 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /dolphin-bsp/src/main/java/edu/snu/dolphin/bsp/core/KeyValueStoreService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 Seoul National University 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package edu.snu.dolphin.bsp.core; 17 | 18 | 19 | import org.apache.reef.driver.context.ServiceConfiguration; 20 | import org.apache.reef.tang.Configuration; 21 | import org.apache.reef.tang.Tang; 22 | 23 | import java.util.logging.Logger; 24 | 25 | /** 26 | * Key-value store service used to pass the current stage's result to the next stage 27 | * Should be inserted alongside a context. 28 | */ 29 | public final class KeyValueStoreService { 30 | private static final Logger LOG = Logger.getLogger(KeyValueStoreService.class.getName()); 31 | 32 | /** 33 | * Should not be instantiated. 34 | */ 35 | private KeyValueStoreService() { 36 | } 37 | 38 | public static Configuration getServiceConfiguration() { 39 | final Configuration partialServiceConf = ServiceConfiguration.CONF 40 | .set(ServiceConfiguration.SERVICES, KeyValueStore.class) 41 | .build(); 42 | 43 | return Tang.Factory.getTang() 44 | .newConfigurationBuilder(partialServiceConf) 45 | .build(); 46 | } 47 | } 48 | --------------------------------------------------------------------------------